diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96bdb9e3dec589939a9e31008f5d70accbe6c226..20128f48bac5ed423faed7d664012004ac93ef50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,9 +2,10 @@ stages: - deploy variables: - OPENSHIFT_SERVER: https://okd-dev.hbp.eu - BUILD_ENV: test-build - OP: update + OPENSHIFT_SERVER: $OPENSHIFT_DEV_SERVER + INSTALLATION_ROOT: $INSTALLATION_ROOT + SPACKIFIED_ENV: $SPACKIFIED_ENV + OP: $OPERATION #SPACK_ENV_TAR_FILE: ebrains-spack-builds${CI_PIPELINE_ID}.tar.gz SPACK_ENV_TAR_FILE: ebrains-spack-builds.tar.gz @@ -12,13 +13,13 @@ deploy-build-environment: stage: deploy before_script: - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" - - tar czf ${SPACK_ENV_TAR_FILE} packages/ repo.yaml spack.yaml + - tar czf ${SPACK_ENV_TAR_FILE} packages/ repo.yaml spack.yaml create_JupyterLab_kernel.sh - mkdir copy_folder - mv ${SPACK_ENV_TAR_FILE} copy_folder script: # create job description file - chmod a+x create_job.sh - - ./create_job.sh $BUILD_ENV $OP $SPACK_ENV_TAR_FILE $CI_PIPELINE_ID + - ./create_job.sh $INSTALLATION_ROOT $SPACKIFIED_ENV $OP $SPACK_ENV_TAR_FILE $CI_PIPELINE_ID - cat simplejob.yml # select the project in openshift - oc project jupyterhub-int @@ -38,6 +39,7 @@ deploy-build-environment: - if [ $(cat log.txt |grep "Error:"|wc -l) -gt 0 ]; then exit 1;fi; # delete the job from OpenShift as we have the logs here - oc delete job simplejob${CI_PIPELINE_ID} || true + resource_group: shared-NFS-mount tags: - shell-runner diff --git a/README.md b/README.md index be45501e73f1abeca0f7c3557b4bf33a56f39095..ebd275d9c94672199ba0a022330f0e6a74eb9adf 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,26 @@ # ebrains-spack-builds -Build and distribute software with Spack +Build and distribute software tools with Spack for the EBRAINS Collaboratory Lab containers. -**Quickstart** +**Quickstart for Notebooks** +- Open a terminal at a running Collaboratory Lab Container and execute the following: +``` +#update modulepath of JupterLab container to include spack arch modulefiles path +module use /srv/test-build/spack/share/spack/modules/linux-centos7-broadwell/ + +#start JupyterLab kernel made for spack +jupyter kernelspec install --user /srv/jupyterlab_kernels/int/release20210930/spack_python_kernel_release_20210930/ +``` +- Then select File > New Launcher +- Open a new Notebook with the "EBRAINS_release_20210930" kernel +- Import the tools available with spack: +``` +import nest +import arbor +import neuron +``` + +**Quickstart for terminal** - Open a terminal at a running Collaboratory Lab Container and execute the following: ``` git clone https://gitlab.ebrains.eu/akarmas/ebrains-spack-builds.git @@ -19,51 +37,29 @@ Build and distribute software with Spack The following variable(s) must be set up if not or re-configured if tokens expire. -OPENSHIFT_TOKEN: Token to login to OpenShift cluster (with the "gitlab" service account) +OPENSHIFT_TOKEN: Token to login to OpenShift cluster (with the "gitlab" service account) +OPENSHIFT_DEV_SERVER: The URL of the OpenShift Development cluster needed for deploying software in lab-int environment +BUILD_ENV: The name of the environment to deploy the software of the next commit +OPERATION: The operation to perform on the spack environment (one of the following: testing, create, update, delete) ## Copy spack .yaml files and packages to the Openshift job pod that does the build -The gitlab runner copies the spack .yaml files and packages to the OpenShift job pod. +The gitlab runner copies the various files needed for the build to the OpenShift job pod. +- It copies the {spack, repo}.yaml files, the create_JupyterLab_kernel.sh script and the packages/ directory - The runner waits until the job's pod is running to start copying the files -- The pod (built from tc/ebrains-spack-build-env:latest image) waits until the necessary file(s) has finished copying so that it can continue the build process +- The pod (built from [tc/ebrains-spack-build-env:latest image](https://docker-registry.ebrains.eu/harbor/projects/8/repositories/ebrains-spack-build-env)) waits until the necessary file(s) has finished copying so that it can continue the build process ## Bulding software binaries with Spack -**ToDo: The current build path needs to be automated with CI (e.g. gitlab runners)** - -- As a reference you can find [here](https://spack.readthedocs.io/en/latest/command_index.html) the Spack commands list -- We will need to have different Spack environments to test specs (and build packages) and when tests are passing release specs from testing environments to production environment - -- First of all we need to create an appropriate build environment (at a dedicated VM or container image or gitlab runner). The build environment must run on the same OS as the Collaboratory base container image and fulfill all Spack's pre-requisites ( [1](https://ashki23.github.io/spack.html), [2](https://spack.readthedocs.io/en/latest/getting_started.html) ) -- The Spack installation folder at build time must (currently) be: -``` -/opt/app-root/src -``` -to match the Spack installation directory on the current Collab base container image -- At the build environment a recent version of gcc must be running. Install the most appropriate gcc version based on the OS of the build environment. Below you can find the instructions to install the appropriate gcc version for CentOS 7 (current OS of Collaboratory base image). -Instructions to install **"devtoolset-9"** [here](https://linuxize.com/post/how-to-install-gcc-compiler-on-centos-7/). -- Then we can start building and installing tools with Spack: -``` -spack install arbor %gcc@9.3.1 -spack install neuron %gcc@9.3.1 -spack install nest %gcc@9.3.1 +python -``` -and perform tests for the installations: - -https://docs.arbor-sim.org/en/stable/tutorial/single_cell_model.html -https://neuron.yale.edu/neuron/static/docs/neuronpython/firststeps.html -https://nest-simulator.readthedocs.io/en/nest-2.20.1/auto_examples/one_neuron.html - -- (Potentially) fix any potential errors for shared libraries (info [1](https://github.com/cdr/code-server/issues/766), [2](https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html), [3](https://serverkurma.com/linux/how-to-install-and-update-gcc-on-centos-7/)) - -## Delivery of the software binaries - -**ToDo:This methodology will change, and the delivery will be be implemented via Gitlab CI and use of the shared NFS drive** - -- After build is complete, we move on to zip the **"$SPACK_ROOT"** and the **~/.spack** folders of the build environment and transfer them to CSCS Object Storage (currently, in the future it will be a shared NFS drive) -- The artifacts that were built can now be used from the Collaboratory running containers to load and run the available simulation tools +- The build process is powered by Spack a multi-platform package manager that builds and installs multiple versions and configurations of software. +- A Job (object) in OpenShift is responsible for the build process. +- The gitlab runner starts a new Job that runs on an OpenShift pod that uses the container image developed in [this](https://gitlab.ebrains.eu/akarmas/ebrains-spack-build-env/) repository that holds all the Spack specifics needed for the build process. All the Spack configuration necessary for a successfull build is to be changed from the Spack configuration files that are found in the present repository. +- The OpenShift Job's pod mounts an NFS drive that is also mounted by all Collaboratory Lab containers and performs the entire build process with Spack on that NFS drive and as a result all the installed software is readily available to Collaboratory Lab containers +- A schema of the build process can be found [here](https://drive.ebrains.eu/smart-link/6adcd99f-c088-472e-a596-37ac38869051/) ## Activating software in the Collaboratory Lab containers -- Currently to load the pre-built simulation tools in the Collaboratory Lab containers refer to **Quickstart** at the beggining of this file (currently the Object Storage at CSCS is used to download the pre-built software as a stop-gap measure until a shared NFS drive is available). -- This process will change in the future as all simulation tools will be available in the Collaboratory running containers from a shared NFS drive +- Currently to activate the pre-built simulation tools in the Collaboratory Lab containers refer to **Quickstart** at the beggining of this file +- There are two options: i) for using the simulation tools directly in the notebooks and ii) for using the simulation tools from a terminal in a Collaboratory Lab container + +**ToDo: put the necessary activation commands in the startup script of a JupyterLab conatiner spawned in OpenShift to hide all implementation details from the users** diff --git a/create_JupyterLab_kernel.sh b/create_JupyterLab_kernel.sh new file mode 100644 index 0000000000000000000000000000000000000000..4f366ae0b576fcd92f63d2e810e16110968c19fd --- /dev/null +++ b/create_JupyterLab_kernel.sh @@ -0,0 +1,57 @@ +#!/bin/bash +#title :create_JupyterLab_kernel.sh +#description :Script to create a spackified JupyterLab kernel conf and place it to NFS where it can be loaded by all users. +#usage :./create_JupyterLab_kernel.sh $INSTALLATION_ROOT $ENV $LAB_KERNEL_PATH +#============================================================================== + +INSTALLATION_ROOT=$1 +SPACKIFIED_ENV=$2 +LAB_KERNEL_PATH=$3 + +# capture the empty env +cd /opt/app-root/src +env >> before.txt + +# load spack, spack repos and spack env +cp -r /srv/$INSTALLATION_ROOT/spack/.spack ~ +source /srv/$INSTALLATION_ROOT/spack/share/spack/setup-env.sh +spack repo add /srv/$INSTALLATION_ROOT/ebrains-spack-builds +# no need to activate as the env is already activated in the context it is used +#spack env activate $SPACKIFIED_ENV + +module use /srv/$INSTALLATION_ROOT/spack/share/spack/modules/linux-centos7-broadwell/ +source /srv/$INSTALLATION_ROOT/spack/var/spack/environments/$SPACKIFIED_ENV/loads + +# capture the env after spack activation +cd /opt/app-root/src +env >> after.txt + +# prepare the env file required for the JupyterLab kernel +mkdir $LAB_KERNEL_PATH/bin +cat <<EOF > $LAB_KERNEL_PATH/bin/env.sh +#!/usr/bin/env bash +set -euxo pipefail +EOF + +# load here all tools +#spack load --sh -r python@3.8.11 py-ipykernel py-pip py-numpy@1.21.0 py-scipy py-pandas py-seaborn py-matplotlib arbor nest@3.0 neuron py-pynn tvb-data tvb-library meta-brainscales %gcc@10.3.0 >> $LAB_KERNEL_PATH/bin/env.sh + +# append the necessary env variables for spack env and tools +cd /opt/app-root/src +diff before.txt after.txt|grep ">"|cut -c 3- >> $LAB_KERNEL_PATH/bin/env.sh + +# end of env creation +cat <<EOF >>$LAB_KERNEL_PATH/bin/env.sh +python -m ipykernel_launcher -f \$@ +EOF +chmod +x $LAB_KERNEL_PATH/bin/env.sh +# create the new kernel's configuration file +mkdir $LAB_KERNEL_PATH/spack_python_kernel_release_20210930 +cat <<EOF >$LAB_KERNEL_PATH/spack_python_kernel_release_20210930/kernel.json +{ + "argv": ["$LAB_KERNEL_PATH/bin/env.sh", "{connection_file}"], + "display_name": "EBRAINS_release_20210930", + "name": "spack_python_kernel_release_20210930", + "language": "python" +} +EOF diff --git a/create_job.sh b/create_job.sh index 25eee6c04a7743297cb46906c08b9a50d5fd2f57..96172bc367744305c41794e7d38984a150bbc004 100644 --- a/create_job.sh +++ b/create_job.sh @@ -1,9 +1,10 @@ #!/bin/bash -BUILD_ENV=$1 -OP=$2 -SPACK_ENV_TAR_FILE=$3 -OC_JOB_ID=$4 +INSTALLATION_ROOT=$1 +SPACKIFIED_ENV=$2 +OP=$3 +SPACK_ENV_TAR_FILE=$4 +OC_JOB_ID=$5 cat <<EOT >> simplejob.yml apiVersion: batch/v1 @@ -30,7 +31,7 @@ spec: volumeMounts: - name: sharedbin mountPath: /srv - command: ["/usr/local/bin/deploy-build-env.sh", "$BUILD_ENV", "$OP", "$SPACK_ENV_TAR_FILE"] + command: ["/usr/local/bin/deploy-build-env.sh", "$INSTALLATION_ROOT", "$SPACKIFIED_ENV", "$OP", "$SPACK_ENV_TAR_FILE"] volumes: - name: sharedbin persistentVolumeClaim: diff --git a/load_sim_tools.sh b/load_sim_tools.sh index 581c88db21c87454438e7037120c10d9c7cfd722..52765ec7c3373cac4eb73b1d6e4b5cf3f4398420 100644 --- a/load_sim_tools.sh +++ b/load_sim_tools.sh @@ -8,22 +8,17 @@ echo "Setting up environment..." -cp -r /srv/test-build/spack/.spack ~ -source /srv/test-build/spack/share/spack/setup-env.sh +INSTALLATION_ROOT="test-build" +SPACKIFIED_ENV="ebrains-spack-builds" -cd /srv/test-build/ +cp -r /srv/$INSTALLATION_ROOT/spack/.spack ~ +source /srv/$INSTALLATION_ROOT/spack/share/spack/setup-env.sh + +cd /srv/$INSTALLATION_ROOT spack repo add ebrains-spack-builds -cd ~ -echo "Loading packages..." +spack env activate $SPACKIFIED_ENV -spack load -r python %gcc@10.3.0 -spack load -r py-numpy %gcc@10.3.0 -spack load -r py-pip %gcc@10.3.0 -spack load -r arbor %gcc@10.3.0 -spack load -r neuron %gcc@10.3.0 -spack load -r nest@2.20.0 %gcc@10.3.0 -spack load -r py-pynn %gcc@10.3.0 +module use /srv/$INSTALLATION_ROOT/spack/share/spack/modules/linux-centos7-broadwell/ +source /srv/$INSTALLATION_ROOT/spack/var/spack/environments/$SPACKIFIED_ENV/loads echo "Everything ready!" -#echo "Starting Python..." -#python diff --git a/packages/genpybind/package.py b/packages/genpybind/package.py new file mode 100644 index 0000000000000000000000000000000000000000..b2a639b8c0605f322bed8d3e3b0fa4ba9dda03cd --- /dev/null +++ b/packages/genpybind/package.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Genpybind(WafPackage): + """Autogeneration of Python bindings from manually annotated C++ headers""" + + homepage = "https://github.com/kljohann/genpybind" + url = "https://github.com/kljohann/genpybind/archive/v0.1.0.tar.gz" + git = "https://github.com/kljohann/genpybind.git" + + version('0.2.1', sha256='e4d993f0c65cb5cf635cec7df899cbd91af1f0bd8a3626f33e9e0925f5383384') + version('0.2.0', sha256='9d1e9d026a9e355e282aca549a2af108bedcc5bc59ba0b76a6072f88e4c0be4c') + version('0.1.1-pre', commit="9d06a3ad4b6b917c8fcc07261a97b13a3079bcba") + version('0.1.0', sha256='f25cb2b3180103cb96c42fb8d37be8b1f06b7721f6aa08841d5ae16361896407') + version('master', branch='master') + version('develop', branch='develop') + + version('visions', branch='master', git='https://github.com/electronicvisions/genpybind') + version('ebrains', tag='ebrains_release-1-rc1', git='https://github.com/electronicvisions/genpybind') + + depends_on( + 'llvm+clang+python+visionary@5.0.0:', + type=('build', 'link', 'run')) + depends_on('binutils', type='build') + depends_on('python@2.7:', type=('build', 'run')) + + extends('python') + + def configure_args(self): + args = super(Genpybind, self).configure_args() + + if self.spec.satisfies("@visions") or self.spec.satisfies("@ebrains"): + # currently only our HEAD supports the rename + # TODO: adapt once the change is upstream + args.append("--genpybind-disable-tests") + else: + args.append("--disable-tests") + return args diff --git a/packages/llvm/constexpr_longdouble.patch b/packages/llvm/constexpr_longdouble.patch new file mode 100644 index 0000000000000000000000000000000000000000..f825b1421009f1698761943028214564f82ac825 --- /dev/null +++ b/packages/llvm/constexpr_longdouble.patch @@ -0,0 +1,15 @@ +--- a/libcxx/include/thread ++++ b/libcxx/include/thread +@@ -435,7 +435,12 @@ sleep_for(const chrono::duration<_Rep, _Period>& __d) + using namespace chrono; + if (__d > duration<_Rep, _Period>::zero()) + { ++#if ! (defined(_LIBCPP_COMPILER_GCC) && (__powerpc__ || __POWERPC__)) ++ // GCC's long double const folding is incomplete for IBM128 long doubles. + _LIBCPP_CONSTEXPR duration<long double> _Max = nanoseconds::max(); ++#else ++ _LIBCPP_CONSTEXPR duration<long double> _Max = duration<long double>(ULLONG_MAX/1000000000ULL) ; ++#endif + nanoseconds __ns; + if (__d < _Max) + { diff --git a/packages/llvm/constexpr_longdouble_9.0.patch b/packages/llvm/constexpr_longdouble_9.0.patch new file mode 100644 index 0000000000000000000000000000000000000000..bbc5ab7385e54c7269848b0a7a71bb3bac22d552 --- /dev/null +++ b/packages/llvm/constexpr_longdouble_9.0.patch @@ -0,0 +1,12 @@ +-- a/libcxx/include/thread ++++ b/libcxx/include/thread +@@ -370,7 +370,7 @@ sleep_for(const chrono::duration<_Rep, _Period>& __d) + using namespace chrono; + if (__d > duration<_Rep, _Period>::zero()) + { +-#if defined(_LIBCPP_COMPILER_GCC) && (__powerpc__ || __POWERPC__) ++#if ! (defined(_LIBCPP_COMPILER_GCC) && (__powerpc__ || __POWERPC__)) + // GCC's long double const folding is incomplete for IBM128 long doubles. + _LIBCPP_CONSTEXPR duration<long double> _Max = nanoseconds::max(); + #else + diff --git a/packages/llvm/lldb_external_ncurses-10.patch b/packages/llvm/lldb_external_ncurses-10.patch new file mode 100644 index 0000000000000000000000000000000000000000..34ed0e3cd2130cb18e0f72e86e5db1aace562787 --- /dev/null +++ b/packages/llvm/lldb_external_ncurses-10.patch @@ -0,0 +1,31 @@ +diff --git a/lldb/include/lldb/Host/Config.h.cmake b/lldb/include/lldb/Host/Config.h.cmake +--- a/lldb/include/lldb/Host/Config.h.cmake ++++ b/lldb/include/lldb/Host/Config.h.cmake +@@ -38,6 +38,8 @@ + + #cmakedefine01 LLDB_ENABLE_CURSES + ++#cmakedefine01 CURSES_HAVE_NCURSES_CURSES_H ++ + #cmakedefine01 LLDB_ENABLE_LIBEDIT + + #cmakedefine01 LLDB_ENABLE_LIBXML2 +diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp +--- a/lldb/source/Core/IOHandlerCursesGUI.cpp ++++ b/lldb/source/Core/IOHandlerCursesGUI.cpp +@@ -10,9 +10,14 @@ + #include "lldb/Host/Config.h" + + #if LLDB_ENABLE_CURSES ++#if CURSES_HAVE_NCURSES_CURSES_H ++#include <ncurses/curses.h> ++#include <ncurses/panel.h> ++#else + #include <curses.h> + #include <panel.h> + #endif ++#endif + + #if defined(__APPLE__) + #include <deque> + diff --git a/packages/llvm/llvm11-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch b/packages/llvm/llvm11-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch new file mode 100644 index 0000000000000000000000000000000000000000..8946be6a511291cd05dafcb925d8ce60dfa063de --- /dev/null +++ b/packages/llvm/llvm11-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch @@ -0,0 +1,162 @@ +From e673a5527dd2df322884eb2498736483df05957d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 11:17:59 +0100 +Subject: [PATCH 2/5] [libclang] Add support for obtaining fully qualified + names of types + +This patch allows retrieving the fully qualified names of types +through libclang and clang.cindex (Python). +--- + clang/bindings/python/clang/cindex.py | 13 +++++++++++ + .../python/tests/cindex/test_cursor.py | 8 +++++++ + clang/include/clang-c/Index.h | 10 ++++++++- + clang/tools/libclang/CMakeLists.txt | 1 + + clang/tools/libclang/CXType.cpp | 22 +++++++++++++++++++ + clang/tools/libclang/libclang.exports | 1 + + 6 files changed, 54 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 8e5a9fe0068..c309f7017b2 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -2427,6 +2427,14 @@ class Type(Structure): + """Retrieve the spelling of this Type.""" + return conf.lib.clang_getTypeSpelling(self) + ++ @property ++ def fully_qualified_name(self): ++ """Retrieve the fully qualified name of this Type.""" ++ if not hasattr(self, '_fully_qualified_name'): ++ self._fully_qualified_name = conf.lib.clang_getFullyQualifiedTypeName(self) ++ ++ return self._fully_qualified_name ++ + def __eq__(self, other): + if type(other) != type(self): + return False +@@ -3869,6 +3877,11 @@ functionList = [ + _CXString, + _CXString.from_result), + ++ ("clang_getFullyQualifiedTypeName", ++ [Type], ++ _CXString, ++ _CXString.from_result), ++ + ("clang_hashCursor", + [Cursor], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index ef875e97247..6a53c7205df 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -316,6 +316,14 @@ class TestCursor(unittest.TestCase): + underlying = typedef.underlying_typedef_type + self.assertEqual(underlying.kind, TypeKind.INT) + ++ def test_fully_qualified_type_name(): ++ source = 'namespace uiae { struct X { typedef int sometype; }; }' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'sometype') ++ fqn = cls.type.fully_qualified_name ++ self.assertTrue(fqn.endswith("uiae::X::sometype"), fqn) ++ + def test_semantic_parent(self): + tu = get_tu(kParentTest, 'cpp') + curs = get_cursors(tu, 'f') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 74badac740b..b0c62fe948e 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 60 ++#define CINDEX_VERSION_MINOR 61 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -3389,6 +3389,14 @@ CINDEX_LINKAGE CXType clang_getCursorType(CXCursor C); + */ + CINDEX_LINKAGE CXString clang_getTypeSpelling(CXType CT); + ++/** ++ * Retrieve the fully qualified name of the underlying type. ++ * This includes full qualification of all template parameters etc. ++ * ++ * If the type is invalid, an empty string is returned. ++ */ ++CINDEX_LINKAGE CXString clang_getFullyQualifiedTypeName(CXType CT); ++ + /** + * Retrieve the underlying type of a typedef declaration. + * +diff --git a/tools/clang/tools/libclang/CMakeLists.txt b/tools/clang/tools/libclang/CMakeLists.txt +index 613ead1a36b..a583fa206d1 100644 +--- a/tools/clang/tools/libclang/CMakeLists.txt ++++ b/tools/clang/tools/libclang/CMakeLists.txt +@@ -43,6 +43,7 @@ set(LIBS + clangSema + clangSerialization + clangTooling ++ clangToolingCore + ) + + if (CLANG_ENABLE_ARCMT) +diff --git a/tools/clang/tools/libclang/CXType.cpp b/tools/clang/tools/libclang/CXType.cpp +index acecf87d0cd..afdeb467769 100644 +--- a/tools/clang/tools/libclang/CXType.cpp ++++ b/tools/clang/tools/libclang/CXType.cpp +@@ -19,6 +19,7 @@ + #include "clang/AST/DeclObjC.h" + #include "clang/AST/DeclTemplate.h" + #include "clang/AST/Expr.h" ++#include "clang/AST/QualTypeNames.h" + #include "clang/AST/Type.h" + #include "clang/Basic/AddressSpaces.h" + #include "clang/Frontend/ASTUnit.h" +@@ -302,6 +303,27 @@ CXString clang_getTypeSpelling(CXType CT) { + return cxstring::createDup(OS.str()); + } + ++CXString clang_getFullyQualifiedTypeName(CXType CT) { ++ QualType T = GetQualType(CT); ++ if (T.isNull()) ++ return cxstring::createEmpty(); ++ ++ // For builtin types (but not typedefs pointing to builtin types) return their ++ // spelling. Otherwise "bool" will be turned into "_Bool". ++ const Type *TP = T.getTypePtrOrNull(); ++ if (TP && TP->isBuiltinType() && T->getAs<TypedefType>() == nullptr) ++ return clang_getTypeSpelling(CT); ++ ++ CXTranslationUnit TU = GetTU(CT); ++ ASTContext &Ctx = cxtu::getASTUnit(TU)->getASTContext(); ++ PrintingPolicy Policy(Ctx.getPrintingPolicy()); ++ Policy.SuppressScope = false; ++ Policy.AnonymousTagLocations = false; ++ Policy.PolishForDeclaration = true; ++ std::string name = TypeName::getFullyQualifiedName(T, Ctx, Policy, /*WithGlobalNsPrefix=*/true); ++ return cxstring::createDup(name.c_str()); ++} ++ + CXType clang_getTypedefDeclUnderlyingType(CXCursor C) { + using namespace cxcursor; + CXTranslationUnit TU = cxcursor::getCursorTU(C); +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 3c76090d64f..6e860e7263e 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -241,6 +241,7 @@ clang_getFileLocation + clang_getFileName + clang_getFileTime + clang_getFileUniqueID ++clang_getFullyQualifiedTypeName + clang_getFunctionTypeCallingConv + clang_getIBOutletCollectionType + clang_getIncludedFile +-- +2.23.0 + diff --git a/packages/llvm/llvm11-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch b/packages/llvm/llvm11-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch new file mode 100644 index 0000000000000000000000000000000000000000..03b410710f1ed20e969ecf7dd00bbc76a4c703db --- /dev/null +++ b/packages/llvm/llvm11-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch @@ -0,0 +1,261 @@ +From 075a7a3e667fe3d923de6d7a6929e61922c8b139 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 21:19:51 +0100 +Subject: [PATCH 3/5] [libclang] Add option to keep whitespace when tokenizing + +Introduces new `clang_tokenizeRange` function which accepts options to control +tokenization behavior. `clang_tokenize` is kept for backwards compatibility. +--- + clang/bindings/python/clang/cindex.py | 31 ++++++++++++++---- + .../python/tests/cindex/test_cursor.py | 9 ++++++ + clang/include/clang-c/Index.h | 32 +++++++++++++++++-- + clang/tools/libclang/CIndex.cpp | 15 +++++++-- + clang/tools/libclang/libclang.exports | 1 + + 5 files changed, 75 insertions(+), 13 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index c309f7017b2..1589acc9e7e 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -529,6 +529,13 @@ class TokenGroup(object): + + You should not instantiate this class outside of this module. + """ ++ ++ # Default tokenization mode. ++ TOKENIZE_NONE = 0 ++ ++ # Used to indicate that tokens for whitespace should be returned. ++ TOKENIZE_KEEP_WHITESPACE = 1 ++ + def __init__(self, tu, memory, count): + self._tu = tu + self._memory = memory +@@ -538,7 +545,7 @@ class TokenGroup(object): + conf.lib.clang_disposeTokens(self._tu, self._memory, self._count) + + @staticmethod +- def get_tokens(tu, extent): ++ def get_tokens(tu, extent, options=0): + """Helper method to return all tokens in an extent. + + This functionality is needed multiple places in this module. We define +@@ -547,8 +554,8 @@ class TokenGroup(object): + tokens_memory = POINTER(Token)() + tokens_count = c_uint() + +- conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), +- byref(tokens_count)) ++ conf.lib.clang_tokenizeRange( ++ tu, extent, byref(tokens_memory), byref(tokens_count), options) + + count = int(tokens_count.value) + +@@ -1852,13 +1859,16 @@ class Cursor(Structure): + for descendant in child.walk_preorder(): + yield descendant + +- def get_tokens(self): ++ def get_tokens(self, options=0): + """Obtain Token instances formulating that compose this Cursor. + + This is a generator for Token instances. It returns all tokens which + occupy the extent this cursor occupies. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ +- return TokenGroup.get_tokens(self._tu, self.extent) ++ return TokenGroup.get_tokens(self._tu, self.extent, options) + + def get_field_offsetof(self): + """Returns the offsetof the FIELD_DECL pointed by this Cursor.""" +@@ -3080,18 +3090,21 @@ class TranslationUnit(ClangObject): + return CodeCompletionResults(ptr) + return None + +- def get_tokens(self, locations=None, extent=None): ++ def get_tokens(self, locations=None, extent=None, options=0): + """Obtain tokens in this translation unit. + + This is a generator for Token instances. The caller specifies a range + of source code to obtain tokens for. The range can be specified as a + 2-tuple of SourceLocation or as a SourceRange. If both are defined, + behavior is undefined. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ + if locations is not None: + extent = SourceRange(start=locations[0], end=locations[1]) + +- return TokenGroup.get_tokens(self, extent) ++ return TokenGroup.get_tokens(self, extent, options) + + class File(ClangObject): + """ +@@ -3969,6 +3982,10 @@ functionList = [ + ("clang_tokenize", + [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]), + ++ ("clang_tokenizeRange", ++ [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint), ++ c_uint]), ++ + ("clang_visitChildren", + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 6a53c7205df..0965c1f4ae1 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -10,6 +10,7 @@ import unittest + from clang.cindex import AvailabilityKind + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind ++from clang.cindex import TokenGroup + from clang.cindex import TranslationUnit + from clang.cindex import TypeKind + from .util import get_cursor +@@ -488,6 +489,14 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tokens[0].spelling, 'int') + self.assertEqual(tokens[1].spelling, 'foo') + ++ def test_get_tokens_with_whitespace(): ++ source = 'class C { void f(); }\nvoid C::f() { }' ++ tu = get_tu(source) ++ ++ tokens = list(tu.cursor.get_tokens(TokenGroup.TOKENIZE_KEEP_WHITESPACE)) ++ self.assertEqual(''.join(t.spelling for t in tokens), source) ++ self.assertEqual(len(tokens), 27, [t.spelling for t in tokens]) ++ + def test_get_token_cursor(self): + """Ensure we can map tokens to cursors.""" + tu = get_tu('class A {}; int foo(A var = A());', lang='cpp') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index b0c62fe948e..84ed03b8920 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 61 ++#define CINDEX_VERSION_MINOR 62 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -4969,6 +4969,28 @@ CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit, + */ + CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + ++typedef enum { ++ /** ++ * \brief Used to indicate that no special tokenization options are needed. ++ */ ++ CXTokenize_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that tokens for whitespace should be returned. ++ */ ++ CXTokenize_KeepWhitespace = 0x1 ++} CXTokenize_Flags; ++ ++/** ++ * \brief Tokenize the source code described by the given range into raw ++ * lexical tokens. ++ * ++ * \see clang_tokenizeRange ++ * ++ */ ++CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens); ++ + /** + * Tokenize the source code described by the given range into raw + * lexical tokens. +@@ -4985,9 +5007,13 @@ CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + * \param NumTokens will be set to the number of tokens in the \c *Tokens + * array. + * ++ * \param options A bitmask of options that affects tokenization. This should be ++ * a bitwise OR of the CXTokenize_XXX flags. ++ * + */ +-CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, +- CXToken **Tokens, unsigned *NumTokens); ++CINDEX_LINKAGE void clang_tokenizeRange(CXTranslationUnit TU, ++ CXSourceRange Range, CXToken **Tokens, ++ unsigned *NumTokens, unsigned options); + + /** + * Annotate the given set of tokens by providing cursors for each token +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 1dc961f58a2..3a283e76ed8 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -6670,7 +6670,7 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) { + } + + static void getTokens(ASTUnit *CXXUnit, SourceRange Range, +- SmallVectorImpl<CXToken> &CXTokens) { ++ SmallVectorImpl<CXToken> &CXTokens, unsigned options) { + SourceManager &SourceMgr = CXXUnit->getSourceManager(); + std::pair<FileID, unsigned> BeginLocInfo + = SourceMgr.getDecomposedSpellingLoc(Range.getBegin()); +@@ -6692,6 +6692,9 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + CXXUnit->getASTContext().getLangOpts(), + Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end()); + Lex.SetCommentRetentionState(true); ++ if (options & CXTokenize_KeepWhitespace) { ++ Lex.SetKeepWhitespaceMode(true); ++ } + + // Lex tokens until we hit the end of the range. + const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second; +@@ -6765,7 +6768,7 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + SourceLocation End = SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second); + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, SourceRange(Begin, End), CXTokens); ++ getTokens(CXXUnit, SourceRange(Begin, End), CXTokens, CXTokenize_None); + + if (CXTokens.empty()) + return NULL; +@@ -6913,6 +6913,12 @@ CXToken *clang_getToken(CXTranslationUni + + void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, CXToken **Tokens, + unsigned *NumTokens) { ++ return clang_tokenizeRange(TU, Range, Tokens, NumTokens, CXTokenize_None); ++} ++ ++void clang_tokenizeRange(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens, ++ unsigned options) { + LOG_FUNC_SECTION { *Log << TU << ' ' << Range; } + + if (Tokens) +@@ -6804,7 +6813,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + return; + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, R, CXTokens); ++ getTokens(CXXUnit, R, CXTokens, options); + + if (CXTokens.empty()) + return; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6e860e7263e..6af6c0ca3e8 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -338,6 +338,7 @@ clang_suspendTranslationUnit + clang_sortCodeCompletionResults + clang_toggleCrashRecovery + clang_tokenize ++clang_tokenizeRange + clang_CompilationDatabase_fromDirectory + clang_CompilationDatabase_dispose + clang_CompilationDatabase_getCompileCommands +-- +2.23.0 + diff --git a/packages/llvm/llvm11-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch b/packages/llvm/llvm11-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch new file mode 100644 index 0000000000000000000000000000000000000000..2b2d0e693e60e0ae8bdb93b16b93e419c928472d --- /dev/null +++ b/packages/llvm/llvm11-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch @@ -0,0 +1,437 @@ +From d10d66b8e762c1dd1329d5920f9ef952cf4f6940 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 17 Jul 2017 12:25:49 +0200 +Subject: [PATCH 4/5] [libclang] WIP: Allow visiting of implicit declarations + and template instantiations + +--- + clang/bindings/python/clang/cindex.py | 45 +++++-- + .../python/tests/cindex/test_cursor.py | 33 ++++++ + clang/include/clang-c/Index.h | 33 +++++- + clang/tools/libclang/CIndex.cpp | 112 ++++++++++++++++-- + clang/tools/libclang/CursorVisitor.h | 12 +- + clang/tools/libclang/libclang.exports | 2 + + 6 files changed, 220 insertions(+), 17 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index 1589acc9e7e..b023be6cdc8 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1426,6 +1426,15 @@ class Cursor(Structure): + """ + _fields_ = [("_kind_id", c_int), ("xdata", c_int), ("data", c_void_p * 3)] + ++ # Default behavior. ++ GET_CHILDREN_NONE = 0 ++ ++ # Used to indicate that implicit cursors should be visited. ++ GET_CHILDREN_WITH_IMPLICIT = 1 ++ ++ # Used to indicate that template instantiations should be visited. ++ GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS = 2 ++ + @staticmethod + def from_location(tu, location): + # We store a reference to the TU in the instance so the TU won't get +@@ -1515,6 +1524,10 @@ class Cursor(Structure): + """ + return conf.lib.clang_EnumDecl_isScoped(self) + ++ def is_implicit(self): ++ """Test whether the cursor refers to an implicit declaration.""" ++ return conf.lib.clang_isImplicit(self) ++ + def get_definition(self): + """ + If the cursor is a reference to a declaration or a declaration of +@@ -1831,8 +1844,12 @@ class Cursor(Structure): + """Returns the value of the indicated arg as an unsigned 64b integer.""" + return conf.lib.clang_Cursor_getTemplateArgumentUnsignedValue(self, num) + +- def get_children(self): +- """Return an iterator for accessing the children of this cursor.""" ++ def get_children(self, with_implicit=False, with_template_instantiations=False): ++ """Return an iterator for accessing the children of this cursor. ++ ++ By default, cursors representing implicit declarations or template instantiations ++ will be skipped. ++ """ + + # FIXME: Expose iteration from CIndex, PR6125. + def visitor(child, parent, children): +@@ -1845,18 +1862,24 @@ class Cursor(Structure): + children.append(child) + return 1 # continue + children = [] +- conf.lib.clang_visitChildren(self, callbacks['cursor_visit'](visitor), +- children) ++ dispatch = conf.lib.clang_visitChildren ++ options = Cursor.GET_CHILDREN_NONE ++ if with_implicit: ++ options |= Cursor.GET_CHILDREN_WITH_IMPLICIT ++ if with_template_instantiations: ++ options |= Cursor.GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS ++ conf.lib.clang_visitChildrenWithOptions( ++ self, callbacks['cursor_visit'](visitor), children, options) + return iter(children) + +- def walk_preorder(self): ++ def walk_preorder(self, **kwargs): + """Depth-first preorder walk over the cursor and its descendants. + + Yields cursors. + """ + yield self +- for child in self.get_children(): +- for descendant in child.walk_preorder(): ++ for child in self.get_children(**kwargs): ++ for descendant in child.walk_preorder(**kwargs): + yield descendant + + def get_tokens(self, options=0): +@@ -3927,6 +3950,10 @@ functionList = [ + [Type], + bool), + ++ ("clang_isImplicit", ++ [Cursor], ++ bool), ++ + ("clang_isInvalid", + [CursorKind], + bool), +@@ -3990,6 +4017,10 @@ functionList = [ + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), + ++ ("clang_visitChildrenWithOptions", ++ [Cursor, callbacks['cursor_visit'], py_object, c_uint], ++ c_uint), ++ + ("clang_Cursor_getNumArguments", + [Cursor], + c_int), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 0965c1f4ae1..d061f37c25c 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -94,6 +94,39 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tu_nodes[2].displayname, 'f0(int, int)') + self.assertEqual(tu_nodes[2].is_definition(), True) + ++ def test_get_children_with_implicit(): ++ tu = get_tu('struct X {}; X x;', lang='cpp') ++ cursor = get_cursor(tu, 'X') ++ ++ children = list(cursor.get_children()) ++ self.assertEqual(len(children), 0, [(c.kind, c.spelling) for c in children]) ++ ++ children = list(cursor.get_children(with_implicit=True)) ++ self.assertNotEqual(len(children), 0) ++ for child in children: ++ self.assertTrue(child.is_implicit()) ++ self.assertEqual(child.spelling, "X") ++ self.assertIn(child.kind, [CursorKind.CONSTRUCTOR, CursorKind.STRUCT_DECL]) ++ ++ def test_get_children_with_template_instantiations(): ++ tu = get_tu( ++ 'template <typename T> T frobnicate(T val);' ++ 'extern template int frobnicate<int>(int);', ++ lang='cpp') ++ cursor = get_cursor(tu, 'frobnicate') ++ self.assertEqual(cursor.kind, CursorKind.FUNCTION_TEMPLATE) ++ ++ for child in cursor.get_children(): ++ # should not return an instantiation: ++ self.assertNotEqual(child.kind, CursorKind.FUNCTION_DECL) ++ ++ for child in cursor.get_children(with_template_instantiations=True): ++ if child.kind == CursorKind.FUNCTION_DECL: ++ self.assertEqual(child.spelling, 'frobnicate') ++ break ++ else: ++ self.fail("Couldn't find template instantiation") ++ + def test_references(self): + """Ensure that references to TranslationUnit are kept.""" + tu = get_tu('int x;') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index 84ed03b8920..57acbcba143 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 62 ++#define CINDEX_VERSION_MINOR 63 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -2775,6 +2775,11 @@ CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind); + */ + CINDEX_LINKAGE unsigned clang_isUnexposed(enum CXCursorKind); + ++/*** ++ * \brief Determine whether the given cursor represents an implicit declaration. ++ */ ++CINDEX_LINKAGE unsigned clang_isImplicit(CXCursor); ++ + /** + * Describe the linkage of the entity referred to by a cursor. + */ +@@ -4199,6 +4204,32 @@ CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent, + # endif + #endif + ++typedef enum { ++ /** ++ * \brief Default behavior. ++ */ ++ CXVisitChildren_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that implicit cursors should be visited. ++ */ ++ CXVisitChildren_WithImplicit = 0x1, ++ ++ /** ++ * \brief Used to indicate that template instantiations should be visited. ++ */ ++ CXVisitChildren_WithTemplateInstantiations = 0x2 ++} CXVisitChildren_Flags; ++ ++/** ++ * \brief Visits the children of a cursor, allowing to pass extra options. ++ * Behaves identically to clang_visitChildren() in all other respects. ++ */ ++CINDEX_LINKAGE unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options); ++ + /** + * @} + */ +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 3a283e76ed8..7ee6b704647 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -196,9 +196,10 @@ bool CursorVisitor::Visit(CXCursor Curso + return true; // abort. + } + +- // Ignore implicit declarations, unless it's an objc method because +- // currently we should report implicit methods for properties when indexing. +- if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) ++ // Unless instructed otherwise we ignore implicit declarations. ++ // ObjC methods are currently visited in any case, because implicit methods ++ // for properties should be reported when indexing. ++ if (!VisitImplicitDeclarations && D->isImplicit() && !isa<ObjCMethodDecl>(D)) + return false; + } + +@@ -706,10 +706,13 @@ bool CursorVisitor::VisitTagDecl(TagDecl + + bool CursorVisitor::VisitClassTemplateSpecializationDecl( + ClassTemplateSpecializationDecl *D) { +- bool ShouldVisitBody = false; ++ bool ShouldVisitBody = VisitTemplateInstantiations; + switch (D->getSpecializationKind()) { +- case TSK_Undeclared: + case TSK_ImplicitInstantiation: ++ if (VisitTemplateInstantiations && VisitImplicitDeclarations) { ++ break; ++ } ++ case TSK_Undeclared: + // Nothing to visit + return false; + +@@ -715,6 +719,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl( + break; + + case TSK_ExplicitSpecialization: ++ // Always visit body of explicit specializations + ShouldVisitBody = true; + break; + } +@@ -938,7 +941,31 @@ bool CursorVisitor::VisitFunctionTemplat + return true; + + auto *FD = D->getTemplatedDecl(); +- return VisitAttributes(FD) || VisitFunctionDecl(FD); ++ if (VisitAttributes(FD) || VisitFunctionDecl(FD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *FD : D->specializations()) { ++ for (auto *RD : FD->redecls()) { ++ switch (RD->getTemplateSpecializationKind()) { ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -949,6 +976,40 @@ bool CursorVisitor::VisitClassTemplateDe + + auto *CD = D->getTemplatedDecl(); + return VisitAttributes(CD) || VisitCXXRecordDecl(CD); ++ if (VisitAttributes(CD) || VisitCXXRecordDecl(CD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *SD : D->specializations()) { ++ for (auto *RD : SD->redecls()) { ++ // We don't want to visit injected-class-names in this traversal. ++ if (cast<CXXRecordDecl>(RD)->isInjectedClassName()) ++ continue; ++ ++ switch ( ++ cast<ClassTemplateSpecializationDecl>(RD)->getSpecializationKind()) { ++ // Visit the implicit instantiations with the requested pattern. ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ // We don't need to do anything on an explicit instantiation ++ // or explicit specialization because there will be an explicit ++ // node for it elsewhere. ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +@@ -4426,6 +4488,24 @@ unsigned clang_visitChildrenWithBlock(CXCursor parent, + return clang_visitChildren(parent, visitWithBlock, block); + } + ++unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options) { ++ CursorVisitor CursorVis( ++ getCursorTU(parent), visitor, client_data, ++ /*VisitPreprocessorLast=*/false, ++ /*VisitIncludedPreprocessingEntries=*/false, ++ /*RegionOfInterest=*/SourceRange(), ++ /*VisitDeclsOnly=*/false, ++ /*PostChildrenVisitor=*/nullptr, ++ /*VisitImplicitDeclarations=*/(options & CXVisitChildren_WithImplicit), ++ /*VisitTemplateInstantiations=*/ ++ (options & CXVisitChildren_WithTemplateInstantiations)); ++ ++ return CursorVis.VisitChildren(parent); ++} ++ + static CXString getDeclSpelling(const Decl *D) { + if (!D) + return cxstring::createEmpty(); +@@ -5909,6 +5970,22 @@ unsigned clang_isUnexposed(enum CXCursor + } + } + ++unsigned clang_isImplicit(CXCursor Cursor) { ++ if (clang_isInvalid(Cursor.kind)) ++ return false; ++ ++ if (!clang_isDeclaration(Cursor.kind)) ++ return false; ++ ++ const Decl *D = getCursorDecl(Cursor); ++ if (!D) { ++ assert(0 && "Invalid declaration cursor"); ++ return true; // abort. ++ } ++ ++ return D->isImplicit(); ++} ++ + CXCursorKind clang_getCursorKind(CXCursor C) { return C.kind; } + + CXSourceLocation clang_getCursorLocation(CXCursor C) { +diff --git a/tools/clang/tools/libclang/CursorVisitor.h b/clang/tools/libclang/CursorVisitor.h +index b0afa5a0b59..94f4596d5fa 100644 +--- a/tools/clang/tools/libclang/CursorVisitor.h ++++ b/tools/clang/tools/libclang/CursorVisitor.h +@@ -95,6 +95,12 @@ private: + /// record entries. + bool VisitDeclsOnly; + ++ /// \brief Whether we should visit implicit declarations. ++ bool VisitImplicitDeclarations; ++ ++ /// \brief Whether we should recurse into template instantiations. ++ bool VisitTemplateInstantiations; ++ + // FIXME: Eventually remove. This part of a hack to support proper + // iteration over all Decls contained lexically within an ObjC container. + DeclContext::decl_iterator *DI_current; +@@ -152,12 +152,16 @@ public: + bool VisitIncludedPreprocessingEntries = false, + SourceRange RegionOfInterest = SourceRange(), + bool VisitDeclsOnly = false, +- PostChildrenVisitorTy PostChildrenVisitor = nullptr) ++ PostChildrenVisitorTy PostChildrenVisitor = nullptr, ++ bool VisitImplicitDeclarations = false, ++ bool VisitTemplateInstantiations = false) + : TU(TU), AU(cxtu::getASTUnit(TU)), Visitor(Visitor), + PostChildrenVisitor(PostChildrenVisitor), ClientData(ClientData), + VisitPreprocessorLast(VisitPreprocessorLast), + VisitIncludedEntities(VisitIncludedPreprocessingEntries), + RegionOfInterest(RegionOfInterest), VisitDeclsOnly(VisitDeclsOnly), ++ VisitImplicitDeclarations(VisitImplicitDeclarations), ++ VisitTemplateInstantiations(VisitTemplateInstantiations), + DI_current(nullptr), FileDI_current(nullptr) { + Parent.kind = CXCursor_NoDeclFound; + Parent.data[0] = nullptr; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6af6c0ca3e8..d17eb83187d 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -313,6 +313,7 @@ clang_isInvalidDeclaration + clang_isExpression + clang_isFileMultipleIncludeGuarded + clang_isFunctionTypeVariadic ++clang_isImplicit + clang_isInvalid + clang_isPODType + clang_isPreprocessing +@@ -354,6 +355,7 @@ clang_CompileCommand_getNumArgs + clang_CompileCommand_getArg + clang_visitChildren + clang_visitChildrenWithBlock ++clang_visitChildrenWithOptions + clang_ModuleMapDescriptor_create + clang_ModuleMapDescriptor_dispose + clang_ModuleMapDescriptor_setFrameworkModuleName +-- +2.23.0 + diff --git a/packages/llvm/llvm11-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch b/packages/llvm/llvm11-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch new file mode 100644 index 0000000000000000000000000000000000000000..47f035993becf92561a8a9703c5da6925050a5d7 --- /dev/null +++ b/packages/llvm/llvm11-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch @@ -0,0 +1,52 @@ +From 0fa84bbd8c6f8eb8eb6a3bc30e4efe3f2cf4c283 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 31 Jul 2017 14:09:52 +0200 +Subject: [PATCH 5/5] [libclang] WIP: Fix get_tokens in macro expansion + +--- + clang/bindings/python/tests/cindex/test_cursor.py | 15 +++++++++++++++ + clang/tools/libclang/CIndex.cpp | 2 +- + 2 files changed, 16 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index d061f37c25c..38702df74f2 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -552,6 +552,21 @@ class TestCursor(unittest.TestCase): + r_cursor = t_cursor.referenced # should not raise an exception + self.assertEqual(r_cursor.kind, CursorKind.CLASS_DECL) + ++ def test_get_tokens_in_macro_expansion(self): ++ """regression test""" ++ source = "#define IMPL(name) struct name { name(int v = 123); }; \n IMPL(X)" ++ tu = get_tu(source, lang="cpp") ++ ctor = get_cursors(tu, "X")[1] ++ self.assertEqual(ctor.kind, CursorKind.CONSTRUCTOR) ++ p = next(ctor.get_children()) ++ self.assertEqual(p.kind, CursorKind.PARM_DECL, (p.kind, p.spelling)) ++ children = list(p.get_children()) ++ self.assertEqual(len(children), 1, [(c.kind, c.spelling) for c in children]) ++ expr = children[0] ++ tokens = list(expr.get_tokens()) ++ self.assertEqual(len(tokens), 1, [t.spelling for t in tokens]) ++ self.assertEqual(tokens[0].spelling, "123") ++ + def test_get_arguments(self): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 7ee6b704647..e4b95436704 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -147,7 +147,7 @@ CXSourceRange cxloc::translateSourceRang + // location accordingly. + SourceLocation EndLoc = R.getEnd(); + bool IsTokenRange = R.isTokenRange(); +- if (EndLoc.isValid() && EndLoc.isMacroID() && ++ if (false && EndLoc.isValid() && EndLoc.isMacroID() && + !SM.isMacroArgExpansion(EndLoc)) { + CharSourceRange Expansion = SM.getExpansionRange(EndLoc); + EndLoc = Expansion.getEnd(); +-- +2.23.0 + diff --git a/packages/llvm/llvm11_1-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch b/packages/llvm/llvm11_1-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch new file mode 100644 index 0000000000000000000000000000000000000000..d63fd40099c6c4738f67769ab8cc148a3868c783 --- /dev/null +++ b/packages/llvm/llvm11_1-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch @@ -0,0 +1,162 @@ +From e673a5527dd2df322884eb2498736483df05957d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 11:17:59 +0100 +Subject: [PATCH 2/5] [libclang] Add support for obtaining fully qualified + names of types + +This patch allows retrieving the fully qualified names of types +through libclang and clang.cindex (Python). +--- + clang/bindings/python/clang/cindex.py | 13 +++++++++++ + .../python/tests/cindex/test_cursor.py | 8 +++++++ + clang/include/clang-c/Index.h | 10 ++++++++- + clang/tools/libclang/CMakeLists.txt | 1 + + clang/tools/libclang/CXType.cpp | 22 +++++++++++++++++++ + clang/tools/libclang/libclang.exports | 1 + + 6 files changed, 54 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 8e5a9fe0068..c309f7017b2 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -2427,6 +2427,14 @@ class Type(Structure): + """Retrieve the spelling of this Type.""" + return conf.lib.clang_getTypeSpelling(self) + ++ @property ++ def fully_qualified_name(self): ++ """Retrieve the fully qualified name of this Type.""" ++ if not hasattr(self, '_fully_qualified_name'): ++ self._fully_qualified_name = conf.lib.clang_getFullyQualifiedTypeName(self) ++ ++ return self._fully_qualified_name ++ + def __eq__(self, other): + if type(other) != type(self): + return False +@@ -3869,6 +3877,11 @@ functionList = [ + _CXString, + _CXString.from_result), + ++ ("clang_getFullyQualifiedTypeName", ++ [Type], ++ _CXString, ++ _CXString.from_result), ++ + ("clang_hashCursor", + [Cursor], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index ef875e97247..6a53c7205df 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -316,6 +316,14 @@ class TestCursor(unittest.TestCase): + underlying = typedef.underlying_typedef_type + self.assertEqual(underlying.kind, TypeKind.INT) + ++ def test_fully_qualified_type_name(): ++ source = 'namespace uiae { struct X { typedef int sometype; }; }' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'sometype') ++ fqn = cls.type.fully_qualified_name ++ self.assertTrue(fqn.endswith("uiae::X::sometype"), fqn) ++ + def test_semantic_parent(self): + tu = get_tu(kParentTest, 'cpp') + curs = get_cursors(tu, 'f') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 74badac740b..b0c62fe948e 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 61 ++#define CINDEX_VERSION_MINOR 62 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -3389,6 +3389,14 @@ CINDEX_LINKAGE CXType clang_getCursorType(CXCursor C); + */ + CINDEX_LINKAGE CXString clang_getTypeSpelling(CXType CT); + ++/** ++ * Retrieve the fully qualified name of the underlying type. ++ * This includes full qualification of all template parameters etc. ++ * ++ * If the type is invalid, an empty string is returned. ++ */ ++CINDEX_LINKAGE CXString clang_getFullyQualifiedTypeName(CXType CT); ++ + /** + * Retrieve the underlying type of a typedef declaration. + * +diff --git a/tools/clang/tools/libclang/CMakeLists.txt b/tools/clang/tools/libclang/CMakeLists.txt +index 613ead1a36b..a583fa206d1 100644 +--- a/tools/clang/tools/libclang/CMakeLists.txt ++++ b/tools/clang/tools/libclang/CMakeLists.txt +@@ -43,6 +43,7 @@ set(LIBS + clangSema + clangSerialization + clangTooling ++ clangToolingCore + ) + + if (CLANG_ENABLE_ARCMT) +diff --git a/tools/clang/tools/libclang/CXType.cpp b/tools/clang/tools/libclang/CXType.cpp +index acecf87d0cd..afdeb467769 100644 +--- a/tools/clang/tools/libclang/CXType.cpp ++++ b/tools/clang/tools/libclang/CXType.cpp +@@ -19,6 +19,7 @@ + #include "clang/AST/DeclObjC.h" + #include "clang/AST/DeclTemplate.h" + #include "clang/AST/Expr.h" ++#include "clang/AST/QualTypeNames.h" + #include "clang/AST/Type.h" + #include "clang/Basic/AddressSpaces.h" + #include "clang/Frontend/ASTUnit.h" +@@ -302,6 +303,27 @@ CXString clang_getTypeSpelling(CXType CT) { + return cxstring::createDup(OS.str()); + } + ++CXString clang_getFullyQualifiedTypeName(CXType CT) { ++ QualType T = GetQualType(CT); ++ if (T.isNull()) ++ return cxstring::createEmpty(); ++ ++ // For builtin types (but not typedefs pointing to builtin types) return their ++ // spelling. Otherwise "bool" will be turned into "_Bool". ++ const Type *TP = T.getTypePtrOrNull(); ++ if (TP && TP->isBuiltinType() && T->getAs<TypedefType>() == nullptr) ++ return clang_getTypeSpelling(CT); ++ ++ CXTranslationUnit TU = GetTU(CT); ++ ASTContext &Ctx = cxtu::getASTUnit(TU)->getASTContext(); ++ PrintingPolicy Policy(Ctx.getPrintingPolicy()); ++ Policy.SuppressScope = false; ++ Policy.AnonymousTagLocations = false; ++ Policy.PolishForDeclaration = true; ++ std::string name = TypeName::getFullyQualifiedName(T, Ctx, Policy, /*WithGlobalNsPrefix=*/true); ++ return cxstring::createDup(name.c_str()); ++} ++ + CXType clang_getTypedefDeclUnderlyingType(CXCursor C) { + using namespace cxcursor; + CXTranslationUnit TU = cxcursor::getCursorTU(C); +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 3c76090d64f..6e860e7263e 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -241,6 +241,7 @@ clang_getFileLocation + clang_getFileName + clang_getFileTime + clang_getFileUniqueID ++clang_getFullyQualifiedTypeName + clang_getFunctionTypeCallingConv + clang_getIBOutletCollectionType + clang_getIncludedFile +-- +2.23.0 + diff --git a/packages/llvm/llvm11_1-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch b/packages/llvm/llvm11_1-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch new file mode 100644 index 0000000000000000000000000000000000000000..a91937d248ccc9b761201194278e53a39b3f186e --- /dev/null +++ b/packages/llvm/llvm11_1-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch @@ -0,0 +1,261 @@ +From 075a7a3e667fe3d923de6d7a6929e61922c8b139 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 21:19:51 +0100 +Subject: [PATCH 3/5] [libclang] Add option to keep whitespace when tokenizing + +Introduces new `clang_tokenizeRange` function which accepts options to control +tokenization behavior. `clang_tokenize` is kept for backwards compatibility. +--- + clang/bindings/python/clang/cindex.py | 31 ++++++++++++++---- + .../python/tests/cindex/test_cursor.py | 9 ++++++ + clang/include/clang-c/Index.h | 32 +++++++++++++++++-- + clang/tools/libclang/CIndex.cpp | 15 +++++++-- + clang/tools/libclang/libclang.exports | 1 + + 5 files changed, 75 insertions(+), 13 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index c309f7017b2..1589acc9e7e 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -529,6 +529,13 @@ class TokenGroup(object): + + You should not instantiate this class outside of this module. + """ ++ ++ # Default tokenization mode. ++ TOKENIZE_NONE = 0 ++ ++ # Used to indicate that tokens for whitespace should be returned. ++ TOKENIZE_KEEP_WHITESPACE = 1 ++ + def __init__(self, tu, memory, count): + self._tu = tu + self._memory = memory +@@ -538,7 +545,7 @@ class TokenGroup(object): + conf.lib.clang_disposeTokens(self._tu, self._memory, self._count) + + @staticmethod +- def get_tokens(tu, extent): ++ def get_tokens(tu, extent, options=0): + """Helper method to return all tokens in an extent. + + This functionality is needed multiple places in this module. We define +@@ -547,8 +554,8 @@ class TokenGroup(object): + tokens_memory = POINTER(Token)() + tokens_count = c_uint() + +- conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), +- byref(tokens_count)) ++ conf.lib.clang_tokenizeRange( ++ tu, extent, byref(tokens_memory), byref(tokens_count), options) + + count = int(tokens_count.value) + +@@ -1852,13 +1859,16 @@ class Cursor(Structure): + for descendant in child.walk_preorder(): + yield descendant + +- def get_tokens(self): ++ def get_tokens(self, options=0): + """Obtain Token instances formulating that compose this Cursor. + + This is a generator for Token instances. It returns all tokens which + occupy the extent this cursor occupies. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ +- return TokenGroup.get_tokens(self._tu, self.extent) ++ return TokenGroup.get_tokens(self._tu, self.extent, options) + + def get_field_offsetof(self): + """Returns the offsetof the FIELD_DECL pointed by this Cursor.""" +@@ -3080,18 +3090,21 @@ class TranslationUnit(ClangObject): + return CodeCompletionResults(ptr) + return None + +- def get_tokens(self, locations=None, extent=None): ++ def get_tokens(self, locations=None, extent=None, options=0): + """Obtain tokens in this translation unit. + + This is a generator for Token instances. The caller specifies a range + of source code to obtain tokens for. The range can be specified as a + 2-tuple of SourceLocation or as a SourceRange. If both are defined, + behavior is undefined. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ + if locations is not None: + extent = SourceRange(start=locations[0], end=locations[1]) + +- return TokenGroup.get_tokens(self, extent) ++ return TokenGroup.get_tokens(self, extent, options) + + class File(ClangObject): + """ +@@ -3969,6 +3982,10 @@ functionList = [ + ("clang_tokenize", + [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]), + ++ ("clang_tokenizeRange", ++ [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint), ++ c_uint]), ++ + ("clang_visitChildren", + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 6a53c7205df..0965c1f4ae1 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -10,6 +10,7 @@ import unittest + from clang.cindex import AvailabilityKind + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind ++from clang.cindex import TokenGroup + from clang.cindex import TranslationUnit + from clang.cindex import TypeKind + from .util import get_cursor +@@ -488,6 +489,14 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tokens[0].spelling, 'int') + self.assertEqual(tokens[1].spelling, 'foo') + ++ def test_get_tokens_with_whitespace(): ++ source = 'class C { void f(); }\nvoid C::f() { }' ++ tu = get_tu(source) ++ ++ tokens = list(tu.cursor.get_tokens(TokenGroup.TOKENIZE_KEEP_WHITESPACE)) ++ self.assertEqual(''.join(t.spelling for t in tokens), source) ++ self.assertEqual(len(tokens), 27, [t.spelling for t in tokens]) ++ + def test_get_token_cursor(self): + """Ensure we can map tokens to cursors.""" + tu = get_tu('class A {}; int foo(A var = A());', lang='cpp') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index b0c62fe948e..84ed03b8920 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 62 ++#define CINDEX_VERSION_MINOR 63 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -4969,6 +4969,28 @@ CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit, + */ + CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + ++typedef enum { ++ /** ++ * \brief Used to indicate that no special tokenization options are needed. ++ */ ++ CXTokenize_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that tokens for whitespace should be returned. ++ */ ++ CXTokenize_KeepWhitespace = 0x1 ++} CXTokenize_Flags; ++ ++/** ++ * \brief Tokenize the source code described by the given range into raw ++ * lexical tokens. ++ * ++ * \see clang_tokenizeRange ++ * ++ */ ++CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens); ++ + /** + * Tokenize the source code described by the given range into raw + * lexical tokens. +@@ -4985,9 +5007,13 @@ CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + * \param NumTokens will be set to the number of tokens in the \c *Tokens + * array. + * ++ * \param options A bitmask of options that affects tokenization. This should be ++ * a bitwise OR of the CXTokenize_XXX flags. ++ * + */ +-CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, +- CXToken **Tokens, unsigned *NumTokens); ++CINDEX_LINKAGE void clang_tokenizeRange(CXTranslationUnit TU, ++ CXSourceRange Range, CXToken **Tokens, ++ unsigned *NumTokens, unsigned options); + + /** + * Annotate the given set of tokens by providing cursors for each token +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 1dc961f58a2..3a283e76ed8 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -6670,7 +6670,7 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) { + } + + static void getTokens(ASTUnit *CXXUnit, SourceRange Range, +- SmallVectorImpl<CXToken> &CXTokens) { ++ SmallVectorImpl<CXToken> &CXTokens, unsigned options) { + SourceManager &SourceMgr = CXXUnit->getSourceManager(); + std::pair<FileID, unsigned> BeginLocInfo + = SourceMgr.getDecomposedSpellingLoc(Range.getBegin()); +@@ -6692,6 +6692,9 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + CXXUnit->getASTContext().getLangOpts(), + Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end()); + Lex.SetCommentRetentionState(true); ++ if (options & CXTokenize_KeepWhitespace) { ++ Lex.SetKeepWhitespaceMode(true); ++ } + + // Lex tokens until we hit the end of the range. + const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second; +@@ -6765,7 +6768,7 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + SourceLocation End = SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second); + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, SourceRange(Begin, End), CXTokens); ++ getTokens(CXXUnit, SourceRange(Begin, End), CXTokens, CXTokenize_None); + + if (CXTokens.empty()) + return NULL; +@@ -6913,6 +6913,12 @@ CXToken *clang_getToken(CXTranslationUni + + void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, CXToken **Tokens, + unsigned *NumTokens) { ++ return clang_tokenizeRange(TU, Range, Tokens, NumTokens, CXTokenize_None); ++} ++ ++void clang_tokenizeRange(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens, ++ unsigned options) { + LOG_FUNC_SECTION { *Log << TU << ' ' << Range; } + + if (Tokens) +@@ -6804,7 +6813,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + return; + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, R, CXTokens); ++ getTokens(CXXUnit, R, CXTokens, options); + + if (CXTokens.empty()) + return; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6e860e7263e..6af6c0ca3e8 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -338,6 +338,7 @@ clang_suspendTranslationUnit + clang_sortCodeCompletionResults + clang_toggleCrashRecovery + clang_tokenize ++clang_tokenizeRange + clang_CompilationDatabase_fromDirectory + clang_CompilationDatabase_dispose + clang_CompilationDatabase_getCompileCommands +-- +2.23.0 + diff --git a/packages/llvm/llvm11_1-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch b/packages/llvm/llvm11_1-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch new file mode 100644 index 0000000000000000000000000000000000000000..f7ce6f2cd5e79c43215fba244f599a5cde6fcbf7 --- /dev/null +++ b/packages/llvm/llvm11_1-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch @@ -0,0 +1,437 @@ +From d10d66b8e762c1dd1329d5920f9ef952cf4f6940 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 17 Jul 2017 12:25:49 +0200 +Subject: [PATCH 4/5] [libclang] WIP: Allow visiting of implicit declarations + and template instantiations + +--- + clang/bindings/python/clang/cindex.py | 45 +++++-- + .../python/tests/cindex/test_cursor.py | 33 ++++++ + clang/include/clang-c/Index.h | 33 +++++- + clang/tools/libclang/CIndex.cpp | 112 ++++++++++++++++-- + clang/tools/libclang/CursorVisitor.h | 12 +- + clang/tools/libclang/libclang.exports | 2 + + 6 files changed, 220 insertions(+), 17 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index 1589acc9e7e..b023be6cdc8 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1426,6 +1426,15 @@ class Cursor(Structure): + """ + _fields_ = [("_kind_id", c_int), ("xdata", c_int), ("data", c_void_p * 3)] + ++ # Default behavior. ++ GET_CHILDREN_NONE = 0 ++ ++ # Used to indicate that implicit cursors should be visited. ++ GET_CHILDREN_WITH_IMPLICIT = 1 ++ ++ # Used to indicate that template instantiations should be visited. ++ GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS = 2 ++ + @staticmethod + def from_location(tu, location): + # We store a reference to the TU in the instance so the TU won't get +@@ -1515,6 +1524,10 @@ class Cursor(Structure): + """ + return conf.lib.clang_EnumDecl_isScoped(self) + ++ def is_implicit(self): ++ """Test whether the cursor refers to an implicit declaration.""" ++ return conf.lib.clang_isImplicit(self) ++ + def get_definition(self): + """ + If the cursor is a reference to a declaration or a declaration of +@@ -1831,8 +1844,12 @@ class Cursor(Structure): + """Returns the value of the indicated arg as an unsigned 64b integer.""" + return conf.lib.clang_Cursor_getTemplateArgumentUnsignedValue(self, num) + +- def get_children(self): +- """Return an iterator for accessing the children of this cursor.""" ++ def get_children(self, with_implicit=False, with_template_instantiations=False): ++ """Return an iterator for accessing the children of this cursor. ++ ++ By default, cursors representing implicit declarations or template instantiations ++ will be skipped. ++ """ + + # FIXME: Expose iteration from CIndex, PR6125. + def visitor(child, parent, children): +@@ -1845,18 +1862,24 @@ class Cursor(Structure): + children.append(child) + return 1 # continue + children = [] +- conf.lib.clang_visitChildren(self, callbacks['cursor_visit'](visitor), +- children) ++ dispatch = conf.lib.clang_visitChildren ++ options = Cursor.GET_CHILDREN_NONE ++ if with_implicit: ++ options |= Cursor.GET_CHILDREN_WITH_IMPLICIT ++ if with_template_instantiations: ++ options |= Cursor.GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS ++ conf.lib.clang_visitChildrenWithOptions( ++ self, callbacks['cursor_visit'](visitor), children, options) + return iter(children) + +- def walk_preorder(self): ++ def walk_preorder(self, **kwargs): + """Depth-first preorder walk over the cursor and its descendants. + + Yields cursors. + """ + yield self +- for child in self.get_children(): +- for descendant in child.walk_preorder(): ++ for child in self.get_children(**kwargs): ++ for descendant in child.walk_preorder(**kwargs): + yield descendant + + def get_tokens(self, options=0): +@@ -3927,6 +3950,10 @@ functionList = [ + [Type], + bool), + ++ ("clang_isImplicit", ++ [Cursor], ++ bool), ++ + ("clang_isInvalid", + [CursorKind], + bool), +@@ -3990,6 +4017,10 @@ functionList = [ + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), + ++ ("clang_visitChildrenWithOptions", ++ [Cursor, callbacks['cursor_visit'], py_object, c_uint], ++ c_uint), ++ + ("clang_Cursor_getNumArguments", + [Cursor], + c_int), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 0965c1f4ae1..d061f37c25c 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -94,6 +94,39 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tu_nodes[2].displayname, 'f0(int, int)') + self.assertEqual(tu_nodes[2].is_definition(), True) + ++ def test_get_children_with_implicit(): ++ tu = get_tu('struct X {}; X x;', lang='cpp') ++ cursor = get_cursor(tu, 'X') ++ ++ children = list(cursor.get_children()) ++ self.assertEqual(len(children), 0, [(c.kind, c.spelling) for c in children]) ++ ++ children = list(cursor.get_children(with_implicit=True)) ++ self.assertNotEqual(len(children), 0) ++ for child in children: ++ self.assertTrue(child.is_implicit()) ++ self.assertEqual(child.spelling, "X") ++ self.assertIn(child.kind, [CursorKind.CONSTRUCTOR, CursorKind.STRUCT_DECL]) ++ ++ def test_get_children_with_template_instantiations(): ++ tu = get_tu( ++ 'template <typename T> T frobnicate(T val);' ++ 'extern template int frobnicate<int>(int);', ++ lang='cpp') ++ cursor = get_cursor(tu, 'frobnicate') ++ self.assertEqual(cursor.kind, CursorKind.FUNCTION_TEMPLATE) ++ ++ for child in cursor.get_children(): ++ # should not return an instantiation: ++ self.assertNotEqual(child.kind, CursorKind.FUNCTION_DECL) ++ ++ for child in cursor.get_children(with_template_instantiations=True): ++ if child.kind == CursorKind.FUNCTION_DECL: ++ self.assertEqual(child.spelling, 'frobnicate') ++ break ++ else: ++ self.fail("Couldn't find template instantiation") ++ + def test_references(self): + """Ensure that references to TranslationUnit are kept.""" + tu = get_tu('int x;') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index 84ed03b8920..57acbcba143 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 63 ++#define CINDEX_VERSION_MINOR 64 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -2775,6 +2775,11 @@ CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind); + */ + CINDEX_LINKAGE unsigned clang_isUnexposed(enum CXCursorKind); + ++/*** ++ * \brief Determine whether the given cursor represents an implicit declaration. ++ */ ++CINDEX_LINKAGE unsigned clang_isImplicit(CXCursor); ++ + /** + * Describe the linkage of the entity referred to by a cursor. + */ +@@ -4199,6 +4204,32 @@ CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent, + # endif + #endif + ++typedef enum { ++ /** ++ * \brief Default behavior. ++ */ ++ CXVisitChildren_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that implicit cursors should be visited. ++ */ ++ CXVisitChildren_WithImplicit = 0x1, ++ ++ /** ++ * \brief Used to indicate that template instantiations should be visited. ++ */ ++ CXVisitChildren_WithTemplateInstantiations = 0x2 ++} CXVisitChildren_Flags; ++ ++/** ++ * \brief Visits the children of a cursor, allowing to pass extra options. ++ * Behaves identically to clang_visitChildren() in all other respects. ++ */ ++CINDEX_LINKAGE unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options); ++ + /** + * @} + */ +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 3a283e76ed8..7ee6b704647 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -196,9 +196,10 @@ bool CursorVisitor::Visit(CXCursor Curso + return true; // abort. + } + +- // Ignore implicit declarations, unless it's an objc method because +- // currently we should report implicit methods for properties when indexing. +- if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) ++ // Unless instructed otherwise we ignore implicit declarations. ++ // ObjC methods are currently visited in any case, because implicit methods ++ // for properties should be reported when indexing. ++ if (!VisitImplicitDeclarations && D->isImplicit() && !isa<ObjCMethodDecl>(D)) + return false; + } + +@@ -706,10 +706,13 @@ bool CursorVisitor::VisitTagDecl(TagDecl + + bool CursorVisitor::VisitClassTemplateSpecializationDecl( + ClassTemplateSpecializationDecl *D) { +- bool ShouldVisitBody = false; ++ bool ShouldVisitBody = VisitTemplateInstantiations; + switch (D->getSpecializationKind()) { +- case TSK_Undeclared: + case TSK_ImplicitInstantiation: ++ if (VisitTemplateInstantiations && VisitImplicitDeclarations) { ++ break; ++ } ++ case TSK_Undeclared: + // Nothing to visit + return false; + +@@ -715,6 +719,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl( + break; + + case TSK_ExplicitSpecialization: ++ // Always visit body of explicit specializations + ShouldVisitBody = true; + break; + } +@@ -938,7 +941,31 @@ bool CursorVisitor::VisitFunctionTemplat + return true; + + auto *FD = D->getTemplatedDecl(); +- return VisitAttributes(FD) || VisitFunctionDecl(FD); ++ if (VisitAttributes(FD) || VisitFunctionDecl(FD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *FD : D->specializations()) { ++ for (auto *RD : FD->redecls()) { ++ switch (RD->getTemplateSpecializationKind()) { ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -949,6 +976,40 @@ bool CursorVisitor::VisitClassTemplateDe + + auto *CD = D->getTemplatedDecl(); + return VisitAttributes(CD) || VisitCXXRecordDecl(CD); ++ if (VisitAttributes(CD) || VisitCXXRecordDecl(CD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *SD : D->specializations()) { ++ for (auto *RD : SD->redecls()) { ++ // We don't want to visit injected-class-names in this traversal. ++ if (cast<CXXRecordDecl>(RD)->isInjectedClassName()) ++ continue; ++ ++ switch ( ++ cast<ClassTemplateSpecializationDecl>(RD)->getSpecializationKind()) { ++ // Visit the implicit instantiations with the requested pattern. ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ // We don't need to do anything on an explicit instantiation ++ // or explicit specialization because there will be an explicit ++ // node for it elsewhere. ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +@@ -4426,6 +4488,24 @@ unsigned clang_visitChildrenWithBlock(CXCursor parent, + return clang_visitChildren(parent, visitWithBlock, block); + } + ++unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options) { ++ CursorVisitor CursorVis( ++ getCursorTU(parent), visitor, client_data, ++ /*VisitPreprocessorLast=*/false, ++ /*VisitIncludedPreprocessingEntries=*/false, ++ /*RegionOfInterest=*/SourceRange(), ++ /*VisitDeclsOnly=*/false, ++ /*PostChildrenVisitor=*/nullptr, ++ /*VisitImplicitDeclarations=*/(options & CXVisitChildren_WithImplicit), ++ /*VisitTemplateInstantiations=*/ ++ (options & CXVisitChildren_WithTemplateInstantiations)); ++ ++ return CursorVis.VisitChildren(parent); ++} ++ + static CXString getDeclSpelling(const Decl *D) { + if (!D) + return cxstring::createEmpty(); +@@ -5909,6 +5970,22 @@ unsigned clang_isUnexposed(enum CXCursor + } + } + ++unsigned clang_isImplicit(CXCursor Cursor) { ++ if (clang_isInvalid(Cursor.kind)) ++ return false; ++ ++ if (!clang_isDeclaration(Cursor.kind)) ++ return false; ++ ++ const Decl *D = getCursorDecl(Cursor); ++ if (!D) { ++ assert(0 && "Invalid declaration cursor"); ++ return true; // abort. ++ } ++ ++ return D->isImplicit(); ++} ++ + CXCursorKind clang_getCursorKind(CXCursor C) { return C.kind; } + + CXSourceLocation clang_getCursorLocation(CXCursor C) { +diff --git a/tools/clang/tools/libclang/CursorVisitor.h b/clang/tools/libclang/CursorVisitor.h +index b0afa5a0b59..94f4596d5fa 100644 +--- a/tools/clang/tools/libclang/CursorVisitor.h ++++ b/tools/clang/tools/libclang/CursorVisitor.h +@@ -95,6 +95,12 @@ private: + /// record entries. + bool VisitDeclsOnly; + ++ /// \brief Whether we should visit implicit declarations. ++ bool VisitImplicitDeclarations; ++ ++ /// \brief Whether we should recurse into template instantiations. ++ bool VisitTemplateInstantiations; ++ + // FIXME: Eventually remove. This part of a hack to support proper + // iteration over all Decls contained lexically within an ObjC container. + DeclContext::decl_iterator *DI_current; +@@ -152,12 +152,16 @@ public: + bool VisitIncludedPreprocessingEntries = false, + SourceRange RegionOfInterest = SourceRange(), + bool VisitDeclsOnly = false, +- PostChildrenVisitorTy PostChildrenVisitor = nullptr) ++ PostChildrenVisitorTy PostChildrenVisitor = nullptr, ++ bool VisitImplicitDeclarations = false, ++ bool VisitTemplateInstantiations = false) + : TU(TU), AU(cxtu::getASTUnit(TU)), Visitor(Visitor), + PostChildrenVisitor(PostChildrenVisitor), ClientData(ClientData), + VisitPreprocessorLast(VisitPreprocessorLast), + VisitIncludedEntities(VisitIncludedPreprocessingEntries), + RegionOfInterest(RegionOfInterest), VisitDeclsOnly(VisitDeclsOnly), ++ VisitImplicitDeclarations(VisitImplicitDeclarations), ++ VisitTemplateInstantiations(VisitTemplateInstantiations), + DI_current(nullptr), FileDI_current(nullptr) { + Parent.kind = CXCursor_NoDeclFound; + Parent.data[0] = nullptr; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6af6c0ca3e8..d17eb83187d 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -313,6 +313,7 @@ clang_isInvalidDeclaration + clang_isExpression + clang_isFileMultipleIncludeGuarded + clang_isFunctionTypeVariadic ++clang_isImplicit + clang_isInvalid + clang_isPODType + clang_isPreprocessing +@@ -354,6 +355,7 @@ clang_CompileCommand_getNumArgs + clang_CompileCommand_getArg + clang_visitChildren + clang_visitChildrenWithBlock ++clang_visitChildrenWithOptions + clang_ModuleMapDescriptor_create + clang_ModuleMapDescriptor_dispose + clang_ModuleMapDescriptor_setFrameworkModuleName +-- +2.23.0 + diff --git a/packages/llvm/llvm5-0001-libclang-Add-support-for-checking-abstractness-of-re.patch b/packages/llvm/llvm5-0001-libclang-Add-support-for-checking-abstractness-of-re.patch new file mode 100644 index 0000000000000000000000000000000000000000..62aabd59000ea8e99255a341945fc76ddfffbf74 --- /dev/null +++ b/packages/llvm/llvm5-0001-libclang-Add-support-for-checking-abstractness-of-re.patch @@ -0,0 +1,146 @@ +From 85f611be67af9f9b5677917b4fee5bd15b5dc394 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Mon, 31 Jul 2017 14:09:08 +0200 +Subject: [PATCH 01/12] [libclang] Add support for checking abstractness of + records + +--- + bindings/python/clang/cindex.py | 10 ++++++++++ + bindings/python/tests/cindex/test_cursor.py | 11 +++++++++++ + include/clang-c/Index.h | 6 ++++++ + test/Index/load-classes.cpp | 2 +- + tools/c-index-test/c-index-test.c | 2 ++ + tools/libclang/CIndex.cpp | 11 +++++++++++ + tools/libclang/libclang.exports | 1 + + 7 files changed, 42 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 236803a9ab..0f01d171ad 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1478,6 +1478,12 @@ class Cursor(Structure): + """ + return conf.lib.clang_CXXMethod_isVirtual(self) + ++ def is_abstract_record(self): ++ """Returns True if the cursor refers to a C++ record declaration ++ that has pure virtual member functions. ++ """ ++ return conf.lib.clang_CXXRecord_isAbstract(self) ++ + def is_scoped_enum(self): + """Returns True if the cursor refers to a scoped enum declaration. + """ +@@ -3319,6 +3325,10 @@ functionList = [ + [Cursor], + bool), + ++ ("clang_CXXRecord_isAbstract", ++ [Cursor], ++ bool), ++ + ("clang_EnumDecl_isScoped", + [Cursor], + bool), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 4787ea931e..85c455fd73 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -255,6 +255,17 @@ def test_is_virtual_method(): + assert foo.is_virtual_method() + assert not bar.is_virtual_method() + ++def test_is_abstract_record(): ++ """Ensure Cursor.is_abstract_record works.""" ++ source = 'struct X { virtual void x() = 0; }; struct Y : X { void x(); };' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'X') ++ assert cls.is_abstract_record() ++ ++ cls = get_cursor(tu, 'Y') ++ assert not cls.is_abstract_record() ++ + def test_is_scoped_enum(): + """Ensure Cursor.is_scoped_enum works.""" + source = 'class X {}; enum RegularEnum {}; enum class ScopedEnum {};' +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 3b5ea9fa53..89957e8526 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -4419,6 +4419,12 @@ CINDEX_LINKAGE unsigned clang_CXXMethod_isStatic(CXCursor C); + CINDEX_LINKAGE unsigned clang_CXXMethod_isVirtual(CXCursor C); + + /** ++ * \brief Determine if a C++ record is abstract, i.e. whether a class or struct ++ * has a pure virtual member function. ++ */ ++CINDEX_LINKAGE unsigned clang_CXXRecord_isAbstract(CXCursor C); ++ ++/** + * \brief Determine if an enum declaration refers to a scoped enum. + */ + CINDEX_LINKAGE unsigned clang_EnumDecl_isScoped(CXCursor C); +diff --git a/tools/clang/test/Index/load-classes.cpp b/tools/clang/test/Index/load-classes.cpp +index 8b1ed317e3..b6c25b4f75 100644 +--- a/tools/clang/test/Index/load-classes.cpp ++++ b/tools/clang/test/Index/load-classes.cpp +@@ -29,7 +29,7 @@ X::X(int value) { + } + + // RUN: c-index-test -test-load-source all %s | FileCheck %s +-// CHECK: load-classes.cpp:3:8: StructDecl=X:3:8 (Definition) Extent=[3:1 - 26:2] ++// CHECK: load-classes.cpp:3:8: StructDecl=X:3:8 (Definition) (abstract) Extent=[3:1 - 26:2] + // CHECK: load-classes.cpp:4:3: CXXConstructor=X:4:3 (converting constructor) Extent=[4:3 - 4:15] [access=public] + // FIXME: missing TypeRef in the constructor name + // CHECK: load-classes.cpp:4:9: ParmDecl=value:4:9 (Definition) Extent=[4:5 - 4:14] +diff --git a/tools/clang/tools/c-index-test/c-index-test.c b/tools/clang/tools/c-index-test/c-index-test.c +index cf3581e259..08a187ffdd 100644 +--- a/tools/clang/tools/c-index-test/c-index-test.c ++++ b/tools/clang/tools/c-index-test/c-index-test.c +@@ -804,6 +804,8 @@ static void PrintCursor(CXCursor Cursor, const char *CommentSchemaFile) { + printf(" (const)"); + if (clang_CXXMethod_isPureVirtual(Cursor)) + printf(" (pure)"); ++ if (clang_CXXRecord_isAbstract(Cursor)) ++ printf(" (abstract)"); + if (clang_EnumDecl_isScoped(Cursor)) + printf(" (scoped)"); + if (clang_Cursor_isVariadic(Cursor)) +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/clang/tools/libclang/CIndex.cpp +index ca21b6c6f6..621bc42076 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -7846,6 +7846,17 @@ unsigned clang_CXXMethod_isVirtual(CXCursor C) { + return (Method && Method->isVirtual()) ? 1 : 0; + } + ++unsigned clang_CXXRecord_isAbstract(CXCursor C) { ++ if (!clang_isDeclaration(C.kind)) ++ return 0; ++ ++ const auto *D = cxcursor::getCursorDecl(C); ++ const auto *RD = dyn_cast_or_null<CXXRecordDecl>(D); ++ if (RD) ++ RD = RD->getDefinition(); ++ return (RD && RD->isAbstract()) ? 1 : 0; ++} ++ + unsigned clang_EnumDecl_isScoped(CXCursor C) { + if (!clang_isDeclaration(C.kind)) + return 0; +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index e0d178a529..9ddc055125 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -12,6 +12,7 @@ clang_CXXMethod_isConst + clang_CXXMethod_isPureVirtual + clang_CXXMethod_isStatic + clang_CXXMethod_isVirtual ++clang_CXXRecord_isAbstract + clang_EnumDecl_isScoped + clang_Cursor_getArgument + clang_Cursor_getNumTemplateArguments +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0002-libclang-Keep-track-of-TranslationUnit-instance-when.patch b/packages/llvm/llvm5-0002-libclang-Keep-track-of-TranslationUnit-instance-when.patch new file mode 100644 index 0000000000000000000000000000000000000000..d5d076234c36c8d14daeb86b80f1e7553f258161 --- /dev/null +++ b/packages/llvm/llvm5-0002-libclang-Keep-track-of-TranslationUnit-instance-when.patch @@ -0,0 +1,61 @@ +From 0c4382f31fefe6c5575842427c83e1c26fe00efa Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Fri, 28 Jul 2017 12:25:20 +0200 +Subject: [PATCH 02/12] [libclang] Keep track of TranslationUnit instance when + annotating tokens + +Previously the _tu was not propagated to the returned cursor, leading to errors when calling any +method on that cursor (e.g. cursor.referenced). +--- + bindings/python/clang/cindex.py | 1 + + bindings/python/tests/cindex/test_cursor.py | 22 ++++++++++++++++++++++ + 2 files changed, 23 insertions(+) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 0f01d171ad..ecff13f7a5 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -3199,6 +3199,7 @@ class Token(Structure): + def cursor(self): + """The Cursor this Token corresponds to.""" + cursor = Cursor() ++ cursor._tu = self._tu + + conf.lib.clang_annotateTokens(self._tu, byref(self), 1, byref(cursor)) + +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 85c455fd73..87fd76ed0e 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -406,6 +406,28 @@ def test_get_tokens(): + assert tokens[0].spelling == 'int' + assert tokens[1].spelling == 'foo' + ++def test_get_token_cursor(): ++ """Ensure we can map tokens to cursors.""" ++ tu = get_tu('class A {}; int foo(A var = A());', lang='cpp') ++ foo = get_cursor(tu, 'foo') ++ ++ for cursor in foo.walk_preorder(): ++ if cursor.kind.is_expression() and not cursor.kind.is_statement(): ++ break ++ else: ++ assert False, "Could not find default value expression" ++ ++ tokens = list(cursor.get_tokens()) ++ assert len(tokens) == 4, [t.spelling for t in tokens] ++ assert tokens[0].spelling == '=' ++ assert tokens[1].spelling == 'A' ++ assert tokens[2].spelling == '(' ++ assert tokens[3].spelling == ')' ++ t_cursor = tokens[1].cursor ++ assert t_cursor.kind == CursorKind.TYPE_REF ++ r_cursor = t_cursor.referenced # should not raise an exception ++ assert r_cursor.kind == CursorKind.CLASS_DECL ++ + def test_get_arguments(): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0003-Fix-warnings-in-Tooling-QualTypeNamesTest.patch b/packages/llvm/llvm5-0003-Fix-warnings-in-Tooling-QualTypeNamesTest.patch new file mode 100644 index 0000000000000000000000000000000000000000..2fa9a8c80ce5aeffb04977ce941395bdc6543fa3 --- /dev/null +++ b/packages/llvm/llvm5-0003-Fix-warnings-in-Tooling-QualTypeNamesTest.patch @@ -0,0 +1,57 @@ +From 1ef6a85f73cb4f06405dfe20c225bcc0da9f9776 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Wed, 26 Jul 2017 15:09:28 +0200 +Subject: [PATCH 03/12] Fix warnings in Tooling/QualTypeNamesTest + +The code in question uses variadic templates and alias declarations +and thus needs `-std=c++11`. +--- + unittests/Tooling/QualTypeNamesTest.cpp | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +index edd5060ba0..532fae6f5a 100644 +--- a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp ++++ b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +@@ -35,7 +35,7 @@ struct TypeNameVisitor : TestVisitor<TypeNameVisitor> { + EXPECT_TRUE(false) << "Typename::getFullyQualifiedName failed for " + << VD->getQualifiedNameAsString() << std::endl + << " Actual: " << ActualName << std::endl +- << " Exepcted: " << ExpectedName; ++ << " Expected: " << ExpectedName; + } + } + return true; +@@ -163,7 +163,7 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + "};\n" + "EnumScopeClass::AnEnum AnEnumVar;\n", + TypeNameVisitor::Lang_CXX11 +-); ++ ); + + TypeNameVisitor Complex; + Complex.ExpectedQualTypeNames["CheckTX"] = "B::TX"; +@@ -180,7 +180,9 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " typedef tuple<X> TX;" + " TX CheckTX;" + " struct A { typedef int X; };" +- "}"); ++ "}", ++ TypeNameVisitor::Lang_CXX11 ++ ); + + TypeNameVisitor GlobalNsPrefix; + GlobalNsPrefix.WithGlobalNsPrefix = true; +@@ -215,7 +217,8 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " aStruct CheckK;\n" + " }\n" + " }\n" +- "}\n" ++ "}\n", ++ TypeNameVisitor::Lang_CXX11 + ); + } + +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0004-Defer-addition-of-keywords-to-identifier-table-when-.patch b/packages/llvm/llvm5-0004-Defer-addition-of-keywords-to-identifier-table-when-.patch new file mode 100644 index 0000000000000000000000000000000000000000..426b22fdbca50d1ab82a8dafd8355dcd9217e490 --- /dev/null +++ b/packages/llvm/llvm5-0004-Defer-addition-of-keywords-to-identifier-table-when-.patch @@ -0,0 +1,220 @@ +From ba4c926983036f010c3e4d28be48cfdee8495f87 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Sun, 9 Jul 2017 13:03:02 +0200 +Subject: [PATCH 04/12] Defer addition of keywords to identifier table when + loading AST + +In ASTUnit::LoadFromASTFile, the preprocesor object is set up using +default-constructed LangOptions (which only later get populated). +Then, in the constructor of IdentifierTable, these default-constructed +LangOptions were used in the call to AddKeywords, leading to wrong +initialization of the identifier table. + +This change defers adding the keywords to the identifier table until +after the language options have been loaded from the AST file. +--- + include/clang/Basic/IdentifierTable.h | 6 ++-- + include/clang/Lex/Preprocessor.h | 3 +- + lib/Basic/IdentifierTable.cpp | 17 +++++----- + lib/Frontend/ASTUnit.cpp | 10 ++++-- + lib/Lex/Preprocessor.cpp | 6 ++-- + unittests/libclang/LibclangTest.cpp | 64 +++++++++++++++++++++++++++++++++++ + 6 files changed, 91 insertions(+), 15 deletions(-) + +diff --git a/tools/clang/include/clang/Basic/IdentifierTable.h b/tools/clang/include/clang/Basic/IdentifierTable.h +index 3938e09890..1172869d45 100644 +--- a/tools/clang/include/clang/Basic/IdentifierTable.h ++++ b/tools/clang/include/clang/Basic/IdentifierTable.h +@@ -472,9 +472,11 @@ class IdentifierTable { + + public: + /// \brief Create the identifier table, populating it with info about the +- /// language keywords for the language specified by \p LangOpts. ++ /// language keywords for the language specified by \p LangOpts if ++ /// \p DeferKeywordAddition is not set. + IdentifierTable(const LangOptions &LangOpts, +- IdentifierInfoLookup* externalLookup = nullptr); ++ IdentifierInfoLookup *externalLookup = nullptr, ++ bool DeferKeywordAddition = false); + + /// \brief Set the external identifier lookup mechanism. + void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) { +diff --git a/tools/clang/include/clang/Lex/Preprocessor.h b/tools/clang/include/clang/Lex/Preprocessor.h +index 49a95986fd..6570e4c76e 100644 +--- a/tools/clang/include/clang/Lex/Preprocessor.h ++++ b/tools/clang/include/clang/Lex/Preprocessor.h +@@ -692,7 +692,8 @@ public: + HeaderSearch &Headers, ModuleLoader &TheModuleLoader, + IdentifierInfoLookup *IILookup = nullptr, + bool OwnsHeaderSearch = false, +- TranslationUnitKind TUKind = TU_Complete); ++ TranslationUnitKind TUKind = TU_Complete, ++ bool DeferKeywordAddition = false); + + ~Preprocessor(); + +diff --git a/tools/clang/lib/Basic/IdentifierTable.cpp b/tools/clang/lib/Basic/IdentifierTable.cpp +index fe7829ec50..cfb0b1a702 100644 +--- a/tools/clang/lib/Basic/IdentifierTable.cpp ++++ b/tools/clang/lib/Basic/IdentifierTable.cpp +@@ -73,17 +73,15 @@ IdentifierIterator *IdentifierInfoLookup::getIdentifiers() { + } + + IdentifierTable::IdentifierTable(const LangOptions &LangOpts, +- IdentifierInfoLookup* externalLookup) +- : HashTable(8192), // Start with space for 8K identifiers. +- ExternalLookup(externalLookup) { ++ IdentifierInfoLookup *externalLookup, ++ bool DeferKeywordAddition) ++ : HashTable(8192), // Start with space for 8K identifiers. ++ ExternalLookup(externalLookup) { + + // Populate the identifier table with info about keywords for the current + // language. +- AddKeywords(LangOpts); +- +- +- // Add the '_experimental_modules_import' contextual keyword. +- get("import").setModulesImport(true); ++ if (!DeferKeywordAddition) ++ AddKeywords(LangOpts); + } + + //===----------------------------------------------------------------------===// +@@ -230,6 +228,9 @@ void IdentifierTable::AddKeywords(const LangOptions &LangOpts) { + + if (LangOpts.DeclSpecKeyword) + AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this); ++ ++ // Add the '_experimental_modules_import' contextual keyword. ++ get("import").setModulesImport(true); + } + + /// \brief Checks if the specified token kind represents a keyword in the +diff --git a/tools/clang/lib/Frontend/ASTUnit.cpp b/tools/clang/lib/Frontend/ASTUnit.cpp +index 07f847ca94..875f21d69a 100644 +--- a/tools/clang/lib/Frontend/ASTUnit.cpp ++++ b/tools/clang/lib/Frontend/ASTUnit.cpp +@@ -536,6 +536,10 @@ private: + // Initialize the preprocessor. + PP.Initialize(*Target); + ++ // Populate the identifier table with info about keywords for the current ++ // language. ++ PP.getIdentifierTable().AddKeywords(LangOpt); ++ + if (!Context) + return; + +@@ -718,11 +722,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( + HeaderSearch &HeaderInfo = *AST->HeaderInfo; + unsigned Counter; + ++ // As the language options have not been loaded yet, adding keywords to the ++ // identifier table is deferred and will be initiated by ASTInfoCollector. + AST->PP = std::make_shared<Preprocessor>( + AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts, + AST->getSourceManager(), *AST->PCMCache, HeaderInfo, AST->ModuleLoader, +- /*IILookup=*/nullptr, +- /*OwnsHeaderSearch=*/false); ++ /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false, TU_Complete, ++ /*DeferKeywordAddition=*/true); + Preprocessor &PP = *AST->PP; + + if (ToLoad >= LoadASTOnly) +diff --git a/tools/clang/lib/Lex/Preprocessor.cpp b/tools/clang/lib/Lex/Preprocessor.cpp +index 158d0eca27..64365385a3 100644 +--- a/tools/clang/lib/Lex/Preprocessor.cpp ++++ b/tools/clang/lib/Lex/Preprocessor.cpp +@@ -73,12 +73,14 @@ Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, + SourceManager &SM, MemoryBufferCache &PCMCache, + HeaderSearch &Headers, ModuleLoader &TheModuleLoader, + IdentifierInfoLookup *IILookup, bool OwnsHeaders, +- TranslationUnitKind TUKind) ++ TranslationUnitKind TUKind, ++ bool DeferKeywordAddition) + : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr), + AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM), + PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), + HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), +- ExternalSource(nullptr), Identifiers(opts, IILookup), ++ ExternalSource(nullptr), ++ Identifiers(opts, IILookup, DeferKeywordAddition), + PragmaHandlers(new PragmaNamespace(StringRef())), + IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr), + CodeCompletionFile(nullptr), CodeCompletionOffset(0), +diff --git a/tools/clang/unittests/libclang/LibclangTest.cpp b/tools/clang/unittests/libclang/LibclangTest.cpp +index f2a96d6be6..27c8ac7b3d 100644 +--- a/tools/clang/unittests/libclang/LibclangTest.cpp ++++ b/tools/clang/unittests/libclang/LibclangTest.cpp +@@ -572,3 +572,67 @@ TEST_F(LibclangReparseTest, clang_parseTranslationUnit2FullArgv) { + EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); + DisplayDiagnostics(); + } ++ ++class LibclangSerializationTest : public LibclangParseTest { ++public: ++ bool SaveAndLoadTU(const std::string &Filename) { ++ unsigned options = clang_defaultSaveOptions(ClangTU); ++ if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) != ++ CXSaveError_None) { ++ DEBUG(llvm::dbgs() << "Saving failed\n"); ++ return false; ++ } ++ ++ clang_disposeTranslationUnit(ClangTU); ++ ++ ClangTU = clang_createTranslationUnit(Index, Filename.c_str()); ++ ++ if (!ClangTU) { ++ DEBUG(llvm::dbgs() << "Loading failed\n"); ++ return false; ++ } ++ ++ return true; ++ } ++}; ++ ++TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) { ++ // Ensure that "class" is recognized as a keyword token after serializing ++ // and reloading the AST, as it is not a keyword for the default LangOptions. ++ std::string HeaderName = "test.h"; ++ WriteFile(HeaderName, "enum class Something {};"); ++ ++ const char *Argv[] = {"-xc++-header", "-std=c++11"}; ++ ++ ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv, ++ sizeof(Argv) / sizeof(Argv[0]), nullptr, ++ 0, TUFlags); ++ ++ auto CheckTokenKinds = [=]() { ++ CXSourceRange Range = ++ clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU)); ++ ++ CXToken *Tokens; ++ unsigned int NumTokens; ++ clang_tokenize(ClangTU, Range, &Tokens, &NumTokens); ++ ++ ASSERT_EQ(6u, NumTokens); ++ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0])); ++ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1])); ++ EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2])); ++ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3])); ++ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4])); ++ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5])); ++ ++ clang_disposeTokens(ClangTU, Tokens, NumTokens); ++ }; ++ ++ CheckTokenKinds(); ++ ++ std::string ASTName = "test.ast"; ++ WriteFile(ASTName, ""); ++ ++ ASSERT_TRUE(SaveAndLoadTU(ASTName)); ++ ++ CheckTokenKinds(); ++} +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0005-Tooling-Fully-qualify-template-parameters-of-nested-.patch b/packages/llvm/llvm5-0005-Tooling-Fully-qualify-template-parameters-of-nested-.patch new file mode 100644 index 0000000000000000000000000000000000000000..e23cb61895f88a3009cbc06cc5dd0d9006a0ef27 --- /dev/null +++ b/packages/llvm/llvm5-0005-Tooling-Fully-qualify-template-parameters-of-nested-.patch @@ -0,0 +1,77 @@ +From 4427624b30117c6db67b9ea7c3f16e396edc3748 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Wed, 26 Jul 2017 15:06:10 +0200 +Subject: [PATCH 05/12] [Tooling] Fully qualify template parameters of nested + name specifier in getFullyQualifiedName + +--- + lib/Tooling/Core/QualTypeNames.cpp | 18 ++++++++++++++---- + unittests/Tooling/QualTypeNamesTest.cpp | 8 +++++++- + 2 files changed, 21 insertions(+), 5 deletions(-) + +diff --git a/tools/clang/lib/Tooling/Core/QualTypeNames.cpp b/tools/clang/lib/Tooling/Core/QualTypeNames.cpp +index 721c2c92fc..934624dd14 100644 +--- a/tools/clang/lib/Tooling/Core/QualTypeNames.cpp ++++ b/tools/clang/lib/Tooling/Core/QualTypeNames.cpp +@@ -370,11 +370,21 @@ NestedNameSpecifier *createNestedNameSpecifier(const ASTContext &Ctx, + const TypeDecl *TD, + bool FullyQualify, + bool WithGlobalNsPrefix) { ++ const Type *TypePtr = TD->getTypeForDecl(); ++ // In case of template specializations iterate over the arguments and ++ // fully qualify them as well. ++ if (isa<const TemplateSpecializationType>(TypePtr) || ++ isa<const RecordType>(TypePtr)) { ++ // We are asked to fully qualify and we have a Record Type (which ++ // may point to a template specialization) or Template ++ // Specialization Type. We need to fully qualify their arguments. ++ ++ TypePtr = getFullyQualifiedTemplateType(Ctx, TypePtr, WithGlobalNsPrefix); ++ } ++ + return NestedNameSpecifier::Create( +- Ctx, +- createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), +- false /*No TemplateKeyword*/, +- TD->getTypeForDecl()); ++ Ctx, createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), ++ false /*No TemplateKeyword*/, TypePtr); + } + + /// \brief Return the fully qualified type, including fully-qualified +diff --git a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +index 532fae6f5a..e27089b986 100644 +--- a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp ++++ b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +@@ -63,6 +63,10 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + // Template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckC"] = + "A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>"; ++ // Template parameters of nested name specifier should also be fully expanded. ++ Visitor.ExpectedQualTypeNames["CheckNested"] = ++ // "typename A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>::nested"; ++ "typename A::B::Template0<int, A::B::Class0>::nested"; + // Recursive template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckD"] = + "A::B::Template0<A::B::Template1<A::B::C::MyInt, A::B::AnotherClass>, " +@@ -105,7 +109,7 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " using InnerAlias = OuterTemplateClass<T>;\n" + " InnerAlias<int> AliasTypeVal;\n" + " }\n" +- " template<class X, class Y> class Template0;" ++ " template<class X, class Y> struct Template0 { typedef int nested; };" + " template<class X, class Y> class Template1;" + " typedef B::Class0 AnotherClass;\n" + " void Function1(Template0<C::MyInt,\n" +@@ -113,6 +117,8 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " void Function2(Template0<Template1<C::MyInt, AnotherClass>,\n" + " Template0<int, long> > CheckD);\n" + " void Function3(const B::Class0* CheckM);\n" ++ " void Function4(typename Template0<C::MyInt,\n" ++ " AnotherClass>::nested CheckNested);\n" + " }\n" + "template<typename... Values> class Variadic {};\n" + "Variadic<int, B::Template0<int, char>, " +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0006-libclang-Add-support-for-obtaining-fully-qualified-n.patch b/packages/llvm/llvm5-0006-libclang-Add-support-for-obtaining-fully-qualified-n.patch new file mode 100644 index 0000000000000000000000000000000000000000..441dc96ecf8216395359a5a207feea5e54063f06 --- /dev/null +++ b/packages/llvm/llvm5-0006-libclang-Add-support-for-obtaining-fully-qualified-n.patch @@ -0,0 +1,136 @@ +From a24d66fca11152dbd9b8abb246dfa340e2d6843b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Mon, 10 Jul 2017 13:53:25 +0200 +Subject: [PATCH 06/12] [libclang] Add support for obtaining fully qualified + names of types + +This patch allows retrieving the fully qualified names of types +through libclang and clang.cindex (Python). +--- + bindings/python/clang/cindex.py | 13 +++++++++++++ + bindings/python/tests/cindex/test_cursor.py | 8 ++++++++ + include/clang-c/Index.h | 8 ++++++++ + tools/libclang/CXType.cpp | 18 ++++++++++++++++++ + tools/libclang/libclang.exports | 1 + + 5 files changed, 48 insertions(+) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index ecff13f7a5..496e1089ad 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -2314,6 +2314,14 @@ class Type(Structure): + """Retrieve the spelling of this Type.""" + return conf.lib.clang_getTypeSpelling(self) + ++ @property ++ def fully_qualified_name(self): ++ """Retrieve the fully qualified name of this Type.""" ++ if not hasattr(self, '_fully_qualified_name'): ++ self._fully_qualified_name = conf.lib.clang_getFullyQualifiedTypeName(self) ++ ++ return self._fully_qualified_name ++ + def __eq__(self, other): + if type(other) != type(self): + return False +@@ -3750,6 +3758,11 @@ functionList = [ + _CXString, + _CXString.from_result), + ++ ("clang_getFullyQualifiedTypeName", ++ [Type], ++ _CXString, ++ _CXString.from_result), ++ + ("clang_hashCursor", + [Cursor], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 87fd76ed0e..3cd499ea11 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -291,6 +291,14 @@ def test_underlying_type(): + underlying = typedef.underlying_typedef_type + assert underlying.kind == TypeKind.INT + ++def test_fully_qualified_type_name(): ++ source = 'namespace uiae { struct X { typedef int sometype; }; }' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'sometype') ++ assert cls.type.fully_qualified_name.endswith( ++ "uiae::X::sometype") ++ + kParentTest = """\ + class C { + void f(); +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 89957e8526..402ca9a436 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -3241,6 +3241,14 @@ CINDEX_LINKAGE CXType clang_getCursorType(CXCursor C); + CINDEX_LINKAGE CXString clang_getTypeSpelling(CXType CT); + + /** ++ * \brief Retrieve the fully qualified name of the underlying type. ++ * This includes full qualification of all template parameters etc. ++ * ++ * If the type is invalid, an empty string is returned. ++ */ ++CINDEX_LINKAGE CXString clang_getFullyQualifiedTypeName(CXType CT); ++ ++/** + * \brief Retrieve the underlying type of a typedef declaration. + * + * If the cursor does not reference a typedef declaration, an invalid type is +diff --git a/tools/clang/tools/libclang/CXType.cpp b/tools/clang/tools/libclang/CXType.cpp +index d2cb509059..e99f513d13 100644 +--- a/tools/clang/tools/libclang/CXType.cpp ++++ b/tools/clang/tools/libclang/CXType.cpp +@@ -23,6 +23,7 @@ + #include "clang/AST/Type.h" + #include "clang/Basic/AddressSpaces.h" + #include "clang/Frontend/ASTUnit.h" ++#include "clang/Tooling/Core/QualTypeNames.h" + + using namespace clang; + +@@ -282,6 +283,23 @@ CXString clang_getTypeSpelling(CXType CT) { + return cxstring::createDup(OS.str()); + } + ++CXString clang_getFullyQualifiedTypeName(CXType CT) { ++ QualType T = GetQualType(CT); ++ if (T.isNull()) ++ return cxstring::createEmpty(); ++ ++ // For builtin types (but not typedefs pointing to builtin types) return their ++ // spelling. Otherwise "bool" will be turned into "_Bool". ++ const Type *TP = T.getTypePtrOrNull(); ++ if (TP && TP->isBuiltinType() && T->getAs<TypedefType>() == nullptr) ++ return clang_getTypeSpelling(CT); ++ ++ CXTranslationUnit TU = GetTU(CT); ++ ASTContext &Ctx = cxtu::getASTUnit(TU)->getASTContext(); ++ std::string name = TypeName::getFullyQualifiedName(T, Ctx, /*WithGlobalNsPrefix=*/true); ++ return cxstring::createDup(name.c_str()); ++} ++ + CXType clang_getTypedefDeclUnderlyingType(CXCursor C) { + using namespace cxcursor; + CXTranslationUnit TU = cxcursor::getCursorTU(C); +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 9ddc055125..9c56e88052 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -221,6 +221,7 @@ clang_getFileLocation + clang_getFileName + clang_getFileTime + clang_getFileUniqueID ++clang_getFullyQualifiedTypeName + clang_getFunctionTypeCallingConv + clang_getIBOutletCollectionType + clang_getIncludedFile +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0007-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch b/packages/llvm/llvm5-0007-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch new file mode 100644 index 0000000000000000000000000000000000000000..d8625c32efa08e8ec47c430d4ec114dfda36b6da --- /dev/null +++ b/packages/llvm/llvm5-0007-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch @@ -0,0 +1,242 @@ +From 25c705cda6508b56ac4ab4ccd8c08a1a8f911941 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Mon, 10 Jul 2017 14:44:22 +0200 +Subject: [PATCH 07/12] [libclang] Add option to keep whitespace when + tokenizing + +--- + bindings/python/clang/cindex.py | 31 ++++++++++++++++++++++------- + bindings/python/tests/cindex/test_cursor.py | 9 +++++++++ + include/clang-c/Index.h | 30 ++++++++++++++++++++++++++-- + tools/libclang/CIndex.cpp | 13 ++++++++++-- + tools/libclang/libclang.exports | 1 + + 5 files changed, 73 insertions(+), 11 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 496e1089ad..a5bb58b44f 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -514,6 +514,13 @@ class TokenGroup(object): + + You should not instantiate this class outside of this module. + """ ++ ++ # Default tokenization mode. ++ TOKENIZE_NONE = 0 ++ ++ # Used to indicate that tokens for whitespace should be returned. ++ TOKENIZE_KEEP_WHITESPACE = 1 ++ + def __init__(self, tu, memory, count): + self._tu = tu + self._memory = memory +@@ -523,7 +530,7 @@ class TokenGroup(object): + conf.lib.clang_disposeTokens(self._tu, self._memory, self._count) + + @staticmethod +- def get_tokens(tu, extent): ++ def get_tokens(tu, extent, options=0): + """Helper method to return all tokens in an extent. + + This functionality is needed multiple places in this module. We define +@@ -532,8 +539,8 @@ class TokenGroup(object): + tokens_memory = POINTER(Token)() + tokens_count = c_uint() + +- conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), +- byref(tokens_count)) ++ conf.lib.clang_tokenizeRange( ++ tu, extent, byref(tokens_memory), byref(tokens_count), options) + + count = int(tokens_count.value) + +@@ -1801,13 +1808,16 @@ class Cursor(Structure): + for descendant in child.walk_preorder(): + yield descendant + +- def get_tokens(self): ++ def get_tokens(self, options=0): + """Obtain Token instances formulating that compose this Cursor. + + This is a generator for Token instances. It returns all tokens which + occupy the extent this cursor occupies. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ +- return TokenGroup.get_tokens(self._tu, self.extent) ++ return TokenGroup.get_tokens(self._tu, self.extent, options) + + def get_field_offsetof(self): + """Returns the offsetof the FIELD_DECL pointed by this Cursor.""" +@@ -2971,18 +2981,21 @@ class TranslationUnit(ClangObject): + return CodeCompletionResults(ptr) + return None + +- def get_tokens(self, locations=None, extent=None): ++ def get_tokens(self, locations=None, extent=None, options=0): + """Obtain tokens in this translation unit. + + This is a generator for Token instances. The caller specifies a range + of source code to obtain tokens for. The range can be specified as a + 2-tuple of SourceLocation or as a SourceRange. If both are defined, + behavior is undefined. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ + if locations is not None: + extent = SourceRange(start=locations[0], end=locations[1]) + +- return TokenGroup.get_tokens(self, extent) ++ return TokenGroup.get_tokens(self, extent, options) + + class File(ClangObject): + """ +@@ -3850,6 +3863,10 @@ functionList = [ + ("clang_tokenize", + [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]), + ++ ("clang_tokenizeRange", ++ [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint), ++ c_uint]), ++ + ("clang_visitChildren", + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 3cd499ea11..2d50ec5901 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -3,6 +3,7 @@ import gc + + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind ++from clang.cindex import TokenGroup + from clang.cindex import TranslationUnit + from clang.cindex import TypeKind + from .util import get_cursor +@@ -436,6 +437,14 @@ def test_get_token_cursor(): + r_cursor = t_cursor.referenced # should not raise an exception + assert r_cursor.kind == CursorKind.CLASS_DECL + ++def test_get_tokens_with_whitespace(): ++ source = 'class C { void f(); }\nvoid C::f() { }' ++ tu = get_tu(source) ++ ++ tokens = list(tu.cursor.get_tokens(TokenGroup.TOKENIZE_KEEP_WHITESPACE)) ++ assert ''.join(t.spelling for t in tokens) == source ++ assert len(tokens) == 27 ++ + def test_get_arguments(): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 402ca9a436..7fd17366ee 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -4616,6 +4616,28 @@ CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit, + */ + CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + ++typedef enum { ++ /** ++ * \brief Used to indicate that no special tokenization options are needed. ++ */ ++ CXTokenize_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that tokens for whitespace should be returned. ++ */ ++ CXTokenize_KeepWhitespace = 0x1 ++} CXTokenize_Flags; ++ ++/** ++ * \brief Tokenize the source code described by the given range into raw ++ * lexical tokens. ++ * ++ * \see clang_tokenizeRange ++ * ++ */ ++CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens); ++ + /** + * \brief Tokenize the source code described by the given range into raw + * lexical tokens. +@@ -4632,9 +4654,13 @@ CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + * \param NumTokens will be set to the number of tokens in the \c *Tokens + * array. + * ++ * \param options A bitmask of options that affects tokenization. This should be ++ * a bitwise OR of the CXTokenize_XXX flags. ++ * + */ +-CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, +- CXToken **Tokens, unsigned *NumTokens); ++CINDEX_LINKAGE void clang_tokenizeRange(CXTranslationUnit TU, ++ CXSourceRange Range, CXToken **Tokens, ++ unsigned *NumTokens, unsigned options); + + /** + * \brief Annotate the given set of tokens by providing cursors for each token +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/clang/tools/libclang/CIndex.cpp +index 621bc42076..04fd775fb0 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -6292,7 +6292,7 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) { + } + + static void getTokens(ASTUnit *CXXUnit, SourceRange Range, +- SmallVectorImpl<CXToken> &CXTokens) { ++ SmallVectorImpl<CXToken> &CXTokens, unsigned options) { + SourceManager &SourceMgr = CXXUnit->getSourceManager(); + std::pair<FileID, unsigned> BeginLocInfo + = SourceMgr.getDecomposedSpellingLoc(Range.getBegin()); +@@ -6314,6 +6314,9 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + CXXUnit->getASTContext().getLangOpts(), + Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end()); + Lex.SetCommentRetentionState(true); ++ if (options & CXTokenize_KeepWhitespace) { ++ Lex.SetKeepWhitespaceMode(true); ++ } + + // Lex tokens until we hit the end of the range. + const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second; +@@ -6365,6 +6368,12 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + + void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + CXToken **Tokens, unsigned *NumTokens) { ++ return clang_tokenizeRange(TU, Range, Tokens, NumTokens, CXTokenize_None); ++} ++ ++void clang_tokenizeRange(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens, ++ unsigned options) { + LOG_FUNC_SECTION { + *Log << TU << ' ' << Range; + } +@@ -6390,7 +6399,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + return; + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, R, CXTokens); ++ getTokens(CXXUnit, R, CXTokens, options); + + if (CXTokens.empty()) + return; +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 9c56e88052..b8e3df23ef 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -316,6 +316,7 @@ clang_suspendTranslationUnit + clang_sortCodeCompletionResults + clang_toggleCrashRecovery + clang_tokenize ++clang_tokenizeRange + clang_CompilationDatabase_fromDirectory + clang_CompilationDatabase_dispose + clang_CompilationDatabase_getCompileCommands +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0008-Fix-printing-policy-for-AST-context-loaded-from-file.patch b/packages/llvm/llvm5-0008-Fix-printing-policy-for-AST-context-loaded-from-file.patch new file mode 100644 index 0000000000000000000000000000000000000000..95ae8c9fc8bf36b8c094afd30ae6cc8abcd8c8ee --- /dev/null +++ b/packages/llvm/llvm5-0008-Fix-printing-policy-for-AST-context-loaded-from-file.patch @@ -0,0 +1,137 @@ +From f612539aaee3a46abbbc8fa30d0263d9b49dca86 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Mon, 10 Jul 2017 12:34:10 +0200 +Subject: [PATCH 08/12] Fix printing policy for AST context loaded from file + +In ASTUnit::LoadFromASTFile, the context object is set up using default-constructed +LangOptions (which only later get populated). As the language options are used in the constructor +of PrintingPolicy, this needs to be updated explicitly after the language options are available. +--- + lib/Frontend/ASTUnit.cpp | 3 ++ + unittests/Frontend/ASTUnitTest.cpp | 87 ++++++++++++++++++++++++++++++++++++++ + unittests/Frontend/CMakeLists.txt | 1 + + 3 files changed, 91 insertions(+) + create mode 100644 unittests/Frontend/ASTUnitTest.cpp + +diff --git a/tools/clang/lib/Frontend/ASTUnit.cpp b/tools/clang/lib/Frontend/ASTUnit.cpp +index 875f21d69a..bb1ea6c4f8 100644 +--- a/tools/clang/lib/Frontend/ASTUnit.cpp ++++ b/tools/clang/lib/Frontend/ASTUnit.cpp +@@ -546,6 +546,9 @@ private: + // Initialize the ASTContext + Context->InitBuiltinTypes(*Target); + ++ // Adjust printing policy based on language options. ++ Context->setPrintingPolicy(PrintingPolicy(LangOpt)); ++ + // We didn't have access to the comment options when the ASTContext was + // constructed, so register them now. + Context->getCommentCommandTraits().registerCommentOptions( +diff --git a/tools/clang/unittests/Frontend/ASTUnitTest.cpp b/tools/clang/unittests/Frontend/ASTUnitTest.cpp +new file mode 100644 +index 0000000000..a7d08a992f +--- /dev/null ++++ b/tools/clang/unittests/Frontend/ASTUnitTest.cpp +@@ -0,0 +1,87 @@ ++//===- unittests/Frontend/ASTUnitTest.cpp - ASTUnit tests -----------------===// ++// ++// The LLVM Compiler Infrastructure ++// ++// This file is distributed under the University of Illinois Open Source ++// License. See LICENSE.TXT for details. ++// ++//===----------------------------------------------------------------------===// ++ ++#include <fstream> ++ ++#include "clang/Frontend/ASTUnit.h" ++#include "clang/Frontend/CompilerInstance.h" ++#include "clang/Frontend/CompilerInvocation.h" ++#include "clang/Frontend/PCHContainerOperations.h" ++#include "llvm/Support/FileSystem.h" ++#include "llvm/Support/Path.h" ++#include "llvm/Support/ToolOutputFile.h" ++#include "gtest/gtest.h" ++ ++using namespace llvm; ++using namespace clang; ++ ++namespace { ++ ++TEST(ASTUnit, SaveLoadPreservesLangOptionsInPrintingPolicy) { ++ // Check that the printing policy is restored with the correct language ++ // options when loading an ASTUnit from a file. To this end, an ASTUnit ++ // for a C++ translation unit is set up and written to a temporary file. ++ ++ // By default `UseVoidForZeroParams` is true for non-C++ language options, ++ // thus we can check this field after loading the ASTUnit to deduce whether ++ // the correct (C++) language options were used when setting up the printing ++ // policy. ++ ++ { ++ PrintingPolicy PolicyWithDefaultLangOpt(LangOptions{}); ++ EXPECT_TRUE(PolicyWithDefaultLangOpt.UseVoidForZeroParams); ++ } ++ ++ int FD; ++ llvm::SmallString<256> InputFileName; ++ ASSERT_FALSE(llvm::sys::fs::createTemporaryFile("ast-unit", "cpp", FD, InputFileName)); ++ tool_output_file input_file(InputFileName, FD); ++ input_file.os() << ""; ++ ++ const char* Args[] = {"clang", "-xc++", InputFileName.c_str()}; ++ ++ IntrusiveRefCntPtr<DiagnosticsEngine> Diags = ++ CompilerInstance::createDiagnostics(new DiagnosticOptions()); ++ ++ std::shared_ptr<CompilerInvocation> CInvok = ++ createInvocationFromCommandLine(Args, Diags); ++ ++ if (!CInvok) ++ FAIL() << "could not create compiler invocation"; ++ ++ FileManager *FileMgr = ++ new FileManager(FileSystemOptions(), vfs::getRealFileSystem()); ++ auto PCHContainerOps = std::make_shared<PCHContainerOperations>(); ++ ++ std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation( ++ CInvok, PCHContainerOps, Diags, FileMgr); ++ ++ if (!AST) ++ FAIL() << "failed to create ASTUnit"; ++ ++ EXPECT_FALSE(AST->getASTContext().getPrintingPolicy().UseVoidForZeroParams); ++ ++ llvm::SmallString<256> ASTFileName; ++ ASSERT_FALSE(llvm::sys::fs::createTemporaryFile("ast-unit", "ast", FD, ASTFileName)); ++ tool_output_file ast_file(ASTFileName, FD); ++ AST->Save(ASTFileName.str()); ++ ++ EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName)); ++ ++ std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile( ++ ASTFileName.str(), PCHContainerOps->getRawReader(), ASTUnit::LoadEverything, Diags, ++ FileSystemOptions(), /*UseDebugInfo=*/false); ++ ++ if (!AU) ++ FAIL() << "failed to load ASTUnit"; ++ ++ EXPECT_FALSE(AU->getASTContext().getPrintingPolicy().UseVoidForZeroParams); ++} ++ ++} // anonymous namespace +diff --git a/tools/clang/unittests/Frontend/CMakeLists.txt b/tools/clang/unittests/Frontend/CMakeLists.txt +index 674f77bd01..4312151c04 100644 +--- a/tools/clang/unittests/Frontend/CMakeLists.txt ++++ b/tools/clang/unittests/Frontend/CMakeLists.txt +@@ -3,6 +3,7 @@ set(LLVM_LINK_COMPONENTS + ) + + add_clang_unittest(FrontendTests ++ ASTUnitTest.cpp + FrontendActionTest.cpp + CodeGenActionTest.cpp + ) +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0009-libclang-Visit-attributes-for-function-and-class-tem.patch b/packages/llvm/llvm5-0009-libclang-Visit-attributes-for-function-and-class-tem.patch new file mode 100644 index 0000000000000000000000000000000000000000..d38d190db89771be6ee98ec92a0e3426a943f19e --- /dev/null +++ b/packages/llvm/llvm5-0009-libclang-Visit-attributes-for-function-and-class-tem.patch @@ -0,0 +1,97 @@ +From 2d3c0e5f3e3e6da701f3a5010a9700253deec16d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Fri, 21 Jul 2017 10:16:45 +0200 +Subject: [PATCH 09/12] [libclang] Visit attributes for function and class + templates + +--- + bindings/python/tests/cindex/test_cursor.py | 20 ++++++++++++++++++++ + test/Index/annotate-attribute.cpp | 12 ++++++++++++ + tools/libclang/CIndex.cpp | 6 ++++-- + 3 files changed, 36 insertions(+), 2 deletions(-) + +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 2d50ec5901..863919e4c5 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -397,6 +397,26 @@ def test_annotation_attribute(): + else: + assert False, "Couldn't find annotation" + ++def test_annotation_template(): ++ annotation = '__attribute__ ((annotate("annotation")))' ++ for source, kind in [ ++ ('int foo (T value) %s;', CursorKind.FUNCTION_TEMPLATE), ++ ('class %s foo {};', CursorKind.CLASS_TEMPLATE), ++ ]: ++ source = 'template<typename T> ' + (source % annotation) ++ tu = get_tu(source, lang="cpp") ++ ++ foo = get_cursor(tu, 'foo') ++ assert foo is not None ++ assert foo.kind == kind ++ ++ for c in foo.get_children(): ++ if c.kind == CursorKind.ANNOTATE_ATTR: ++ assert c.displayname == "annotation" ++ break ++ else: ++ assert False, "Couldn't find annotation for {}".format(kind) ++ + def test_result_type(): + tu = get_tu('int foo();') + foo = get_cursor(tu, 'foo') +diff --git a/tools/clang/test/Index/annotate-attribute.cpp b/tools/clang/test/Index/annotate-attribute.cpp +index d822210e49..bf415fc8fe 100644 +--- a/tools/clang/test/Index/annotate-attribute.cpp ++++ b/tools/clang/test/Index/annotate-attribute.cpp +@@ -16,6 +16,12 @@ protected: + void methodWithoutAttribute(); + }; + ++template <typename T> ++class __attribute__((annotate("works"))) TemplateTest {}; ++ ++template <typename T> ++int templateFunction(T value) __attribute__((annotate("works"))); ++ + // CHECK: ClassDecl=Test:3:7 (Definition) Extent=[3:1 - 17:2] + // CHECK-NEXT: CXXAccessSpecifier=:4:1 (Definition) Extent=[4:1 - 4:8] + // CHECK-NEXT: CXXMethod=aMethod:5:51 Extent=[5:3 - 5:60] +@@ -31,3 +37,9 @@ protected: + // CHECK-NEXT: CompoundStmt= Extent=[12:23 - 12:25] + // CHECK-NEXT: CXXAccessSpecifier=:14:1 (Definition) Extent=[14:1 - 14:11] + // CHECK-NEXT: CXXMethod=methodWithoutAttribute:16:8 Extent=[16:3 - 16:32] ++// CHECK: ClassTemplate=TemplateTest:20:42 (Definition) Extent=[19:1 - 20:57] ++// CHECK-NEXT: TemplateTypeParameter=T:19:20 (Definition) Extent=[19:11 - 19:21] [access=public] ++// CHECK-NEXT: attribute(annotate)=works Extent=[20:22 - 20:39] ++// CHECK: FunctionTemplate=templateFunction:23:5 Extent=[22:1 - 23:65] ++// CHECK-NEXT: TemplateTypeParameter=T:22:20 (Definition) Extent=[22:11 - 22:21] [access=public] ++// CHECK-NEXT: attribute(annotate)=works Extent=[23:46 - 23:63] +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/clang/tools/libclang/CIndex.cpp +index 04fd775fb0..27f74b2aa2 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -907,7 +907,8 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { + if (VisitTemplateParameters(D->getTemplateParameters())) + return true; + +- return VisitFunctionDecl(D->getTemplatedDecl()); ++ auto* FD = D->getTemplatedDecl(); ++ return VisitAttributes(FD) || VisitFunctionDecl(FD); + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -916,7 +917,8 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { + if (VisitTemplateParameters(D->getTemplateParameters())) + return true; + +- return VisitCXXRecordDecl(D->getTemplatedDecl()); ++ auto* CD = D->getTemplatedDecl(); ++ return VisitAttributes(CD) || VisitCXXRecordDecl(CD); + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0010-libclang-Add-support-for-querying-cursor-availabilit.patch b/packages/llvm/llvm5-0010-libclang-Add-support-for-querying-cursor-availabilit.patch new file mode 100644 index 0000000000000000000000000000000000000000..1119bc3b7090fd4d24356697ba86c1fdfc7fbf82 --- /dev/null +++ b/packages/llvm/llvm5-0010-libclang-Add-support-for-querying-cursor-availabilit.patch @@ -0,0 +1,123 @@ +From 3206e3ef840b1ca16346f2c925edc47c5a9697ad Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <dev@jklaehn.de> +Date: Tue, 25 Jul 2017 15:49:14 +0200 +Subject: [PATCH 10/12] [libclang] Add support for querying cursor availability + +This patch allows checking the availability of cursors through libclang and clang.cindex (Python). +This e.g. allows to check whether a C++ member function has been marked as deleted. +--- + bindings/python/clang/cindex.py | 33 +++++++++++++++++++++++++++++ + bindings/python/tests/cindex/test_cursor.py | 25 ++++++++++++++++++++++ + 2 files changed, 58 insertions(+) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index a5bb58b44f..b21f2b75f2 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1586,6 +1586,16 @@ class Cursor(Structure): + return StorageClass.from_id(self._storage_class) + + @property ++ def availability(self): ++ """ ++ Retrieves the availability of the entity pointed at by the cursor. ++ """ ++ if not hasattr(self, '_availability'): ++ self._availability = conf.lib.clang_getCursorAvailability(self) ++ ++ return AvailabilityKind.from_id(self._availability) ++ ++ @property + def access_specifier(self): + """ + Retrieves the access specifier (if any) of the entity pointed at by the +@@ -1925,6 +1935,24 @@ StorageClass.OPENCLWORKGROUPLOCAL = StorageClass(5) + StorageClass.AUTO = StorageClass(6) + StorageClass.REGISTER = StorageClass(7) + ++### Availability Kinds ### ++ ++class AvailabilityKind(BaseEnumeration): ++ """ ++ Describes the availability of an entity. ++ """ ++ ++ # The unique kind objects, indexed by id. ++ _kinds = [] ++ _name_map = None ++ ++ def __repr__(self): ++ return 'AvailabilityKind.%s' % (self.name,) ++ ++AvailabilityKind.AVAILABLE = AvailabilityKind(0) ++AvailabilityKind.DEPRECATED = AvailabilityKind(1) ++AvailabilityKind.NOT_AVAILABLE = AvailabilityKind(2) ++AvailabilityKind.NOT_ACCESSIBLE = AvailabilityKind(3) + + ### C++ access specifiers ### + +@@ -3472,6 +3500,10 @@ functionList = [ + [TranslationUnit, SourceLocation], + Cursor), + ++ ("clang_getCursorAvailability", ++ [Cursor], ++ c_int), ++ + ("clang_getCursorDefinition", + [Cursor], + Cursor, +@@ -4096,6 +4128,7 @@ conf = Config() + register_enumerations() + + __all__ = [ ++ 'AvailabilityKind', + 'Config', + 'CodeCompletionResults', + 'CompilationDatabase', +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 863919e4c5..6c20577302 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -1,6 +1,7 @@ + import ctypes + import gc + ++from clang.cindex import AvailabilityKind + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind + from clang.cindex import TokenGroup +@@ -425,6 +426,30 @@ def test_result_type(): + t = foo.result_type + assert t.kind == TypeKind.INT + ++def test_availability(): ++ tu = get_tu('class A { A(A const&) = delete; };', lang='cpp') ++ ++ # AvailabilityKind.AVAILABLE ++ cursor = get_cursor(tu, 'A') ++ assert cursor.kind == CursorKind.CLASS_DECL ++ assert cursor.availability == AvailabilityKind.AVAILABLE ++ ++ # AvailabilityKind.NOT_AVAILABLE ++ cursors = get_cursors(tu, 'A') ++ for c in cursors: ++ if c.kind == CursorKind.CONSTRUCTOR: ++ assert c.availability == AvailabilityKind.NOT_AVAILABLE ++ break ++ else: ++ assert False, "Could not find cursor for deleted constructor" ++ ++ # AvailabilityKind.DEPRECATED ++ tu = get_tu('void test() __attribute__((deprecated));', lang='cpp') ++ cursor = get_cursor(tu, 'test') ++ assert cursor.availability == AvailabilityKind.DEPRECATED ++ ++ # AvailabilityKind.NOT_ACCESSIBLE is only used in the code completion results ++ + def test_get_tokens(): + """Ensure we can map cursors back to tokens.""" + tu = get_tu('int foo(int i);') +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0011-libclang-Allow-visiting-of-implicit-declarations-and.patch b/packages/llvm/llvm5-0011-libclang-Allow-visiting-of-implicit-declarations-and.patch new file mode 100644 index 0000000000000000000000000000000000000000..b9835bbd5060fde987f0b2ef0216e8f56e0de36f --- /dev/null +++ b/packages/llvm/llvm5-0011-libclang-Allow-visiting-of-implicit-declarations-and.patch @@ -0,0 +1,433 @@ +From c7701fe91022a116e0a37f410e70fe8906f56441 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 17 Jul 2017 12:25:49 +0200 +Subject: [PATCH 11/12] [libclang] Allow visiting of implicit declarations and + template instantiations (WIP!) + +--- + bindings/python/clang/cindex.py | 45 +++++++++-- + bindings/python/tests/cindex/test_cursor.py | 33 ++++++++ + include/clang-c/Index.h | 31 ++++++++ + tools/libclang/CIndex.cpp | 112 ++++++++++++++++++++++++++-- + tools/libclang/CursorVisitor.h | 12 ++- + tools/libclang/libclang.exports | 2 + + 6 files changed, 219 insertions(+), 16 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index b21f2b75f2..2416fd1803 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1407,6 +1407,15 @@ class Cursor(Structure): + """ + _fields_ = [("_kind_id", c_int), ("xdata", c_int), ("data", c_void_p * 3)] + ++ # Default behavior. ++ GET_CHILDREN_NONE = 0 ++ ++ # Used to indicate that implicit cursors should be visited. ++ GET_CHILDREN_WITH_IMPLICIT = 1 ++ ++ # Used to indicate that template instantiations should be visited. ++ GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS = 2 ++ + @staticmethod + def from_location(tu, location): + # We store a reference to the TU in the instance so the TU won't get +@@ -1496,6 +1505,10 @@ class Cursor(Structure): + """ + return conf.lib.clang_EnumDecl_isScoped(self) + ++ def is_implicit(self): ++ """Test whether the cursor refers to an implicit declaration.""" ++ return conf.lib.clang_isImplicit(self) ++ + def get_definition(self): + """ + If the cursor is a reference to a declaration or a declaration of +@@ -1790,8 +1803,12 @@ class Cursor(Structure): + """Returns the value of the indicated arg as an unsigned 64b integer.""" + return conf.lib.clang_Cursor_getTemplateArgumentUnsignedValue(self, num) + +- def get_children(self): +- """Return an iterator for accessing the children of this cursor.""" ++ def get_children(self, with_implicit=False, with_template_instantiations=False): ++ """Return an iterator for accessing the children of this cursor. ++ ++ By default, cursors representing implicit declarations or template instantiations ++ will be skipped. ++ """ + + # FIXME: Expose iteration from CIndex, PR6125. + def visitor(child, parent, children): +@@ -1804,18 +1821,24 @@ class Cursor(Structure): + children.append(child) + return 1 # continue + children = [] +- conf.lib.clang_visitChildren(self, callbacks['cursor_visit'](visitor), +- children) ++ dispatch = conf.lib.clang_visitChildren ++ options = Cursor.GET_CHILDREN_NONE ++ if with_implicit: ++ options |= Cursor.GET_CHILDREN_WITH_IMPLICIT ++ if with_template_instantiations: ++ options |= Cursor.GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS ++ conf.lib.clang_visitChildrenWithOptions( ++ self, callbacks['cursor_visit'](visitor), children, options) + return iter(children) + +- def walk_preorder(self): ++ def walk_preorder(self, **kwargs): + """Depth-first preorder walk over the cursor and its descendants. + + Yields cursors. + """ + yield self +- for child in self.get_children(): +- for descendant in child.walk_preorder(): ++ for child in self.get_children(**kwargs): ++ for descendant in child.walk_preorder(**kwargs): + yield descendant + + def get_tokens(self, options=0): +@@ -3840,6 +3863,10 @@ functionList = [ + [Type], + bool), + ++ ("clang_isImplicit", ++ [Cursor], ++ bool), ++ + ("clang_isInvalid", + [CursorKind], + bool), +@@ -3903,6 +3930,10 @@ functionList = [ + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), + ++ ("clang_visitChildrenWithOptions", ++ [Cursor, callbacks['cursor_visit'], py_object, c_uint], ++ c_uint), ++ + ("clang_Cursor_getNumArguments", + [Cursor], + c_int), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 6c20577302..43606b605c 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -70,6 +70,39 @@ def test_get_children(): + assert tu_nodes[2].displayname == 'f0(int, int)' + assert tu_nodes[2].is_definition() == True + ++def test_get_children_with_implicit(): ++ tu = get_tu('struct X {}; X x;', lang='cpp') ++ cursor = get_cursor(tu, 'X') ++ ++ children = list(cursor.get_children()) ++ assert len(children) == 0 ++ ++ children = list(cursor.get_children(with_implicit=True)) ++ assert len(children) > 0 ++ for child in children: ++ assert child.is_implicit() ++ assert child.spelling == "X" ++ assert child.kind in [CursorKind.CONSTRUCTOR, CursorKind.STRUCT_DECL] ++ ++def test_get_children_with_template_instantiations(): ++ tu = get_tu( ++ 'template <typename T> T frobnicate(T val);' ++ 'extern template int frobnicate<int>(int);', ++ lang='cpp') ++ cursor = get_cursor(tu, 'frobnicate') ++ assert cursor.kind == CursorKind.FUNCTION_TEMPLATE ++ ++ for child in cursor.get_children(): ++ # should not return an instantiation: ++ assert child.kind != CursorKind.FUNCTION_DECL ++ ++ for child in cursor.get_children(with_template_instantiations=True): ++ if child.kind == CursorKind.FUNCTION_DECL: ++ assert child.spelling == 'frobnicate' ++ break ++ else: ++ assert False, "Couldn't find template instantiation" ++ + def test_references(): + """Ensure that references to TranslationUnit are kept.""" + tu = get_tu('int x;') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 7fd17366ee..abe70e9566 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -2670,6 +2670,11 @@ CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind); + */ + CINDEX_LINKAGE unsigned clang_isUnexposed(enum CXCursorKind); + ++/*** ++ * \brief Determine whether the given cursor represents an implicit declaration. ++ */ ++CINDEX_LINKAGE unsigned clang_isImplicit(CXCursor); ++ + /** + * \brief Describe the linkage of the entity referred to by a cursor. + */ +@@ -3961,6 +3966,32 @@ CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent, + # endif + #endif + ++typedef enum { ++ /** ++ * \brief Default behavior. ++ */ ++ CXVisitChildren_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that implicit cursors should be visited. ++ */ ++ CXVisitChildren_WithImplicit = 0x1, ++ ++ /** ++ * \brief Used to indicate that template instantiations should be visited. ++ */ ++ CXVisitChildren_WithTemplateInstantiations = 0x2 ++} CXVisitChildren_Flags; ++ ++/** ++ * \brief Visits the children of a cursor, allowing to pass extra options. ++ * Behaves identically to clang_visitChildren() in all other respects. ++ */ ++CINDEX_LINKAGE unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options); ++ + /** + * @} + */ +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/clang/tools/libclang/CIndex.cpp +index 27f74b2aa2..f32611b8d7 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -192,10 +192,11 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) { + assert(0 && "Invalid declaration cursor"); + return true; // abort. + } +- +- // Ignore implicit declarations, unless it's an objc method because +- // currently we should report implicit methods for properties when indexing. +- if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) ++ ++ // Unless instructed otherwise we ignore implicit declarations. ++ // ObjC methods are currently visited in any case, because implicit methods ++ // for properties should be reported when indexing. ++ if (!VisitImplicitDeclarations && D->isImplicit() && !isa<ObjCMethodDecl>(D)) + return false; + } + +@@ -700,10 +701,13 @@ bool CursorVisitor::VisitTagDecl(TagDecl *D) { + + bool CursorVisitor::VisitClassTemplateSpecializationDecl( + ClassTemplateSpecializationDecl *D) { +- bool ShouldVisitBody = false; ++ bool ShouldVisitBody = VisitTemplateInstantiations; + switch (D->getSpecializationKind()) { +- case TSK_Undeclared: + case TSK_ImplicitInstantiation: ++ if (VisitTemplateInstantiations && VisitImplicitDeclarations) { ++ break; ++ } ++ case TSK_Undeclared: + // Nothing to visit + return false; + +@@ -712,6 +716,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl( + break; + + case TSK_ExplicitSpecialization: ++ // Always visit body of explicit specializations + ShouldVisitBody = true; + break; + } +@@ -908,7 +913,31 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { + return true; + + auto* FD = D->getTemplatedDecl(); +- return VisitAttributes(FD) || VisitFunctionDecl(FD); ++ if (VisitAttributes(FD) || VisitFunctionDecl(FD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *FD : D->specializations()) { ++ for (auto *RD : FD->redecls()) { ++ switch (RD->getTemplateSpecializationKind()) { ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -918,7 +947,40 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { + return true; + + auto* CD = D->getTemplatedDecl(); +- return VisitAttributes(CD) || VisitCXXRecordDecl(CD); ++ if (VisitAttributes(CD) || VisitCXXRecordDecl(CD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *SD : D->specializations()) { ++ for (auto *RD : SD->redecls()) { ++ // We don't want to visit injected-class-names in this traversal. ++ if (cast<CXXRecordDecl>(RD)->isInjectedClassName()) ++ continue; ++ ++ switch ( ++ cast<ClassTemplateSpecializationDecl>(RD)->getSpecializationKind()) { ++ // Visit the implicit instantiations with the requested pattern. ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ // We don't need to do anything on an explicit instantiation ++ // or explicit specialization because there will be an explicit ++ // node for it elsewhere. ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +@@ -4314,6 +4376,24 @@ unsigned clang_visitChildrenWithBlock(CXCursor parent, + return clang_visitChildren(parent, visitWithBlock, block); + } + ++unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options) { ++ CursorVisitor CursorVis( ++ getCursorTU(parent), visitor, client_data, ++ /*VisitPreprocessorLast=*/false, ++ /*VisitIncludedPreprocessingEntries=*/false, ++ /*RegionOfInterest=*/SourceRange(), ++ /*VisitDeclsOnly=*/false, ++ /*PostChildrenVisitor=*/nullptr, ++ /*VisitImplicitDeclarations=*/(options & CXVisitChildren_WithImplicit), ++ /*VisitTemplateInstantiations=*/ ++ (options & CXVisitChildren_WithTemplateInstantiations)); ++ ++ return CursorVis.VisitChildren(parent); ++} ++ + static CXString getDeclSpelling(const Decl *D) { + if (!D) + return cxstring::createEmpty(); +@@ -5402,6 +5482,22 @@ unsigned clang_isUnexposed(enum CXCursorKind K) { + } + } + ++unsigned clang_isImplicit(CXCursor Cursor) { ++ if (clang_isInvalid(Cursor.kind)) ++ return false; ++ ++ if (!clang_isDeclaration(Cursor.kind)) ++ return false; ++ ++ const Decl *D = getCursorDecl(Cursor); ++ if (!D) { ++ assert(0 && "Invalid declaration cursor"); ++ return true; // abort. ++ } ++ ++ return D->isImplicit(); ++} ++ + CXCursorKind clang_getCursorKind(CXCursor C) { + return C.kind; + } +diff --git a/tools/clang/tools/libclang/CursorVisitor.h b/tools/clang/tools/libclang/CursorVisitor.h +index 82f251a348..c659e866ef 100644 +--- a/tools/clang/tools/libclang/CursorVisitor.h ++++ b/tools/clang/tools/libclang/CursorVisitor.h +@@ -96,6 +96,12 @@ private: + /// record entries. + bool VisitDeclsOnly; + ++ /// \brief Whether we should visit implicit declarations. ++ bool VisitImplicitDeclarations; ++ ++ /// \brief Whether we should recurse into template instantiations. ++ bool VisitTemplateInstantiations; ++ + // FIXME: Eventually remove. This part of a hack to support proper + // iteration over all Decls contained lexically within an ObjC container. + DeclContext::decl_iterator *DI_current; +@@ -147,7 +153,9 @@ public: + bool VisitIncludedPreprocessingEntries = false, + SourceRange RegionOfInterest = SourceRange(), + bool VisitDeclsOnly = false, +- PostChildrenVisitorTy PostChildrenVisitor = nullptr) ++ PostChildrenVisitorTy PostChildrenVisitor = nullptr, ++ bool VisitImplicitDeclarations = false, ++ bool VisitTemplateInstantiations = false) + : TU(TU), AU(cxtu::getASTUnit(TU)), + Visitor(Visitor), PostChildrenVisitor(PostChildrenVisitor), + ClientData(ClientData), +@@ -155,6 +163,8 @@ public: + VisitIncludedEntities(VisitIncludedPreprocessingEntries), + RegionOfInterest(RegionOfInterest), + VisitDeclsOnly(VisitDeclsOnly), ++ VisitImplicitDeclarations(VisitImplicitDeclarations), ++ VisitTemplateInstantiations(VisitTemplateInstantiations), + DI_current(nullptr), FileDI_current(nullptr) + { + Parent.kind = CXCursor_NoDeclFound; +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index b8e3df23ef..59c46ae09e 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -291,6 +291,7 @@ clang_isDeclaration + clang_isExpression + clang_isFileMultipleIncludeGuarded + clang_isFunctionTypeVariadic ++clang_isImplicit + clang_isInvalid + clang_isPODType + clang_isPreprocessing +@@ -332,6 +333,7 @@ clang_CompileCommand_getNumArgs + clang_CompileCommand_getArg + clang_visitChildren + clang_visitChildrenWithBlock ++clang_visitChildrenWithOptions + clang_ModuleMapDescriptor_create + clang_ModuleMapDescriptor_dispose + clang_ModuleMapDescriptor_setFrameworkModuleName +-- +2.13.0 + diff --git a/packages/llvm/llvm5-0012-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch b/packages/llvm/llvm5-0012-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch new file mode 100644 index 0000000000000000000000000000000000000000..0be6629fff035d51a29441f2d5c45caecffe377c --- /dev/null +++ b/packages/llvm/llvm5-0012-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch @@ -0,0 +1,54 @@ +From 8566bbfaf8a34bf088cacf632b647922257d7d5f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 31 Jul 2017 14:09:52 +0200 +Subject: [PATCH 12/12] [libclang] WIP: Fix get_tokens in macro expansion + +--- + bindings/python/tests/cindex/test_cursor.py | 15 +++++++++++++++ + tools/libclang/CIndex.cpp | 4 ++-- + 2 files changed, 17 insertions(+), 2 deletions(-) + +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index 43606b605c..5c6f9fb320 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -523,6 +523,21 @@ def test_get_tokens_with_whitespace(): + assert ''.join(t.spelling for t in tokens) == source + assert len(tokens) == 27 + ++def test_get_tokens_in_macro(): ++ """regression test""" ++ source = "#define IMPL(name) struct name { name(int v = 0); }; \n IMPL(X)" ++ tu = get_tu(source, lang="cpp") ++ ctor = get_cursors(tu, "X")[1] ++ assert ctor.kind == CursorKind.CONSTRUCTOR ++ p = next(ctor.get_children()) ++ assert p.kind == CursorKind.PARM_DECL ++ children = list(p.get_children()) ++ assert len(children) == 1 ++ expr = children[0] ++ tokens = list(expr.get_tokens()) ++ assert len(tokens) == 1 ++ assert tokens[0].spelling == "0" ++ + def test_get_arguments(): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/clang/tools/libclang/CIndex.cpp +index f32611b8d7..5849c7dfde 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -144,8 +144,8 @@ CXSourceRange cxloc::translateSourceRange(const SourceManager &SM, + // We want the last character in this location, so we will adjust the + // location accordingly. + SourceLocation EndLoc = R.getEnd(); +- if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) +- EndLoc = SM.getExpansionRange(EndLoc).second; ++ // if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) ++ // EndLoc = SM.getExpansionRange(EndLoc).second; + if (R.isTokenRange() && EndLoc.isValid()) { + unsigned Length = Lexer::MeasureTokenLength(SM.getSpellingLoc(EndLoc), + SM, LangOpts); +-- +2.13.0 + diff --git a/packages/llvm/llvm7-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch b/packages/llvm/llvm7-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch new file mode 100644 index 0000000000000000000000000000000000000000..eba3b7a7164106b0b6a6dc84b2dd3b5c08f5023f --- /dev/null +++ b/packages/llvm/llvm7-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch @@ -0,0 +1,77 @@ +From 5916da23627103563e38702de2d3bcff65d60406 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Wed, 26 Jul 2017 15:06:10 +0200 +Subject: [PATCH 1/5] [Tooling] Fully qualify template parameters of nested + name specifier in getFullyQualifiedName + +--- + tools/clang/lib/AST/QualTypeNames.cpp | 18 ++++++++++++++---- + tools/clang/unittests/Tooling/QualTypeNamesTest.cpp | 8 +++++++- + 2 files changed, 21 insertions(+), 5 deletions(-) + +diff --git a/tools/clang/lib/AST/QualTypeNames.cpp b/tools/clang/lib/AST/QualTypeNames.cpp +index 8b605ef295..8fcedccf48 100644 +--- a/tools/clang/lib/AST/QualTypeNames.cpp ++++ b/tools/clang/lib/AST/QualTypeNames.cpp +@@ -359,11 +359,21 @@ NestedNameSpecifier *createNestedNameSpecifier(const ASTContext &Ctx, + const TypeDecl *TD, + bool FullyQualify, + bool WithGlobalNsPrefix) { ++ const Type *TypePtr = TD->getTypeForDecl(); ++ // In case of template specializations iterate over the arguments and ++ // fully qualify them as well. ++ if (isa<const TemplateSpecializationType>(TypePtr) || ++ isa<const RecordType>(TypePtr)) { ++ // We are asked to fully qualify and we have a Record Type (which ++ // may point to a template specialization) or Template ++ // Specialization Type. We need to fully qualify their arguments. ++ ++ TypePtr = getFullyQualifiedTemplateType(Ctx, TypePtr, WithGlobalNsPrefix); ++ } ++ + return NestedNameSpecifier::Create( +- Ctx, +- createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), +- false /*No TemplateKeyword*/, +- TD->getTypeForDecl()); ++ Ctx, createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), ++ false /*No TemplateKeyword*/, TypePtr); + } + + /// Return the fully qualified type, including fully-qualified +diff --git a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +index b4c56f7bd5..e9ab495098 100644 +--- a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp ++++ b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +@@ -67,6 +67,10 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + // Template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckC"] = + "A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>"; ++ // Template parameters of nested name specifier should also be fully expanded. ++ Visitor.ExpectedQualTypeNames["CheckNested"] = ++ // "typename A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>::nested"; ++ "typename A::B::Template0<int, A::B::Class0>::nested"; + // Recursive template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckD"] = + "A::B::Template0<A::B::Template1<A::B::C::MyInt, A::B::AnotherClass>, " +@@ -109,7 +113,7 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " using InnerAlias = OuterTemplateClass<T>;\n" + " InnerAlias<int> AliasTypeVal;\n" + " }\n" +- " template<class X, class Y> class Template0;" ++ " template<class X, class Y> struct Template0 { typedef int nested; };" + " template<class X, class Y> class Template1;" + " typedef B::Class0 AnotherClass;\n" + " void Function1(Template0<C::MyInt,\n" +@@ -117,6 +121,8 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " void Function2(Template0<Template1<C::MyInt, AnotherClass>,\n" + " Template0<int, long> > CheckD);\n" + " void Function3(const B::Class0* CheckM);\n" ++ " void Function4(typename Template0<C::MyInt,\n" ++ " AnotherClass>::nested CheckNested);\n" + " }\n" + "template<typename... Values> class Variadic {};\n" + "Variadic<int, B::Template0<int, char>, " +-- +2.18.0 + diff --git a/packages/llvm/llvm7-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch b/packages/llvm/llvm7-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch new file mode 100644 index 0000000000000000000000000000000000000000..ec669b0dfdba49729c2b7f8ad6d4477211409169 --- /dev/null +++ b/packages/llvm/llvm7-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch @@ -0,0 +1,153 @@ +From 18b47d781d631667a40a543383cf2510968440e3 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 11:17:59 +0100 +Subject: [PATCH 2/5] [libclang] Add support for obtaining fully qualified + names of types + +This patch allows retrieving the fully qualified names of types +through libclang and clang.cindex (Python). +--- + tools/clang/bindings/python/clang/cindex.py | 13 ++++++++++++ + tools/clang/bindings/python/tests/cindex/test_cursor.py | 8 ++++++++ + tools/clang/include/clang-c/Index.h | 8 ++++++++ + tools/clang/tools/libclang/CMakeLists.txt | 1 + + tools/clang/tools/libclang/CXType.cpp | 22 +++++++++++++++++++++ + tools/clang/tools/libclang/libclang.exports | 1 + + 6 files changed, 53 insertions(+) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 56fcc78763..becf745040 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -2401,6 +2401,14 @@ class Type(Structure): + """Retrieve the spelling of this Type.""" + return conf.lib.clang_getTypeSpelling(self) + ++ @property ++ def fully_qualified_name(self): ++ """Retrieve the fully qualified name of this Type.""" ++ if not hasattr(self, '_fully_qualified_name'): ++ self._fully_qualified_name = conf.lib.clang_getFullyQualifiedTypeName(self) ++ ++ return self._fully_qualified_name ++ + def __eq__(self, other): + if type(other) != type(self): + return False +@@ -3847,6 +3855,11 @@ functionList = [ + _CXString, + _CXString.from_result), + ++ ("clang_getFullyQualifiedTypeName", ++ [Type], ++ _CXString, ++ _CXString.from_result), ++ + ("clang_hashCursor", + [Cursor], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index f5733fd158..26d5f01b23 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -311,6 +311,14 @@ class TestCursor(unittest.TestCase): + underlying = typedef.underlying_typedef_type + self.assertEqual(underlying.kind, TypeKind.INT) + ++ def test_fully_qualified_type_name(): ++ source = 'namespace uiae { struct X { typedef int sometype; }; }' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'sometype') ++ fqn = cls.type.fully_qualified_name ++ self.assertTrue(fqn.endswith("uiae::X::sometype"), fqn) ++ + def test_semantic_parent(self): + tu = get_tu(kParentTest, 'cpp') + curs = get_cursors(tu, 'f') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 65dada38b0..a9bd6f4ebf 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -3318,6 +3318,14 @@ CINDEX_LINKAGE CXType clang_getCursorType(CXCursor C); + */ + CINDEX_LINKAGE CXString clang_getTypeSpelling(CXType CT); + ++/** ++ * Retrieve the fully qualified name of the underlying type. ++ * This includes full qualification of all template parameters etc. ++ * ++ * If the type is invalid, an empty string is returned. ++ */ ++CINDEX_LINKAGE CXString clang_getFullyQualifiedTypeName(CXType CT); ++ + /** + * Retrieve the underlying type of a typedef declaration. + * +diff --git a/tools/clang/tools/libclang/CMakeLists.txt b/tools/clang/tools/libclang/CMakeLists.txt +index e539c8308e..336a5c2e98 100644 +--- a/tools/clang/tools/libclang/CMakeLists.txt ++++ b/tools/clang/tools/libclang/CMakeLists.txt +@@ -41,6 +41,7 @@ set(LIBS + clangLex + clangSema + clangTooling ++ clangToolingCore + ) + + if (CLANG_ENABLE_ARCMT) +diff --git a/tools/libclang/CXType.cpp b/tools/libclang/CXType.cpp +index 7c0f307944..9651c3a84a 100644 +--- a/tools/clang/tools/libclang/CXType.cpp ++++ b/tools/clang/tools/libclang/CXType.cpp +@@ -20,6 +20,7 @@ + #include "clang/AST/DeclObjC.h" + #include "clang/AST/DeclTemplate.h" + #include "clang/AST/Expr.h" ++#include "clang/AST/QualTypeNames.h" + #include "clang/AST/Type.h" + #include "clang/Basic/AddressSpaces.h" + #include "clang/Frontend/ASTUnit.h" +@@ -293,6 +294,27 @@ CXString clang_getTypeSpelling(CXType CT) { + return cxstring::createDup(OS.str()); + } + ++CXString clang_getFullyQualifiedTypeName(CXType CT) { ++ QualType T = GetQualType(CT); ++ if (T.isNull()) ++ return cxstring::createEmpty(); ++ ++ // For builtin types (but not typedefs pointing to builtin types) return their ++ // spelling. Otherwise "bool" will be turned into "_Bool". ++ const Type *TP = T.getTypePtrOrNull(); ++ if (TP && TP->isBuiltinType() && T->getAs<TypedefType>() == nullptr) ++ return clang_getTypeSpelling(CT); ++ ++ CXTranslationUnit TU = GetTU(CT); ++ ASTContext &Ctx = cxtu::getASTUnit(TU)->getASTContext(); ++ PrintingPolicy Policy(Ctx.getPrintingPolicy()); ++ Policy.SuppressScope = false; ++ Policy.AnonymousTagLocations = false; ++ Policy.PolishForDeclaration = true; ++ std::string name = TypeName::getFullyQualifiedName(T, Ctx, Policy, /*WithGlobalNsPrefix=*/true); ++ return cxstring::createDup(name.c_str()); ++} ++ + CXType clang_getTypedefDeclUnderlyingType(CXCursor C) { + using namespace cxcursor; + CXTranslationUnit TU = cxcursor::getCursorTU(C); +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 95a42712c4..87773b9036 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -230,6 +230,7 @@ clang_getFileLocation + clang_getFileName + clang_getFileTime + clang_getFileUniqueID ++clang_getFullyQualifiedTypeName + clang_getFunctionTypeCallingConv + clang_getIBOutletCollectionType + clang_getIncludedFile +-- +2.18.0 + diff --git a/packages/llvm/llvm7-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch b/packages/llvm/llvm7-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch new file mode 100644 index 0000000000000000000000000000000000000000..e00f2c7d412443241994de6fcfeedfddbd881433 --- /dev/null +++ b/packages/llvm/llvm7-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch @@ -0,0 +1,261 @@ +From b0c94bdd15d06359177609cd204da718844bb4d8 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 21:19:51 +0100 +Subject: [PATCH 3/5] [libclang] Add option to keep whitespace when tokenizing + +Introduces new `clang_tokenizeRange` function which accepts options to control +tokenization behavior. `clang_tokenize` is kept for backwards compatibility. +--- + bindings/python/clang/cindex.py | 31 +++++++++++++++----- + bindings/python/tests/cindex/test_cursor.py | 9 ++++++ + include/clang-c/Index.h | 32 +++++++++++++++++++-- + tools/libclang/CIndex.cpp | 15 ++++++++-- + tools/libclang/libclang.exports | 1 + + 5 files changed, 75 insertions(+), 13 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/bindings/python/clang/cindex.py +index becf745040..a43fcff150 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -515,6 +515,13 @@ class TokenGroup(object): + + You should not instantiate this class outside of this module. + """ ++ ++ # Default tokenization mode. ++ TOKENIZE_NONE = 0 ++ ++ # Used to indicate that tokens for whitespace should be returned. ++ TOKENIZE_KEEP_WHITESPACE = 1 ++ + def __init__(self, tu, memory, count): + self._tu = tu + self._memory = memory +@@ -524,7 +531,7 @@ class TokenGroup(object): + conf.lib.clang_disposeTokens(self._tu, self._memory, self._count) + + @staticmethod +- def get_tokens(tu, extent): ++ def get_tokens(tu, extent, options=0): + """Helper method to return all tokens in an extent. + + This functionality is needed multiple places in this module. We define +@@ -533,8 +540,8 @@ class TokenGroup(object): + tokens_memory = POINTER(Token)() + tokens_count = c_uint() + +- conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), +- byref(tokens_count)) ++ conf.lib.clang_tokenizeRange( ++ tu, extent, byref(tokens_memory), byref(tokens_count), options) + + count = int(tokens_count.value) + +@@ -1834,13 +1841,16 @@ class Cursor(Structure): + for descendant in child.walk_preorder(): + yield descendant + +- def get_tokens(self): ++ def get_tokens(self, options=0): + """Obtain Token instances formulating that compose this Cursor. + + This is a generator for Token instances. It returns all tokens which + occupy the extent this cursor occupies. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ +- return TokenGroup.get_tokens(self._tu, self.extent) ++ return TokenGroup.get_tokens(self._tu, self.extent, options) + + def get_field_offsetof(self): + """Returns the offsetof the FIELD_DECL pointed by this Cursor.""" +@@ -3058,18 +3068,21 @@ class TranslationUnit(ClangObject): + return CodeCompletionResults(ptr) + return None + +- def get_tokens(self, locations=None, extent=None): ++ def get_tokens(self, locations=None, extent=None, options=0): + """Obtain tokens in this translation unit. + + This is a generator for Token instances. The caller specifies a range + of source code to obtain tokens for. The range can be specified as a + 2-tuple of SourceLocation or as a SourceRange. If both are defined, + behavior is undefined. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ + if locations is not None: + extent = SourceRange(start=locations[0], end=locations[1]) + +- return TokenGroup.get_tokens(self, extent) ++ return TokenGroup.get_tokens(self, extent, options) + + class File(ClangObject): + """ +@@ -3947,6 +3960,10 @@ functionList = [ + ("clang_tokenize", + [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]), + ++ ("clang_tokenizeRange", ++ [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint), ++ c_uint]), ++ + ("clang_visitChildren", + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/bindings/python/tests/cindex/test_cursor.py +index 26d5f01b23..d7ddf83130 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -5,6 +5,7 @@ import unittest + from clang.cindex import AvailabilityKind + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind ++from clang.cindex import TokenGroup + from clang.cindex import TranslationUnit + from clang.cindex import TypeKind + from .util import get_cursor +@@ -483,6 +484,14 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tokens[0].spelling, 'int') + self.assertEqual(tokens[1].spelling, 'foo') + ++ def test_get_tokens_with_whitespace(): ++ source = 'class C { void f(); }\nvoid C::f() { }' ++ tu = get_tu(source) ++ ++ tokens = list(tu.cursor.get_tokens(TokenGroup.TOKENIZE_KEEP_WHITESPACE)) ++ self.assertEqual(''.join(t.spelling for t in tokens), source) ++ self.assertEqual(len(tokens), 27, [t.spelling for t in tokens]) ++ + def test_get_token_cursor(self): + """Ensure we can map tokens to cursors.""" + tu = get_tu('class A {}; int foo(A var = A());', lang='cpp') +diff --git a/tools/clang/include/clang-c/Index.h b/include/clang-c/Index.h +index a9bd6f4ebf..b77483f5a9 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 49 ++#define CINDEX_VERSION_MINOR 50 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -4799,6 +4799,28 @@ CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit, + */ + CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + ++typedef enum { ++ /** ++ * \brief Used to indicate that no special tokenization options are needed. ++ */ ++ CXTokenize_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that tokens for whitespace should be returned. ++ */ ++ CXTokenize_KeepWhitespace = 0x1 ++} CXTokenize_Flags; ++ ++/** ++ * \brief Tokenize the source code described by the given range into raw ++ * lexical tokens. ++ * ++ * \see clang_tokenizeRange ++ * ++ */ ++CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens); ++ + /** + * Tokenize the source code described by the given range into raw + * lexical tokens. +@@ -4815,9 +4837,13 @@ CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + * \param NumTokens will be set to the number of tokens in the \c *Tokens + * array. + * ++ * \param options A bitmask of options that affects tokenization. This should be ++ * a bitwise OR of the CXTokenize_XXX flags. ++ * + */ +-CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, +- CXToken **Tokens, unsigned *NumTokens); ++CINDEX_LINKAGE void clang_tokenizeRange(CXTranslationUnit TU, ++ CXSourceRange Range, CXToken **Tokens, ++ unsigned *NumTokens, unsigned options); + + /** + * Annotate the given set of tokens by providing cursors for each token +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp +index 499d9abf9a..bbc29af346 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -6585,7 +6585,7 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) { + } + + static void getTokens(ASTUnit *CXXUnit, SourceRange Range, +- SmallVectorImpl<CXToken> &CXTokens) { ++ SmallVectorImpl<CXToken> &CXTokens, unsigned options) { + SourceManager &SourceMgr = CXXUnit->getSourceManager(); + std::pair<FileID, unsigned> BeginLocInfo + = SourceMgr.getDecomposedSpellingLoc(Range.getBegin()); +@@ -6607,6 +6607,9 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + CXXUnit->getASTContext().getLangOpts(), + Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end()); + Lex.SetCommentRetentionState(true); ++ if (options & CXTokenize_KeepWhitespace) { ++ Lex.SetKeepWhitespaceMode(true); ++ } + + // Lex tokens until we hit the end of the range. + const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second; +@@ -6680,7 +6683,7 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + SourceLocation End = SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second); + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, SourceRange(Begin, End), CXTokens); ++ getTokens(CXXUnit, SourceRange(Begin, End), CXTokens, CXTokenize_None); + + if (CXTokens.empty()) + return NULL; +@@ -6694,6 +6697,12 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + + void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + CXToken **Tokens, unsigned *NumTokens) { ++ return clang_tokenizeRange(TU, Range, Tokens, NumTokens, CXTokenize_None); ++} ++ ++void clang_tokenizeRange(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens, ++ unsigned options) { + LOG_FUNC_SECTION { + *Log << TU << ' ' << Range; + } +@@ -6719,7 +6728,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + return; + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, R, CXTokens); ++ getTokens(CXXUnit, R, CXTokens, options); + + if (CXTokens.empty()) + return; +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/libclang/libclang.exports +index 87773b9036..d185e26a2e 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -327,6 +327,7 @@ clang_suspendTranslationUnit + clang_sortCodeCompletionResults + clang_toggleCrashRecovery + clang_tokenize ++clang_tokenizeRange + clang_CompilationDatabase_fromDirectory + clang_CompilationDatabase_dispose + clang_CompilationDatabase_getCompileCommands +-- +2.18.0 + diff --git a/packages/llvm/llvm7-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch b/packages/llvm/llvm7-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch new file mode 100644 index 0000000000000000000000000000000000000000..50cac737fd66ca1f2c1aefd6be4ad21fe3df2085 --- /dev/null +++ b/packages/llvm/llvm7-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch @@ -0,0 +1,433 @@ +From e11c5d12937f1483ab4b0b509ec4a11ba5789de0 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 17 Jul 2017 12:25:49 +0200 +Subject: [PATCH 4/5] [libclang] WIP: Allow visiting of implicit declarations + and template instantiations + +--- + bindings/python/clang/cindex.py | 45 ++++++-- + bindings/python/tests/cindex/test_cursor.py | 33 ++++++ + include/clang-c/Index.h | 31 ++++++ + tools/libclang/CIndex.cpp | 112 ++++++++++++++++++-- + tools/libclang/CursorVisitor.h | 12 ++- + tools/libclang/libclang.exports | 2 + + 6 files changed, 219 insertions(+), 16 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/bindings/python/clang/cindex.py +index a43fcff150..027f2d7f71 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1408,6 +1408,15 @@ class Cursor(Structure): + """ + _fields_ = [("_kind_id", c_int), ("xdata", c_int), ("data", c_void_p * 3)] + ++ # Default behavior. ++ GET_CHILDREN_NONE = 0 ++ ++ # Used to indicate that implicit cursors should be visited. ++ GET_CHILDREN_WITH_IMPLICIT = 1 ++ ++ # Used to indicate that template instantiations should be visited. ++ GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS = 2 ++ + @staticmethod + def from_location(tu, location): + # We store a reference to the TU in the instance so the TU won't get +@@ -1497,6 +1506,10 @@ class Cursor(Structure): + """ + return conf.lib.clang_EnumDecl_isScoped(self) + ++ def is_implicit(self): ++ """Test whether the cursor refers to an implicit declaration.""" ++ return conf.lib.clang_isImplicit(self) ++ + def get_definition(self): + """ + If the cursor is a reference to a declaration or a declaration of +@@ -1813,8 +1826,12 @@ class Cursor(Structure): + """Returns the value of the indicated arg as an unsigned 64b integer.""" + return conf.lib.clang_Cursor_getTemplateArgumentUnsignedValue(self, num) + +- def get_children(self): +- """Return an iterator for accessing the children of this cursor.""" ++ def get_children(self, with_implicit=False, with_template_instantiations=False): ++ """Return an iterator for accessing the children of this cursor. ++ ++ By default, cursors representing implicit declarations or template instantiations ++ will be skipped. ++ """ + + # FIXME: Expose iteration from CIndex, PR6125. + def visitor(child, parent, children): +@@ -1827,18 +1844,24 @@ class Cursor(Structure): + children.append(child) + return 1 # continue + children = [] +- conf.lib.clang_visitChildren(self, callbacks['cursor_visit'](visitor), +- children) ++ dispatch = conf.lib.clang_visitChildren ++ options = Cursor.GET_CHILDREN_NONE ++ if with_implicit: ++ options |= Cursor.GET_CHILDREN_WITH_IMPLICIT ++ if with_template_instantiations: ++ options |= Cursor.GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS ++ conf.lib.clang_visitChildrenWithOptions( ++ self, callbacks['cursor_visit'](visitor), children, options) + return iter(children) + +- def walk_preorder(self): ++ def walk_preorder(self, **kwargs): + """Depth-first preorder walk over the cursor and its descendants. + + Yields cursors. + """ + yield self +- for child in self.get_children(): +- for descendant in child.walk_preorder(): ++ for child in self.get_children(**kwargs): ++ for descendant in child.walk_preorder(**kwargs): + yield descendant + + def get_tokens(self, options=0): +@@ -3905,6 +3928,10 @@ functionList = [ + [Type], + bool), + ++ ("clang_isImplicit", ++ [Cursor], ++ bool), ++ + ("clang_isInvalid", + [CursorKind], + bool), +@@ -3968,6 +3995,10 @@ functionList = [ + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), + ++ ("clang_visitChildrenWithOptions", ++ [Cursor, callbacks['cursor_visit'], py_object, c_uint], ++ c_uint), ++ + ("clang_Cursor_getNumArguments", + [Cursor], + c_int), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/bindings/python/tests/cindex/test_cursor.py +index d7ddf83130..5a4eee4377 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -89,6 +89,39 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tu_nodes[2].displayname, 'f0(int, int)') + self.assertEqual(tu_nodes[2].is_definition(), True) + ++ def test_get_children_with_implicit(): ++ tu = get_tu('struct X {}; X x;', lang='cpp') ++ cursor = get_cursor(tu, 'X') ++ ++ children = list(cursor.get_children()) ++ self.assertEqual(len(children), 0, [(c.kind, c.spelling) for c in children]) ++ ++ children = list(cursor.get_children(with_implicit=True)) ++ self.assertNotEqual(len(children), 0) ++ for child in children: ++ self.assertTrue(child.is_implicit()) ++ self.assertEqual(child.spelling, "X") ++ self.assertIn(child.kind, [CursorKind.CONSTRUCTOR, CursorKind.STRUCT_DECL]) ++ ++ def test_get_children_with_template_instantiations(): ++ tu = get_tu( ++ 'template <typename T> T frobnicate(T val);' ++ 'extern template int frobnicate<int>(int);', ++ lang='cpp') ++ cursor = get_cursor(tu, 'frobnicate') ++ self.assertEqual(cursor.kind, CursorKind.FUNCTION_TEMPLATE) ++ ++ for child in cursor.get_children(): ++ # should not return an instantiation: ++ self.assertNotEqual(child.kind, CursorKind.FUNCTION_DECL) ++ ++ for child in cursor.get_children(with_template_instantiations=True): ++ if child.kind == CursorKind.FUNCTION_DECL: ++ self.assertEqual(child.spelling, 'frobnicate') ++ break ++ else: ++ self.fail("Couldn't find template instantiation") ++ + def test_references(self): + """Ensure that references to TranslationUnit are kept.""" + tu = get_tu('int x;') +diff --git a/tools/clang/include/clang-c/Index.h b/include/clang-c/Index.h +index b77483f5a9..e5a47c0fca 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -2725,6 +2725,11 @@ CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind); + */ + CINDEX_LINKAGE unsigned clang_isUnexposed(enum CXCursorKind); + ++/*** ++ * \brief Determine whether the given cursor represents an implicit declaration. ++ */ ++CINDEX_LINKAGE unsigned clang_isImplicit(CXCursor); ++ + /** + * Describe the linkage of the entity referred to by a cursor. + */ +@@ -4041,6 +4046,32 @@ CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent, + # endif + #endif + ++typedef enum { ++ /** ++ * \brief Default behavior. ++ */ ++ CXVisitChildren_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that implicit cursors should be visited. ++ */ ++ CXVisitChildren_WithImplicit = 0x1, ++ ++ /** ++ * \brief Used to indicate that template instantiations should be visited. ++ */ ++ CXVisitChildren_WithTemplateInstantiations = 0x2 ++} CXVisitChildren_Flags; ++ ++/** ++ * \brief Visits the children of a cursor, allowing to pass extra options. ++ * Behaves identically to clang_visitChildren() in all other respects. ++ */ ++CINDEX_LINKAGE unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options); ++ + /** + * @} + */ +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp +index bbc29af346..6e0a208d55 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -199,10 +199,11 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) { + assert(0 && "Invalid declaration cursor"); + return true; // abort. + } +- +- // Ignore implicit declarations, unless it's an objc method because +- // currently we should report implicit methods for properties when indexing. +- if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) ++ ++ // Unless instructed otherwise we ignore implicit declarations. ++ // ObjC methods are currently visited in any case, because implicit methods ++ // for properties should be reported when indexing. ++ if (!VisitImplicitDeclarations && D->isImplicit() && !isa<ObjCMethodDecl>(D)) + return false; + } + +@@ -707,10 +708,13 @@ bool CursorVisitor::VisitTagDecl(TagDecl *D) { + + bool CursorVisitor::VisitClassTemplateSpecializationDecl( + ClassTemplateSpecializationDecl *D) { +- bool ShouldVisitBody = false; ++ bool ShouldVisitBody = VisitTemplateInstantiations; + switch (D->getSpecializationKind()) { +- case TSK_Undeclared: + case TSK_ImplicitInstantiation: ++ if (VisitTemplateInstantiations && VisitImplicitDeclarations) { ++ break; ++ } ++ case TSK_Undeclared: + // Nothing to visit + return false; + +@@ -719,6 +723,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl( + break; + + case TSK_ExplicitSpecialization: ++ // Always visit body of explicit specializations + ShouldVisitBody = true; + break; + } +@@ -934,7 +939,31 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { + return true; + + auto* FD = D->getTemplatedDecl(); +- return VisitAttributes(FD) || VisitFunctionDecl(FD); ++ if (VisitAttributes(FD) || VisitFunctionDecl(FD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *FD : D->specializations()) { ++ for (auto *RD : FD->redecls()) { ++ switch (RD->getTemplateSpecializationKind()) { ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -944,7 +973,40 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { + return true; + + auto* CD = D->getTemplatedDecl(); +- return VisitAttributes(CD) || VisitCXXRecordDecl(CD); ++ if (VisitAttributes(CD) || VisitCXXRecordDecl(CD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *SD : D->specializations()) { ++ for (auto *RD : SD->redecls()) { ++ // We don't want to visit injected-class-names in this traversal. ++ if (cast<CXXRecordDecl>(RD)->isInjectedClassName()) ++ continue; ++ ++ switch ( ++ cast<ClassTemplateSpecializationDecl>(RD)->getSpecializationKind()) { ++ // Visit the implicit instantiations with the requested pattern. ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ // We don't need to do anything on an explicit instantiation ++ // or explicit specialization because there will be an explicit ++ // node for it elsewhere. ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +@@ -4391,6 +4453,24 @@ unsigned clang_visitChildrenWithBlock(CXCursor parent, + return clang_visitChildren(parent, visitWithBlock, block); + } + ++unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options) { ++ CursorVisitor CursorVis( ++ getCursorTU(parent), visitor, client_data, ++ /*VisitPreprocessorLast=*/false, ++ /*VisitIncludedPreprocessingEntries=*/false, ++ /*RegionOfInterest=*/SourceRange(), ++ /*VisitDeclsOnly=*/false, ++ /*PostChildrenVisitor=*/nullptr, ++ /*VisitImplicitDeclarations=*/(options & CXVisitChildren_WithImplicit), ++ /*VisitTemplateInstantiations=*/ ++ (options & CXVisitChildren_WithTemplateInstantiations)); ++ ++ return CursorVis.VisitChildren(parent); ++} ++ + static CXString getDeclSpelling(const Decl *D) { + if (!D) + return cxstring::createEmpty(); +@@ -5693,6 +5773,22 @@ unsigned clang_isUnexposed(enum CXCursorKind K) { + } + } + ++unsigned clang_isImplicit(CXCursor Cursor) { ++ if (clang_isInvalid(Cursor.kind)) ++ return false; ++ ++ if (!clang_isDeclaration(Cursor.kind)) ++ return false; ++ ++ const Decl *D = getCursorDecl(Cursor); ++ if (!D) { ++ assert(0 && "Invalid declaration cursor"); ++ return true; // abort. ++ } ++ ++ return D->isImplicit(); ++} ++ + CXCursorKind clang_getCursorKind(CXCursor C) { + return C.kind; + } +diff --git a/tools/clang/tools/libclang/CursorVisitor.h b/tools/libclang/CursorVisitor.h +index f2fb68fab9..02085b1783 100644 +--- a/tools/clang/tools/libclang/CursorVisitor.h ++++ b/tools/clang/tools/libclang/CursorVisitor.h +@@ -96,6 +96,12 @@ private: + /// record entries. + bool VisitDeclsOnly; + ++ /// \brief Whether we should visit implicit declarations. ++ bool VisitImplicitDeclarations; ++ ++ /// \brief Whether we should recurse into template instantiations. ++ bool VisitTemplateInstantiations; ++ + // FIXME: Eventually remove. This part of a hack to support proper + // iteration over all Decls contained lexically within an ObjC container. + DeclContext::decl_iterator *DI_current; +@@ -147,7 +153,9 @@ public: + bool VisitIncludedPreprocessingEntries = false, + SourceRange RegionOfInterest = SourceRange(), + bool VisitDeclsOnly = false, +- PostChildrenVisitorTy PostChildrenVisitor = nullptr) ++ PostChildrenVisitorTy PostChildrenVisitor = nullptr, ++ bool VisitImplicitDeclarations = false, ++ bool VisitTemplateInstantiations = false) + : TU(TU), AU(cxtu::getASTUnit(TU)), + Visitor(Visitor), PostChildrenVisitor(PostChildrenVisitor), + ClientData(ClientData), +@@ -155,6 +163,8 @@ public: + VisitIncludedEntities(VisitIncludedPreprocessingEntries), + RegionOfInterest(RegionOfInterest), + VisitDeclsOnly(VisitDeclsOnly), ++ VisitImplicitDeclarations(VisitImplicitDeclarations), ++ VisitTemplateInstantiations(VisitTemplateInstantiations), + DI_current(nullptr), FileDI_current(nullptr) + { + Parent.kind = CXCursor_NoDeclFound; +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/libclang/libclang.exports +index d185e26a2e..1376f29509 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -302,6 +302,7 @@ clang_isInvalidDeclaration + clang_isExpression + clang_isFileMultipleIncludeGuarded + clang_isFunctionTypeVariadic ++clang_isImplicit + clang_isInvalid + clang_isPODType + clang_isPreprocessing +@@ -343,6 +344,7 @@ clang_CompileCommand_getNumArgs + clang_CompileCommand_getArg + clang_visitChildren + clang_visitChildrenWithBlock ++clang_visitChildrenWithOptions + clang_ModuleMapDescriptor_create + clang_ModuleMapDescriptor_dispose + clang_ModuleMapDescriptor_setFrameworkModuleName +-- +2.18.0 + diff --git a/packages/llvm/llvm7-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch b/packages/llvm/llvm7-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch new file mode 100644 index 0000000000000000000000000000000000000000..e71fd2ccac0d55465b2f870e31d245f49843264d --- /dev/null +++ b/packages/llvm/llvm7-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch @@ -0,0 +1,52 @@ +From 40ddcbc8dd69324bb4c97fd808924c5ee967f3d5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 31 Jul 2017 14:09:52 +0200 +Subject: [PATCH 5/5] [libclang] WIP: Fix get_tokens in macro expansion + +--- + bindings/python/tests/cindex/test_cursor.py | 15 +++++++++++++++ + tools/libclang/CIndex.cpp | 2 +- + 2 files changed, 16 insertions(+), 1 deletion(-) + +diff --git a/bindings/python/tests/cindex/test_cursor.py b/bindings/python/tests/cindex/test_cursor.py +index 5a4eee4377..a275d6b8cf 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -547,6 +547,21 @@ class TestCursor(unittest.TestCase): + r_cursor = t_cursor.referenced # should not raise an exception + self.assertEqual(r_cursor.kind, CursorKind.CLASS_DECL) + ++ def test_get_tokens_in_macro_expansion(self): ++ """regression test""" ++ source = "#define IMPL(name) struct name { name(int v = 123); }; \n IMPL(X)" ++ tu = get_tu(source, lang="cpp") ++ ctor = get_cursors(tu, "X")[1] ++ self.assertEqual(ctor.kind, CursorKind.CONSTRUCTOR) ++ p = next(ctor.get_children()) ++ self.assertEqual(p.kind, CursorKind.PARM_DECL, (p.kind, p.spelling)) ++ children = list(p.get_children()) ++ self.assertEqual(len(children), 1, [(c.kind, c.spelling) for c in children]) ++ expr = children[0] ++ tokens = list(expr.get_tokens()) ++ self.assertEqual(len(tokens), 1, [t.spelling for t in tokens]) ++ self.assertEqual(tokens[0].spelling, "123") ++ + def test_get_arguments(self): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp +index 6e0a208d55..2ba96653ff 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -148,7 +148,7 @@ CXSourceRange cxloc::translateSourceRange(const SourceManager &SM, + // location accordingly. + SourceLocation EndLoc = R.getEnd(); + bool IsTokenRange = R.isTokenRange(); +- if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) { ++ if (false /*FIXME*/ && EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) { + CharSourceRange Expansion = SM.getExpansionRange(EndLoc); + EndLoc = Expansion.getEnd(); + IsTokenRange = Expansion.isTokenRange(); +-- +2.18.0 + diff --git a/packages/llvm/llvm7_intel.patch b/packages/llvm/llvm7_intel.patch new file mode 100644 index 0000000000000000000000000000000000000000..710545a619d263c3cc5f3d30bd3e891a8853b7de --- /dev/null +++ b/packages/llvm/llvm7_intel.patch @@ -0,0 +1,50 @@ +diff --git a/llvm/include/llvm/ADT/StringExtras.h b/llvm/include/llvm/ADT/StringExtras.h +index 71b0e7527cb..3304a378f37 100644 +--- a/llvm/include/llvm/ADT/StringExtras.h ++++ b/llvm/include/llvm/ADT/StringExtras.h +@@ -369,7 +369,7 @@ inline size_t join_items_size(const A1 &A, Args &&... Items) { + template <typename IteratorT> + inline std::string join(IteratorT Begin, IteratorT End, StringRef Separator) { + using tag = typename std::iterator_traits<IteratorT>::iterator_category; +- return detail::join_impl(Begin, End, Separator, tag()); ++ return llvm::detail::join_impl(Begin, End, Separator, tag()); + } + + /// Joins the strings in the range [R.begin(), R.end()), adding Separator +diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +index 25b2efd33c9..40144d96044 100644 +--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h ++++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +@@ -186,8 +186,9 @@ public: + /// topological sort) and it's class is the same regardless of block type. + struct BlockNode { + using IndexType = uint32_t; ++ using IndexTypeLimits = std::numeric_limits<IndexType>; + +- IndexType Index = std::numeric_limits<uint32_t>::max(); ++ IndexType Index = IndexTypeLimits::max(); + + BlockNode() = default; + BlockNode(IndexType Index) : Index(Index) {} +diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp +index 4325d57f73d..96e1e1d5503 100644 +--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp ++++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp +@@ -1423,7 +1423,7 @@ VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + if (SymI->second.getAddress() != 0) { + Q->resolve(Name, SymI->second); + if (Q->isFullyResolved()) +- ActionFlags |= NotifyFullyResolved; ++ ActionFlags = static_cast<LookupImplActionFlags>(ActionFlags | NotifyFullyResolved); + } + + // If the symbol is lazy, get the MaterialiaztionUnit for it. +@@ -1456,7 +1456,7 @@ VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + // continue. + Q->notifySymbolReady(); + if (Q->isFullyReady()) +- ActionFlags |= NotifyFullyReady; ++ ActionFlags = static_cast<LookupImplActionFlags>(ActionFlags | NotifyFullyReady); + continue; + } + diff --git a/packages/llvm/llvm9-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch b/packages/llvm/llvm9-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch new file mode 100644 index 0000000000000000000000000000000000000000..e03dd4cf36c098fda1a23ff713d1b4dd5f1e38e3 --- /dev/null +++ b/packages/llvm/llvm9-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch @@ -0,0 +1,77 @@ +From 33d7d68cb694613dfa836cf506d8af012563b9c2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Wed, 26 Jul 2017 15:06:10 +0200 +Subject: [PATCH 1/5] [Tooling] Fully qualify template parameters of nested + name specifier in getFullyQualifiedName + +--- + clang/lib/AST/QualTypeNames.cpp | 18 ++++++++++++++---- + clang/unittests/Tooling/QualTypeNamesTest.cpp | 8 +++++++- + 2 files changed, 21 insertions(+), 5 deletions(-) + +diff --git a/tools/clang/lib/AST/QualTypeNames.cpp b/tools/clang/lib/AST/QualTypeNames.cpp +index f28f00171cc..ecfef57566e 100644 +--- a/tools/clang/lib/AST/QualTypeNames.cpp ++++ b/tools/clang/lib/AST/QualTypeNames.cpp +@@ -356,11 +356,21 @@ NestedNameSpecifier *createNestedNameSpecifier(const ASTContext &Ctx, + const TypeDecl *TD, + bool FullyQualify, + bool WithGlobalNsPrefix) { ++ const Type *TypePtr = TD->getTypeForDecl(); ++ // In case of template specializations iterate over the arguments and ++ // fully qualify them as well. ++ if (isa<const TemplateSpecializationType>(TypePtr) || ++ isa<const RecordType>(TypePtr)) { ++ // We are asked to fully qualify and we have a Record Type (which ++ // may point to a template specialization) or Template ++ // Specialization Type. We need to fully qualify their arguments. ++ ++ TypePtr = getFullyQualifiedTemplateType(Ctx, TypePtr, WithGlobalNsPrefix); ++ } ++ + return NestedNameSpecifier::Create( +- Ctx, +- createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), +- false /*No TemplateKeyword*/, +- TD->getTypeForDecl()); ++ Ctx, createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), ++ false /*No TemplateKeyword*/, TypePtr); + } + + /// Return the fully qualified type, including fully-qualified +diff --git a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +index b6c30297787..e93a0475812 100644 +--- a/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp ++++ b/tools/clang/unittests/Tooling/QualTypeNamesTest.cpp +@@ -66,6 +66,10 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + // Template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckC"] = + "A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>"; ++ // Template parameters of nested name specifier should also be fully expanded. ++ Visitor.ExpectedQualTypeNames["CheckNested"] = ++ // "typename A::B::Template0<A::B::C::MyInt, A::B::AnotherClass>::nested"; ++ "typename A::B::Template0<int, A::B::Class0>::nested"; + // Recursive template parameter expansion. + Visitor.ExpectedQualTypeNames["CheckD"] = + "A::B::Template0<A::B::Template1<A::B::C::MyInt, A::B::AnotherClass>, " +@@ -108,7 +112,7 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " using InnerAlias = OuterTemplateClass<T>;\n" + " InnerAlias<int> AliasTypeVal;\n" + " }\n" +- " template<class X, class Y> class Template0;" ++ " template<class X, class Y> struct Template0 { typedef int nested; };" + " template<class X, class Y> class Template1;" + " typedef B::Class0 AnotherClass;\n" + " void Function1(Template0<C::MyInt,\n" +@@ -116,6 +120,8 @@ TEST(QualTypeNameTest, getFullyQualifiedName) { + " void Function2(Template0<Template1<C::MyInt, AnotherClass>,\n" + " Template0<int, long> > CheckD);\n" + " void Function3(const B::Class0* CheckM);\n" ++ " void Function4(typename Template0<C::MyInt,\n" ++ " AnotherClass>::nested CheckNested);\n" + " }\n" + "template<typename... Values> class Variadic {};\n" + "Variadic<int, B::Template0<int, char>, " +-- +2.23.0 + diff --git a/packages/llvm/llvm9-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch b/packages/llvm/llvm9-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch new file mode 100644 index 0000000000000000000000000000000000000000..b1da65b18e22bc2f7052a4ed269cab6a9db2a8d7 --- /dev/null +++ b/packages/llvm/llvm9-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch @@ -0,0 +1,162 @@ +From e673a5527dd2df322884eb2498736483df05957d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 11:17:59 +0100 +Subject: [PATCH 2/5] [libclang] Add support for obtaining fully qualified + names of types + +This patch allows retrieving the fully qualified names of types +through libclang and clang.cindex (Python). +--- + clang/bindings/python/clang/cindex.py | 13 +++++++++++ + .../python/tests/cindex/test_cursor.py | 8 +++++++ + clang/include/clang-c/Index.h | 10 ++++++++- + clang/tools/libclang/CMakeLists.txt | 1 + + clang/tools/libclang/CXType.cpp | 22 +++++++++++++++++++ + clang/tools/libclang/libclang.exports | 1 + + 6 files changed, 54 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/tools/clang/bindings/python/clang/cindex.py +index 8e5a9fe0068..c309f7017b2 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -2427,6 +2427,14 @@ class Type(Structure): + """Retrieve the spelling of this Type.""" + return conf.lib.clang_getTypeSpelling(self) + ++ @property ++ def fully_qualified_name(self): ++ """Retrieve the fully qualified name of this Type.""" ++ if not hasattr(self, '_fully_qualified_name'): ++ self._fully_qualified_name = conf.lib.clang_getFullyQualifiedTypeName(self) ++ ++ return self._fully_qualified_name ++ + def __eq__(self, other): + if type(other) != type(self): + return False +@@ -3869,6 +3877,11 @@ functionList = [ + _CXString, + _CXString.from_result), + ++ ("clang_getFullyQualifiedTypeName", ++ [Type], ++ _CXString, ++ _CXString.from_result), ++ + ("clang_hashCursor", + [Cursor], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/tools/clang/bindings/python/tests/cindex/test_cursor.py +index ef875e97247..6a53c7205df 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -316,6 +316,14 @@ class TestCursor(unittest.TestCase): + underlying = typedef.underlying_typedef_type + self.assertEqual(underlying.kind, TypeKind.INT) + ++ def test_fully_qualified_type_name(): ++ source = 'namespace uiae { struct X { typedef int sometype; }; }' ++ tu = get_tu(source, lang='cpp') ++ ++ cls = get_cursor(tu, 'sometype') ++ fqn = cls.type.fully_qualified_name ++ self.assertTrue(fqn.endswith("uiae::X::sometype"), fqn) ++ + def test_semantic_parent(self): + tu = get_tu(kParentTest, 'cpp') + curs = get_cursors(tu, 'f') +diff --git a/tools/clang/include/clang-c/Index.h b/tools/clang/include/clang-c/Index.h +index 74badac740b..b0c62fe948e 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 59 ++#define CINDEX_VERSION_MINOR 60 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -3389,6 +3389,14 @@ CINDEX_LINKAGE CXType clang_getCursorType(CXCursor C); + */ + CINDEX_LINKAGE CXString clang_getTypeSpelling(CXType CT); + ++/** ++ * Retrieve the fully qualified name of the underlying type. ++ * This includes full qualification of all template parameters etc. ++ * ++ * If the type is invalid, an empty string is returned. ++ */ ++CINDEX_LINKAGE CXString clang_getFullyQualifiedTypeName(CXType CT); ++ + /** + * Retrieve the underlying type of a typedef declaration. + * +diff --git a/tools/clang/tools/libclang/CMakeLists.txt b/tools/clang/tools/libclang/CMakeLists.txt +index 613ead1a36b..a583fa206d1 100644 +--- a/tools/clang/tools/libclang/CMakeLists.txt ++++ b/tools/clang/tools/libclang/CMakeLists.txt +@@ -43,6 +43,7 @@ set(LIBS + clangSema + clangSerialization + clangTooling ++ clangToolingCore + ) + + if (CLANG_ENABLE_ARCMT) +diff --git a/tools/clang/tools/libclang/CXType.cpp b/tools/clang/tools/libclang/CXType.cpp +index acecf87d0cd..afdeb467769 100644 +--- a/tools/clang/tools/libclang/CXType.cpp ++++ b/tools/clang/tools/libclang/CXType.cpp +@@ -19,6 +19,7 @@ + #include "clang/AST/DeclObjC.h" + #include "clang/AST/DeclTemplate.h" + #include "clang/AST/Expr.h" ++#include "clang/AST/QualTypeNames.h" + #include "clang/AST/Type.h" + #include "clang/Basic/AddressSpaces.h" + #include "clang/Frontend/ASTUnit.h" +@@ -302,6 +303,27 @@ CXString clang_getTypeSpelling(CXType CT) { + return cxstring::createDup(OS.str()); + } + ++CXString clang_getFullyQualifiedTypeName(CXType CT) { ++ QualType T = GetQualType(CT); ++ if (T.isNull()) ++ return cxstring::createEmpty(); ++ ++ // For builtin types (but not typedefs pointing to builtin types) return their ++ // spelling. Otherwise "bool" will be turned into "_Bool". ++ const Type *TP = T.getTypePtrOrNull(); ++ if (TP && TP->isBuiltinType() && T->getAs<TypedefType>() == nullptr) ++ return clang_getTypeSpelling(CT); ++ ++ CXTranslationUnit TU = GetTU(CT); ++ ASTContext &Ctx = cxtu::getASTUnit(TU)->getASTContext(); ++ PrintingPolicy Policy(Ctx.getPrintingPolicy()); ++ Policy.SuppressScope = false; ++ Policy.AnonymousTagLocations = false; ++ Policy.PolishForDeclaration = true; ++ std::string name = TypeName::getFullyQualifiedName(T, Ctx, Policy, /*WithGlobalNsPrefix=*/true); ++ return cxstring::createDup(name.c_str()); ++} ++ + CXType clang_getTypedefDeclUnderlyingType(CXCursor C) { + using namespace cxcursor; + CXTranslationUnit TU = cxcursor::getCursorTU(C); +diff --git a/tools/clang/tools/libclang/libclang.exports b/tools/clang/tools/libclang/libclang.exports +index 3c76090d64f..6e860e7263e 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -241,6 +241,7 @@ clang_getFileLocation + clang_getFileName + clang_getFileTime + clang_getFileUniqueID ++clang_getFullyQualifiedTypeName + clang_getFunctionTypeCallingConv + clang_getIBOutletCollectionType + clang_getIncludedFile +-- +2.23.0 + diff --git a/packages/llvm/llvm9-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch b/packages/llvm/llvm9-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch new file mode 100644 index 0000000000000000000000000000000000000000..e3a0012bd24da636ed172ae308242c2cadb547ae --- /dev/null +++ b/packages/llvm/llvm9-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch @@ -0,0 +1,261 @@ +From 075a7a3e667fe3d923de6d7a6929e61922c8b139 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann@jklaehn.de> +Date: Fri, 3 Nov 2017 21:19:51 +0100 +Subject: [PATCH 3/5] [libclang] Add option to keep whitespace when tokenizing + +Introduces new `clang_tokenizeRange` function which accepts options to control +tokenization behavior. `clang_tokenize` is kept for backwards compatibility. +--- + clang/bindings/python/clang/cindex.py | 31 ++++++++++++++---- + .../python/tests/cindex/test_cursor.py | 9 ++++++ + clang/include/clang-c/Index.h | 32 +++++++++++++++++-- + clang/tools/libclang/CIndex.cpp | 15 +++++++-- + clang/tools/libclang/libclang.exports | 1 + + 5 files changed, 75 insertions(+), 13 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index c309f7017b2..1589acc9e7e 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -529,6 +529,13 @@ class TokenGroup(object): + + You should not instantiate this class outside of this module. + """ ++ ++ # Default tokenization mode. ++ TOKENIZE_NONE = 0 ++ ++ # Used to indicate that tokens for whitespace should be returned. ++ TOKENIZE_KEEP_WHITESPACE = 1 ++ + def __init__(self, tu, memory, count): + self._tu = tu + self._memory = memory +@@ -538,7 +545,7 @@ class TokenGroup(object): + conf.lib.clang_disposeTokens(self._tu, self._memory, self._count) + + @staticmethod +- def get_tokens(tu, extent): ++ def get_tokens(tu, extent, options=0): + """Helper method to return all tokens in an extent. + + This functionality is needed multiple places in this module. We define +@@ -547,8 +554,8 @@ class TokenGroup(object): + tokens_memory = POINTER(Token)() + tokens_count = c_uint() + +- conf.lib.clang_tokenize(tu, extent, byref(tokens_memory), +- byref(tokens_count)) ++ conf.lib.clang_tokenizeRange( ++ tu, extent, byref(tokens_memory), byref(tokens_count), options) + + count = int(tokens_count.value) + +@@ -1852,13 +1859,16 @@ class Cursor(Structure): + for descendant in child.walk_preorder(): + yield descendant + +- def get_tokens(self): ++ def get_tokens(self, options=0): + """Obtain Token instances formulating that compose this Cursor. + + This is a generator for Token instances. It returns all tokens which + occupy the extent this cursor occupies. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ +- return TokenGroup.get_tokens(self._tu, self.extent) ++ return TokenGroup.get_tokens(self._tu, self.extent, options) + + def get_field_offsetof(self): + """Returns the offsetof the FIELD_DECL pointed by this Cursor.""" +@@ -3080,18 +3090,21 @@ class TranslationUnit(ClangObject): + return CodeCompletionResults(ptr) + return None + +- def get_tokens(self, locations=None, extent=None): ++ def get_tokens(self, locations=None, extent=None, options=0): + """Obtain tokens in this translation unit. + + This is a generator for Token instances. The caller specifies a range + of source code to obtain tokens for. The range can be specified as a + 2-tuple of SourceLocation or as a SourceRange. If both are defined, + behavior is undefined. ++ ++ options is a bitwise or of TokenGroup.TOKENIZE_XXX flags which will ++ control tokenization behavior. + """ + if locations is not None: + extent = SourceRange(start=locations[0], end=locations[1]) + +- return TokenGroup.get_tokens(self, extent) ++ return TokenGroup.get_tokens(self, extent, options) + + class File(ClangObject): + """ +@@ -3969,6 +3982,10 @@ functionList = [ + ("clang_tokenize", + [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint)]), + ++ ("clang_tokenizeRange", ++ [TranslationUnit, SourceRange, POINTER(POINTER(Token)), POINTER(c_uint), ++ c_uint]), ++ + ("clang_visitChildren", + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 6a53c7205df..0965c1f4ae1 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -10,6 +10,7 @@ import unittest + from clang.cindex import AvailabilityKind + from clang.cindex import CursorKind + from clang.cindex import TemplateArgumentKind ++from clang.cindex import TokenGroup + from clang.cindex import TranslationUnit + from clang.cindex import TypeKind + from .util import get_cursor +@@ -488,6 +489,14 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tokens[0].spelling, 'int') + self.assertEqual(tokens[1].spelling, 'foo') + ++ def test_get_tokens_with_whitespace(): ++ source = 'class C { void f(); }\nvoid C::f() { }' ++ tu = get_tu(source) ++ ++ tokens = list(tu.cursor.get_tokens(TokenGroup.TOKENIZE_KEEP_WHITESPACE)) ++ self.assertEqual(''.join(t.spelling for t in tokens), source) ++ self.assertEqual(len(tokens), 27, [t.spelling for t in tokens]) ++ + def test_get_token_cursor(self): + """Ensure we can map tokens to cursors.""" + tu = get_tu('class A {}; int foo(A var = A());', lang='cpp') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index b0c62fe948e..84ed03b8920 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 60 ++#define CINDEX_VERSION_MINOR 61 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -4969,6 +4969,28 @@ CINDEX_LINKAGE CXSourceLocation clang_getTokenLocation(CXTranslationUnit, + */ + CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + ++typedef enum { ++ /** ++ * \brief Used to indicate that no special tokenization options are needed. ++ */ ++ CXTokenize_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that tokens for whitespace should be returned. ++ */ ++ CXTokenize_KeepWhitespace = 0x1 ++} CXTokenize_Flags; ++ ++/** ++ * \brief Tokenize the source code described by the given range into raw ++ * lexical tokens. ++ * ++ * \see clang_tokenizeRange ++ * ++ */ ++CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens); ++ + /** + * Tokenize the source code described by the given range into raw + * lexical tokens. +@@ -4985,9 +5007,13 @@ CINDEX_LINKAGE CXSourceRange clang_getTokenExtent(CXTranslationUnit, CXToken); + * \param NumTokens will be set to the number of tokens in the \c *Tokens + * array. + * ++ * \param options A bitmask of options that affects tokenization. This should be ++ * a bitwise OR of the CXTokenize_XXX flags. ++ * + */ +-CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, +- CXToken **Tokens, unsigned *NumTokens); ++CINDEX_LINKAGE void clang_tokenizeRange(CXTranslationUnit TU, ++ CXSourceRange Range, CXToken **Tokens, ++ unsigned *NumTokens, unsigned options); + + /** + * Annotate the given set of tokens by providing cursors for each token +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 1dc961f58a2..3a283e76ed8 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -6670,7 +6670,7 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) { + } + + static void getTokens(ASTUnit *CXXUnit, SourceRange Range, +- SmallVectorImpl<CXToken> &CXTokens) { ++ SmallVectorImpl<CXToken> &CXTokens, unsigned options) { + SourceManager &SourceMgr = CXXUnit->getSourceManager(); + std::pair<FileID, unsigned> BeginLocInfo + = SourceMgr.getDecomposedSpellingLoc(Range.getBegin()); +@@ -6692,6 +6692,9 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range, + CXXUnit->getASTContext().getLangOpts(), + Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end()); + Lex.SetCommentRetentionState(true); ++ if (options & CXTokenize_KeepWhitespace) { ++ Lex.SetKeepWhitespaceMode(true); ++ } + + // Lex tokens until we hit the end of the range. + const char *EffectiveBufferEnd = Buffer.data() + EndLocInfo.second; +@@ -6765,7 +6768,7 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + SourceLocation End = SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second); + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, SourceRange(Begin, End), CXTokens); ++ getTokens(CXXUnit, SourceRange(Begin, End), CXTokens, CXTokenize_None); + + if (CXTokens.empty()) + return NULL; +@@ -6779,6 +6782,12 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) { + + void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + CXToken **Tokens, unsigned *NumTokens) { ++ return clang_tokenizeRange(TU, Range, Tokens, NumTokens, CXTokenize_None); ++} ++ ++void clang_tokenizeRange(CXTranslationUnit TU, CXSourceRange Range, ++ CXToken **Tokens, unsigned *NumTokens, ++ unsigned options) { + LOG_FUNC_SECTION { + *Log << TU << ' ' << Range; + } +@@ -6804,7 +6813,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, + return; + + SmallVector<CXToken, 32> CXTokens; +- getTokens(CXXUnit, R, CXTokens); ++ getTokens(CXXUnit, R, CXTokens, options); + + if (CXTokens.empty()) + return; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6e860e7263e..6af6c0ca3e8 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -338,6 +338,7 @@ clang_suspendTranslationUnit + clang_sortCodeCompletionResults + clang_toggleCrashRecovery + clang_tokenize ++clang_tokenizeRange + clang_CompilationDatabase_fromDirectory + clang_CompilationDatabase_dispose + clang_CompilationDatabase_getCompileCommands +-- +2.23.0 + diff --git a/packages/llvm/llvm9-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch b/packages/llvm/llvm9-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch new file mode 100644 index 0000000000000000000000000000000000000000..ef4a125e82d6b42b73574ce10ced78aa2d597590 --- /dev/null +++ b/packages/llvm/llvm9-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch @@ -0,0 +1,442 @@ +From d10d66b8e762c1dd1329d5920f9ef952cf4f6940 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 17 Jul 2017 12:25:49 +0200 +Subject: [PATCH 4/5] [libclang] WIP: Allow visiting of implicit declarations + and template instantiations + +--- + clang/bindings/python/clang/cindex.py | 45 +++++-- + .../python/tests/cindex/test_cursor.py | 33 ++++++ + clang/include/clang-c/Index.h | 33 +++++- + clang/tools/libclang/CIndex.cpp | 112 ++++++++++++++++-- + clang/tools/libclang/CursorVisitor.h | 12 +- + clang/tools/libclang/libclang.exports | 2 + + 6 files changed, 220 insertions(+), 17 deletions(-) + +diff --git a/tools/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py +index 1589acc9e7e..b023be6cdc8 100644 +--- a/tools/clang/bindings/python/clang/cindex.py ++++ b/tools/clang/bindings/python/clang/cindex.py +@@ -1426,6 +1426,15 @@ class Cursor(Structure): + """ + _fields_ = [("_kind_id", c_int), ("xdata", c_int), ("data", c_void_p * 3)] + ++ # Default behavior. ++ GET_CHILDREN_NONE = 0 ++ ++ # Used to indicate that implicit cursors should be visited. ++ GET_CHILDREN_WITH_IMPLICIT = 1 ++ ++ # Used to indicate that template instantiations should be visited. ++ GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS = 2 ++ + @staticmethod + def from_location(tu, location): + # We store a reference to the TU in the instance so the TU won't get +@@ -1515,6 +1524,10 @@ class Cursor(Structure): + """ + return conf.lib.clang_EnumDecl_isScoped(self) + ++ def is_implicit(self): ++ """Test whether the cursor refers to an implicit declaration.""" ++ return conf.lib.clang_isImplicit(self) ++ + def get_definition(self): + """ + If the cursor is a reference to a declaration or a declaration of +@@ -1831,8 +1844,12 @@ class Cursor(Structure): + """Returns the value of the indicated arg as an unsigned 64b integer.""" + return conf.lib.clang_Cursor_getTemplateArgumentUnsignedValue(self, num) + +- def get_children(self): +- """Return an iterator for accessing the children of this cursor.""" ++ def get_children(self, with_implicit=False, with_template_instantiations=False): ++ """Return an iterator for accessing the children of this cursor. ++ ++ By default, cursors representing implicit declarations or template instantiations ++ will be skipped. ++ """ + + # FIXME: Expose iteration from CIndex, PR6125. + def visitor(child, parent, children): +@@ -1845,18 +1862,24 @@ class Cursor(Structure): + children.append(child) + return 1 # continue + children = [] +- conf.lib.clang_visitChildren(self, callbacks['cursor_visit'](visitor), +- children) ++ dispatch = conf.lib.clang_visitChildren ++ options = Cursor.GET_CHILDREN_NONE ++ if with_implicit: ++ options |= Cursor.GET_CHILDREN_WITH_IMPLICIT ++ if with_template_instantiations: ++ options |= Cursor.GET_CHILDREN_WITH_TEMPLATE_INSTANTIATIONS ++ conf.lib.clang_visitChildrenWithOptions( ++ self, callbacks['cursor_visit'](visitor), children, options) + return iter(children) + +- def walk_preorder(self): ++ def walk_preorder(self, **kwargs): + """Depth-first preorder walk over the cursor and its descendants. + + Yields cursors. + """ + yield self +- for child in self.get_children(): +- for descendant in child.walk_preorder(): ++ for child in self.get_children(**kwargs): ++ for descendant in child.walk_preorder(**kwargs): + yield descendant + + def get_tokens(self, options=0): +@@ -3927,6 +3950,10 @@ functionList = [ + [Type], + bool), + ++ ("clang_isImplicit", ++ [Cursor], ++ bool), ++ + ("clang_isInvalid", + [CursorKind], + bool), +@@ -3990,6 +4017,10 @@ functionList = [ + [Cursor, callbacks['cursor_visit'], py_object], + c_uint), + ++ ("clang_visitChildrenWithOptions", ++ [Cursor, callbacks['cursor_visit'], py_object, c_uint], ++ c_uint), ++ + ("clang_Cursor_getNumArguments", + [Cursor], + c_int), +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index 0965c1f4ae1..d061f37c25c 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -94,6 +94,39 @@ class TestCursor(unittest.TestCase): + self.assertEqual(tu_nodes[2].displayname, 'f0(int, int)') + self.assertEqual(tu_nodes[2].is_definition(), True) + ++ def test_get_children_with_implicit(): ++ tu = get_tu('struct X {}; X x;', lang='cpp') ++ cursor = get_cursor(tu, 'X') ++ ++ children = list(cursor.get_children()) ++ self.assertEqual(len(children), 0, [(c.kind, c.spelling) for c in children]) ++ ++ children = list(cursor.get_children(with_implicit=True)) ++ self.assertNotEqual(len(children), 0) ++ for child in children: ++ self.assertTrue(child.is_implicit()) ++ self.assertEqual(child.spelling, "X") ++ self.assertIn(child.kind, [CursorKind.CONSTRUCTOR, CursorKind.STRUCT_DECL]) ++ ++ def test_get_children_with_template_instantiations(): ++ tu = get_tu( ++ 'template <typename T> T frobnicate(T val);' ++ 'extern template int frobnicate<int>(int);', ++ lang='cpp') ++ cursor = get_cursor(tu, 'frobnicate') ++ self.assertEqual(cursor.kind, CursorKind.FUNCTION_TEMPLATE) ++ ++ for child in cursor.get_children(): ++ # should not return an instantiation: ++ self.assertNotEqual(child.kind, CursorKind.FUNCTION_DECL) ++ ++ for child in cursor.get_children(with_template_instantiations=True): ++ if child.kind == CursorKind.FUNCTION_DECL: ++ self.assertEqual(child.spelling, 'frobnicate') ++ break ++ else: ++ self.fail("Couldn't find template instantiation") ++ + def test_references(self): + """Ensure that references to TranslationUnit are kept.""" + tu = get_tu('int x;') +diff --git a/tools/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h +index 84ed03b8920..57acbcba143 100644 +--- a/tools/clang/include/clang-c/Index.h ++++ b/tools/clang/include/clang-c/Index.h +@@ -32,7 +32,7 @@ + * compatible, thus CINDEX_VERSION_MAJOR is expected to remain stable. + */ + #define CINDEX_VERSION_MAJOR 0 +-#define CINDEX_VERSION_MINOR 61 ++#define CINDEX_VERSION_MINOR 62 + + #define CINDEX_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ +@@ -2775,6 +2775,11 @@ CINDEX_LINKAGE unsigned clang_isPreprocessing(enum CXCursorKind); + */ + CINDEX_LINKAGE unsigned clang_isUnexposed(enum CXCursorKind); + ++/*** ++ * \brief Determine whether the given cursor represents an implicit declaration. ++ */ ++CINDEX_LINKAGE unsigned clang_isImplicit(CXCursor); ++ + /** + * Describe the linkage of the entity referred to by a cursor. + */ +@@ -4199,6 +4204,32 @@ CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent, + # endif + #endif + ++typedef enum { ++ /** ++ * \brief Default behavior. ++ */ ++ CXVisitChildren_None = 0x0, ++ ++ /** ++ * \brief Used to indicate that implicit cursors should be visited. ++ */ ++ CXVisitChildren_WithImplicit = 0x1, ++ ++ /** ++ * \brief Used to indicate that template instantiations should be visited. ++ */ ++ CXVisitChildren_WithTemplateInstantiations = 0x2 ++} CXVisitChildren_Flags; ++ ++/** ++ * \brief Visits the children of a cursor, allowing to pass extra options. ++ * Behaves identically to clang_visitChildren() in all other respects. ++ */ ++CINDEX_LINKAGE unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options); ++ + /** + * @} + */ +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 3a283e76ed8..7ee6b704647 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -195,10 +195,11 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) { + assert(0 && "Invalid declaration cursor"); + return true; // abort. + } +- +- // Ignore implicit declarations, unless it's an objc method because +- // currently we should report implicit methods for properties when indexing. +- if (D->isImplicit() && !isa<ObjCMethodDecl>(D)) ++ ++ // Unless instructed otherwise we ignore implicit declarations. ++ // ObjC methods are currently visited in any case, because implicit methods ++ // for properties should be reported when indexing. ++ if (!VisitImplicitDeclarations && D->isImplicit() && !isa<ObjCMethodDecl>(D)) + return false; + } + +@@ -703,10 +704,13 @@ bool CursorVisitor::VisitTagDecl(TagDecl *D) { + + bool CursorVisitor::VisitClassTemplateSpecializationDecl( + ClassTemplateSpecializationDecl *D) { +- bool ShouldVisitBody = false; ++ bool ShouldVisitBody = VisitTemplateInstantiations; + switch (D->getSpecializationKind()) { +- case TSK_Undeclared: + case TSK_ImplicitInstantiation: ++ if (VisitTemplateInstantiations && VisitImplicitDeclarations) { ++ break; ++ } ++ case TSK_Undeclared: + // Nothing to visit + return false; + +@@ -715,6 +719,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl( + break; + + case TSK_ExplicitSpecialization: ++ // Always visit body of explicit specializations + ShouldVisitBody = true; + break; + } +@@ -930,7 +935,31 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) { + return true; + + auto* FD = D->getTemplatedDecl(); +- return VisitAttributes(FD) || VisitFunctionDecl(FD); ++ if (VisitAttributes(FD) || VisitFunctionDecl(FD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *FD : D->specializations()) { ++ for (auto *RD : FD->redecls()) { ++ switch (RD->getTemplateSpecializationKind()) { ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { +@@ -940,7 +969,40 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) { + return true; + + auto* CD = D->getTemplatedDecl(); +- return VisitAttributes(CD) || VisitCXXRecordDecl(CD); ++ if (VisitAttributes(CD) || VisitCXXRecordDecl(CD)) ++ return true; ++ ++ if (VisitTemplateInstantiations && D == D->getCanonicalDecl()) { ++ for (auto *SD : D->specializations()) { ++ for (auto *RD : SD->redecls()) { ++ // We don't want to visit injected-class-names in this traversal. ++ if (cast<CXXRecordDecl>(RD)->isInjectedClassName()) ++ continue; ++ ++ switch ( ++ cast<ClassTemplateSpecializationDecl>(RD)->getSpecializationKind()) { ++ // Visit the implicit instantiations with the requested pattern. ++ case TSK_Undeclared: ++ case TSK_ImplicitInstantiation: { ++ const Optional<bool> V = handleDeclForVisitation(RD); ++ if (!V.hasValue()) ++ continue; ++ return V.getValue(); ++ } ++ ++ // We don't need to do anything on an explicit instantiation ++ // or explicit specialization because there will be an explicit ++ // node for it elsewhere. ++ case TSK_ExplicitInstantiationDeclaration: ++ case TSK_ExplicitInstantiationDefinition: ++ case TSK_ExplicitSpecialization: ++ break; ++ } ++ } ++ } ++ } ++ ++ return false; + } + + bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) { +@@ -4426,6 +4488,24 @@ unsigned clang_visitChildrenWithBlock(CXCursor parent, + return clang_visitChildren(parent, visitWithBlock, block); + } + ++unsigned clang_visitChildrenWithOptions(CXCursor parent, ++ CXCursorVisitor visitor, ++ CXClientData client_data, ++ unsigned options) { ++ CursorVisitor CursorVis( ++ getCursorTU(parent), visitor, client_data, ++ /*VisitPreprocessorLast=*/false, ++ /*VisitIncludedPreprocessingEntries=*/false, ++ /*RegionOfInterest=*/SourceRange(), ++ /*VisitDeclsOnly=*/false, ++ /*PostChildrenVisitor=*/nullptr, ++ /*VisitImplicitDeclarations=*/(options & CXVisitChildren_WithImplicit), ++ /*VisitTemplateInstantiations=*/ ++ (options & CXVisitChildren_WithTemplateInstantiations)); ++ ++ return CursorVis.VisitChildren(parent); ++} ++ + static CXString getDeclSpelling(const Decl *D) { + if (!D) + return cxstring::createEmpty(); +@@ -5774,6 +5854,22 @@ unsigned clang_isUnexposed(enum CXCursorKind K) { + } + } + ++unsigned clang_isImplicit(CXCursor Cursor) { ++ if (clang_isInvalid(Cursor.kind)) ++ return false; ++ ++ if (!clang_isDeclaration(Cursor.kind)) ++ return false; ++ ++ const Decl *D = getCursorDecl(Cursor); ++ if (!D) { ++ assert(0 && "Invalid declaration cursor"); ++ return true; // abort. ++ } ++ ++ return D->isImplicit(); ++} ++ + CXCursorKind clang_getCursorKind(CXCursor C) { + return C.kind; + } +diff --git a/tools/clang/tools/libclang/CursorVisitor.h b/clang/tools/libclang/CursorVisitor.h +index b0afa5a0b59..94f4596d5fa 100644 +--- a/tools/clang/tools/libclang/CursorVisitor.h ++++ b/tools/clang/tools/libclang/CursorVisitor.h +@@ -95,6 +95,12 @@ private: + /// record entries. + bool VisitDeclsOnly; + ++ /// \brief Whether we should visit implicit declarations. ++ bool VisitImplicitDeclarations; ++ ++ /// \brief Whether we should recurse into template instantiations. ++ bool VisitTemplateInstantiations; ++ + // FIXME: Eventually remove. This part of a hack to support proper + // iteration over all Decls contained lexically within an ObjC container. + DeclContext::decl_iterator *DI_current; +@@ -146,7 +152,9 @@ public: + bool VisitIncludedPreprocessingEntries = false, + SourceRange RegionOfInterest = SourceRange(), + bool VisitDeclsOnly = false, +- PostChildrenVisitorTy PostChildrenVisitor = nullptr) ++ PostChildrenVisitorTy PostChildrenVisitor = nullptr, ++ bool VisitImplicitDeclarations = false, ++ bool VisitTemplateInstantiations = false) + : TU(TU), AU(cxtu::getASTUnit(TU)), + Visitor(Visitor), PostChildrenVisitor(PostChildrenVisitor), + ClientData(ClientData), +@@ -154,6 +162,8 @@ public: + VisitIncludedEntities(VisitIncludedPreprocessingEntries), + RegionOfInterest(RegionOfInterest), + VisitDeclsOnly(VisitDeclsOnly), ++ VisitImplicitDeclarations(VisitImplicitDeclarations), ++ VisitTemplateInstantiations(VisitTemplateInstantiations), + DI_current(nullptr), FileDI_current(nullptr) + { + Parent.kind = CXCursor_NoDeclFound; +diff --git a/tools/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports +index 6af6c0ca3e8..d17eb83187d 100644 +--- a/tools/clang/tools/libclang/libclang.exports ++++ b/tools/clang/tools/libclang/libclang.exports +@@ -313,6 +313,7 @@ clang_isInvalidDeclaration + clang_isExpression + clang_isFileMultipleIncludeGuarded + clang_isFunctionTypeVariadic ++clang_isImplicit + clang_isInvalid + clang_isPODType + clang_isPreprocessing +@@ -354,6 +355,7 @@ clang_CompileCommand_getNumArgs + clang_CompileCommand_getArg + clang_visitChildren + clang_visitChildrenWithBlock ++clang_visitChildrenWithOptions + clang_ModuleMapDescriptor_create + clang_ModuleMapDescriptor_dispose + clang_ModuleMapDescriptor_setFrameworkModuleName +-- +2.23.0 + diff --git a/packages/llvm/llvm9-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch b/packages/llvm/llvm9-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch new file mode 100644 index 0000000000000000000000000000000000000000..68c83bce65c3253dd3653c102f417df67bd51300 --- /dev/null +++ b/packages/llvm/llvm9-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch @@ -0,0 +1,52 @@ +From 0fa84bbd8c6f8eb8eb6a3bc30e4efe3f2cf4c283 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Johann=20Kl=C3=A4hn?= <johann.klaehn@kip.uni-heidelberg.de> +Date: Mon, 31 Jul 2017 14:09:52 +0200 +Subject: [PATCH 5/5] [libclang] WIP: Fix get_tokens in macro expansion + +--- + clang/bindings/python/tests/cindex/test_cursor.py | 15 +++++++++++++++ + clang/tools/libclang/CIndex.cpp | 2 +- + 2 files changed, 16 insertions(+), 1 deletion(-) + +diff --git a/tools/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py +index d061f37c25c..38702df74f2 100644 +--- a/tools/clang/bindings/python/tests/cindex/test_cursor.py ++++ b/tools/clang/bindings/python/tests/cindex/test_cursor.py +@@ -552,6 +552,21 @@ class TestCursor(unittest.TestCase): + r_cursor = t_cursor.referenced # should not raise an exception + self.assertEqual(r_cursor.kind, CursorKind.CLASS_DECL) + ++ def test_get_tokens_in_macro_expansion(self): ++ """regression test""" ++ source = "#define IMPL(name) struct name { name(int v = 123); }; \n IMPL(X)" ++ tu = get_tu(source, lang="cpp") ++ ctor = get_cursors(tu, "X")[1] ++ self.assertEqual(ctor.kind, CursorKind.CONSTRUCTOR) ++ p = next(ctor.get_children()) ++ self.assertEqual(p.kind, CursorKind.PARM_DECL, (p.kind, p.spelling)) ++ children = list(p.get_children()) ++ self.assertEqual(len(children), 1, [(c.kind, c.spelling) for c in children]) ++ expr = children[0] ++ tokens = list(expr.get_tokens()) ++ self.assertEqual(len(tokens), 1, [t.spelling for t in tokens]) ++ self.assertEqual(tokens[0].spelling, "123") ++ + def test_get_arguments(self): + tu = get_tu('void foo(int i, int j);') + foo = get_cursor(tu, 'foo') +diff --git a/tools/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp +index 7ee6b704647..e4b95436704 100644 +--- a/tools/clang/tools/libclang/CIndex.cpp ++++ b/tools/clang/tools/libclang/CIndex.cpp +@@ -145,7 +145,7 @@ CXSourceRange cxloc::translateSourceRange(const SourceManager &SM, + // location accordingly. + SourceLocation EndLoc = R.getEnd(); + bool IsTokenRange = R.isTokenRange(); +- if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) { ++ if (false /*FIXME*/ && EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) { + CharSourceRange Expansion = SM.getExpansionRange(EndLoc); + EndLoc = Expansion.getEnd(); + IsTokenRange = Expansion.isTokenRange(); +-- +2.23.0 + diff --git a/packages/llvm/llvm_gcc7.patch b/packages/llvm/llvm_gcc7.patch new file mode 100644 index 0000000000000000000000000000000000000000..c6d9d33a08aedffa2d3077f0384d1bec6547623e --- /dev/null +++ b/packages/llvm/llvm_gcc7.patch @@ -0,0 +1,10 @@ +--- a/lldb/include/lldb/Utility/TaskPool.h 2016-09-06 16:57:50.000000000 -0400 ++++ b/lldb/include/lldb/Utility/TaskPool.h 2017-08-29 16:29:41.448584015 -0400 +@@ -28,6 +28,7 @@ + + #include <cassert> + #include <cstdint> ++#include <functional> + #include <future> + #include <list> + #include <queue> diff --git a/packages/llvm/llvm_py37.patch b/packages/llvm/llvm_py37.patch new file mode 100644 index 0000000000000000000000000000000000000000..478be87994b9363a05c5b876780cd6379017e5d6 --- /dev/null +++ b/packages/llvm/llvm_py37.patch @@ -0,0 +1,37 @@ +From ecdefed7f6ba11421fe1ecc6c13a135ab7bcda73 Mon Sep 17 00:00:00 2001 +From: Pavel Labath <labath@google.com> +Date: Mon, 23 Jul 2018 11:37:36 +0100 +Subject: [PATCH] Fix PythonString::GetString for >=python-3.7 + +The return value of PyUnicode_AsUTF8AndSize is now "const char *". +--- + .../Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/tools/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/tools/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp +index 6a9d57d5a..94f16b2c7 100644 +--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp ++++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp +@@ -404,14 +404,16 @@ llvm::StringRef PythonString::GetString() const { + return llvm::StringRef(); + + Py_ssize_t size; +- char *c; ++ const char *data; + + #if PY_MAJOR_VERSION >= 3 +- c = PyUnicode_AsUTF8AndSize(m_py_obj, &size); ++ data = PyUnicode_AsUTF8AndSize(m_py_obj, &size); + #else ++ char *c; + PyString_AsStringAndSize(m_py_obj, &c, &size); ++ data = c; + #endif +- return llvm::StringRef(c, size); ++ return llvm::StringRef(data, size); + } + + size_t PythonString::GetSize() const { +-- +2.18.0.233.g985f88cf7e-goog + diff --git a/packages/llvm/llvm_python_path.patch b/packages/llvm/llvm_python_path.patch new file mode 100644 index 0000000000000000000000000000000000000000..9f821cc3654ac5dd8e0f2412ffd1a3209d844b5a --- /dev/null +++ b/packages/llvm/llvm_python_path.patch @@ -0,0 +1,14 @@ +diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake +index dab55707338..6f4c6791141 100644 +--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake ++++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake +@@ -612,6 +612,9 @@ macro(add_custom_libcxx name prefix) + CMAKE_OBJDUMP + CMAKE_STRIP + CMAKE_SYSROOT ++ PYTHON_EXECUTABLE ++ Python3_EXECUTABLE ++ Python2_EXECUTABLE + CMAKE_SYSTEM_NAME) + foreach(variable ${PASSTHROUGH_VARIABLES}) + get_property(is_value_set CACHE ${variable} PROPERTY VALUE SET) diff --git a/packages/llvm/package.py b/packages/llvm/package.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f1d6ba2e227605ef5c55f073d9bc8e1e54395f --- /dev/null +++ b/packages/llvm/package.py @@ -0,0 +1,699 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path +import re +import sys + +import llnl.util.tty as tty + +import spack.util.executable + + +class Llvm(CMakePackage, CudaPackage): + """The LLVM Project is a collection of modular and reusable compiler and + toolchain technologies. Despite its name, LLVM has little to do + with traditional virtual machines, though it does provide helpful + libraries that can be used to build them. The name "LLVM" itself + is not an acronym; it is the full name of the project. + """ + + homepage = "http://llvm.org/" + url = "https://github.com/llvm/llvm-project/archive/llvmorg-7.1.0.tar.gz" + list_url = "http://releases.llvm.org/download.html" + git = "https://github.com/llvm/llvm-project" + maintainers = ['trws', 'naromero77'] + + family = "compiler" # Used by lmod + + # fmt: off + version('main', branch='main') + version('12.0.1', sha256='66b64aa301244975a4aea489f402f205cde2f53dd722dad9e7b77a0459b4c8df') + version('12.0.0', sha256='8e6c99e482bb16a450165176c2d881804976a2d770e0445af4375e78a1fbf19c') + version('11.1.0', sha256='53a0719f3f4b0388013cfffd7b10c7d5682eece1929a9553c722348d1f866e79') + version('11.0.1', sha256='9c7ad8e8ec77c5bde8eb4afa105a318fd1ded7dff3747d14f012758719d7171b') + version('11.0.0', sha256='8ad4ddbafac4f2c8f2ea523c2c4196f940e8e16f9e635210537582a48622a5d5') + version('10.0.1', sha256='c7ccb735c37b4ec470f66a6c35fbae4f029c0f88038f6977180b1a8ddc255637') + version('10.0.0', sha256='b81c96d2f8f40dc61b14a167513d87c0d813aae0251e06e11ae8a4384ca15451') + version('9.0.1', sha256='be7b034641a5fda51ffca7f5d840b1a768737779f75f7c4fd18fe2d37820289a') + version('9.0.0', sha256='7807fac25330e24e9955ca46cd855dd34bbc9cc4fdba8322366206654d1036f2') + version('8.0.1', sha256='5b18f6111c7aee7c0933c355877d4abcfe6cb40c1a64178f28821849c725c841') + version('8.0.0', sha256='d81238b4a69e93e29f74ce56f8107cbfcf0c7d7b40510b7879e98cc031e25167') + version('7.1.0', sha256='71c93979f20e01f1a1cc839a247945f556fa5e63abf2084e8468b238080fd839') + version('7.0.1', sha256='f17a6cd401e8fd8f811fbfbb36dcb4f455f898c9d03af4044807ad005df9f3c0') + version('6.0.1', sha256='aefadceb231f4c195fe6d6cd3b1a010b269c8a22410f339b5a089c2e902aa177') + version('6.0.0', sha256='1946ec629c88d30122afa072d3c6a89cc5d5e4e2bb28dc63b2f9ebcc7917ee64') + version('5.0.2', sha256='fe87aa11558c08856739bfd9bd971263a28657663cb0c3a0af01b94f03b0b795') + version('5.0.1', sha256='84ca454abf262579814a2a2b846569f6e0cb3e16dc33ca3642b4f1dff6fbafd3') + version('5.0.0', sha256='1f1843315657a4371d8ca37f01265fa9aae17dbcf46d2d0a95c1fdb3c6a4bab6') + version('4.0.1', sha256='cd664fb3eec3208c08fb61189c00c9118c290b3be5adb3215a97b24255618be5') + version('4.0.0', sha256='28ca4b2fc434cb1f558e8865386c233c2a6134437249b8b3765ae745ffa56a34') + version('3.9.1', sha256='f5b6922a5c65f9232f83d89831191f2c3ccf4f41fdd8c63e6645bbf578c4ab92') + version('3.9.0', sha256='9c6563a72c8b5b79941c773937d997dd2b1b5b3f640136d02719ec19f35e0333') + version('3.8.1', sha256='69360f0648fde0dc3d3c4b339624613f3bc2a89c4858933bc3871a250ad02826') + version('3.8.0', sha256='b5cc5974cc2fd4e9e49e1bbd0700f872501a8678bd9694fa2b36c65c026df1d1') + version('3.7.1', sha256='d2cb0eb9b8eb21e07605bfe5e7a5c6c5f5f8c2efdac01ec1da6ffacaabe4195a') + version('3.7.0', sha256='dc00bc230be2006fb87b84f6fe4800ca28bc98e6692811a98195da53c9cb28c6') + version('3.6.2', sha256='f75d703a388ba01d607f9cf96180863a5e4a106827ade17b221d43e6db20778a') + version('3.5.1', sha256='5d739684170d5b2b304e4fb521532d5c8281492f71e1a8568187bfa38eb5909d') + # fmt: on + + # NOTE: The debug version of LLVM is an order of magnitude larger than + # the release version, and may take up 20-30 GB of space. If you want + # to save space, build with `build_type=Release`. + + variant( + "clang", + default=True, + description="Build the LLVM C/C++/Objective-C compiler frontend", + ) + variant( + "flang", + default=False, + description="Build the LLVM Fortran compiler frontend " + "(experimental - parser only, needs GCC)", + ) + variant( + "omp_debug", + default=False, + description="Include debugging code in OpenMP runtime libraries", + ) + variant("lldb", default=True, description="Build the LLVM debugger") + variant("lld", default=True, description="Build the LLVM linker") + variant("mlir", default=False, description="Build with MLIR support") + variant( + "internal_unwind", + default=True, + description="Build the libcxxabi libunwind", + ) + variant( + "polly", + default=True, + description="Build the LLVM polyhedral optimization plugin, " + "only builds for 3.7.0+", + ) + variant( + "libcxx", + default=True, + description="Build the LLVM C++ standard library", + ) + variant( + "compiler-rt", + default=True, + description="Build LLVM compiler runtime, including sanitizers", + ) + variant( + "gold", + default=(sys.platform != "darwin"), + description="Add support for LTO with the gold linker plugin", + ) + variant( + "split_dwarf", + default=False, + description="Build with split dwarf information", + ) + variant( + "shared_libs", + default=False, + description="Build all components as shared libraries, faster, " + "less memory to build, less stable", + ) + variant( + "llvm_dylib", + default=False, + description="Build LLVM shared library, containing all " + "components in a single shared library", + ) + variant( + "all_targets", + default=False, + description="Build all supported targets, default targets " + "<current arch>,NVPTX,AMDGPU,CppBackend", + ) + variant( + "build_type", + default="Release", + description="CMake build type", + values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"), + ) + variant( + "omp_tsan", + default=False, + description="Build with OpenMP capable thread sanitizer", + ) + variant('code_signing', default=False, + description="Enable code-signing on macOS") + variant("python", default=False, description="Install python bindings") + + extends("python", when="+python") + + variant('visionary', default=False, + description="Include patches necessary for visionary python " + "bindings generator") + variant('force_full_view', default=False, + description='Force linking of all files into view, including ' + 'known conflicts (e.g. libgomp).') + + patch('llvm5-0001-libclang-Add-support-for-checking-abstractness-of-re.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0002-libclang-Keep-track-of-TranslationUnit-instance-when.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0003-Fix-warnings-in-Tooling-QualTypeNamesTest.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0004-Defer-addition-of-keywords-to-identifier-table-when-.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0005-Tooling-Fully-qualify-template-parameters-of-nested-.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0006-libclang-Add-support-for-obtaining-fully-qualified-n.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0007-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0008-Fix-printing-policy-for-AST-context-loaded-from-file.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0009-libclang-Visit-attributes-for-function-and-class-tem.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0010-libclang-Add-support-for-querying-cursor-availabilit.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0011-libclang-Allow-visiting-of-implicit-declarations-and.patch', when='@5.0:6.999 +visionary', level=2) + patch('llvm5-0012-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch', when='@5.0:6.999 +visionary', level=2) + + patch('llvm7-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch', when='@7.0:7.999 +visionary', level=2) + patch('llvm7-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch', when='@7.0:7.999 +visionary', level=2) + patch('llvm7-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch', when='@7.0:7.999 +visionary', level=2) + patch('llvm7-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch', when='@7.0:7.999 +visionary', level=2) + patch('llvm7-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch', when='@7.0:7.999 +visionary', level=2) + + patch('llvm9-0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch', when='@9.0.0:12.0.999 +visionary', level=2) + patch('llvm9-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch', when='@9.0:10.999 +visionary', level=2) + patch('llvm9-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch', when='@9.0:10.999 +visionary', level=2) + patch('llvm9-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch', when='@9.0:10.999 +visionary', level=2) + patch('llvm9-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch', when='@9.0:10.999 +visionary', level=2) + + # 0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch from above + patch('llvm11-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch', when='@11.0.0:11.0.999 +visionary', level=2) + patch('llvm11-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch', when='@11.0.0:11.0.999 +visionary', level=2) + patch('llvm11-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch', when='@11.0.0:11.0.999 +visionary', level=2) + patch('llvm11-0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch', when='@11.0.0:12.0.999 +visionary', level=2) + + # 0001-Tooling-Fully-qualify-template-parameters-of-nested-.patch from above + patch('llvm11_1-0002-libclang-Add-support-for-obtaining-fully-qualified-n.patch', when='@11.1.0:12.0.999 +visionary', level=2) + patch('llvm11_1-0003-libclang-Add-option-to-keep-whitespace-when-tokenizi.patch', when='@11.1.0:12.0.999 +visionary', level=2) + patch('llvm11_1-0004-libclang-WIP-Allow-visiting-of-implicit-declarations.patch', when='@11.1.0:12.0.999 +visionary', level=2) + # 0005-libclang-WIP-Fix-get_tokens-in-macro-expansion.patch from above + + # Build dependency + depends_on("cmake@3.4.3:", type="build") + depends_on("python@2.7:2.8", when="@:4.999 ~python", type="build") + depends_on("python", when="@5: ~python", type="build") + depends_on("pkgconfig", type="build") + + # Universal dependency + depends_on("python@2.7:2.8", when="@:4.999+python") + depends_on("python", when="@5:+python") + depends_on("z3", when="@9:") + + # openmp dependencies + depends_on("perl-data-dumper", type=("build")) + depends_on("hwloc") + depends_on("libelf", when="+cuda") # libomptarget + depends_on("libffi", when="+cuda") # libomptarget + + # ncurses dependency + depends_on("ncurses+termlib") + + # lldb dependencies + depends_on("swig", when="+lldb") + depends_on("libedit", when="+lldb") + depends_on("py-six", when="@5.0.0: +lldb +python") + + # gold support, required for some features + depends_on("binutils+gold+ld+plugins", when="+gold") + + # polly plugin + depends_on("gmp", when="@:3.6.999 +polly") + depends_on("isl", when="@:3.6.999 +polly") + + conflicts("+llvm_dylib", when="+shared_libs") + conflicts("+lldb", when="~clang") + conflicts("+libcxx", when="~clang") + conflicts("+internal_unwind", when="~clang") + conflicts("+compiler-rt", when="~clang") + conflicts("+flang", when="~clang") + # Introduced in version 11 as a part of LLVM and not a separate package. + conflicts("+flang", when="@:10.999") + + # Older LLVM do not build with newer GCC + conflicts("%gcc@11:", when="@:7") + conflicts("%gcc@8:", when="@:5") + conflicts("%gcc@:5.0.999", when="@8:") + + # OMP TSAN exists in > 5.x + conflicts("+omp_tsan", when="@:5.99") + + # cuda_arch value must be specified + conflicts("cuda_arch=none", when="+cuda", msg="A value for cuda_arch must be specified.") + + # MLIR exists in > 10.x + conflicts("+mlir", when="@:9") + + # code signing is only necessary on macOS", + conflicts('+code_signing', when='platform=linux') + conflicts('+code_signing', when='platform=cray') + + conflicts( + '+code_signing', + when='~lldb platform=darwin', + msg="code signing is only necessary for building the " + "in-tree debug server on macOS. Turning this variant " + "off enables a build of llvm with lldb that uses the " + "system debug server", + ) + + # LLVM bug https://bugs.llvm.org/show_bug.cgi?id=48234 + # CMake bug: https://gitlab.kitware.com/cmake/cmake/-/issues/21469 + # Fixed in upstream versions of both + conflicts('^cmake@3.19.0', when='@6.0.0:11.0.0') + + # Github issue #4986 + patch("llvm_gcc7.patch", when="@4.0.0:4.0.1+lldb %gcc@7.0:") + + # https://github.com/spack/spack/issues/24270 + patch('https://src.fedoraproject.org/rpms/llvm10/raw/7ce7ebd066955ea95ba2b491c41fbc6e4ee0643a/f/llvm10-gcc11.patch', + sha256='958c64838c9d469be514eef195eca0f8c3ab069bc4b64a48fad59991c626bab8', + when='@8:10 %gcc@11:') + + # Backport from llvm master + additional fix + # see https://bugs.llvm.org/show_bug.cgi?id=39696 + # for a bug report about this problem in llvm master. + patch("constexpr_longdouble.patch", when="@6:8+libcxx") + patch("constexpr_longdouble_9.0.patch", when="@9:10.0.0+libcxx") + + # Backport from llvm master; see + # https://bugs.llvm.org/show_bug.cgi?id=38233 + # for a bug report about this problem in llvm master. + patch("llvm_py37.patch", when="@4:6 ^python@3.7:") + + # https://bugs.llvm.org/show_bug.cgi?id=39696 + patch("thread-p9.patch", when="@develop+libcxx") + + # https://github.com/spack/spack/issues/19625, + # merged in llvm-11.0.0_rc2, but not found in 11.0.1 + patch("lldb_external_ncurses-10.patch", when="@10.0.0:11.0.1+lldb") + + # https://github.com/spack/spack/issues/19908 + # merged in llvm main prior to 12.0.0 + patch("llvm_python_path.patch", when="@11.0.0") + + # Workaround for issue https://github.com/spack/spack/issues/18197 + patch('llvm7_intel.patch', when='@7 %intel@18.0.2,19.0.4') + + # The functions and attributes below implement external package + # detection for LLVM. See: + # + # https://spack.readthedocs.io/en/latest/packaging_guide.html#making-a-package-discoverable-with-spack-external-find + executables = ['clang', 'flang', 'ld.lld', 'lldb'] + + @classmethod + def filter_detected_exes(cls, prefix, exes_in_prefix): + result = [] + for exe in exes_in_prefix: + # Executables like lldb-vscode-X are daemon listening + # on some port and would hang Spack during detection. + # clang-cl and clang-cpp are dev tools that we don't + # need to test + if any(x in exe for x in ('vscode', 'cpp', '-cl', '-gpu')): + continue + result.append(exe) + return result + + @classmethod + def determine_version(cls, exe): + version_regex = re.compile( + # Normal clang compiler versions are left as-is + r'clang version ([^ )\n]+)-svn[~.\w\d-]*|' + # Don't include hyphenated patch numbers in the version + # (see https://github.com/spack/spack/pull/14365 for details) + r'clang version ([^ )\n]+?)-[~.\w\d-]*|' + r'clang version ([^ )\n]+)|' + # LLDB + r'lldb version ([^ )\n]+)|' + # LLD + r'LLD ([^ )\n]+) \(compatible with GNU linkers\)' + ) + try: + compiler = Executable(exe) + output = compiler('--version', output=str, error=str) + if 'Apple' in output: + return None + match = version_regex.search(output) + if match: + return match.group(match.lastindex) + except spack.util.executable.ProcessError: + pass + except Exception as e: + tty.debug(e) + + return None + + @classmethod + def determine_variants(cls, exes, version_str): + variants, compilers = ['+clang'], {} + lld_found, lldb_found = False, False + for exe in exes: + if 'clang++' in exe: + compilers['cxx'] = exe + elif 'clang' in exe: + compilers['c'] = exe + elif 'flang' in exe: + variants.append('+flang') + compilers['fc'] = exe + compilers['f77'] = exe + elif 'ld.lld' in exe: + lld_found = True + compilers['ld'] = exe + elif 'lldb' in exe: + lldb_found = True + compilers['lldb'] = exe + + variants.append('+lld' if lld_found else '~lld') + variants.append('+lldb' if lldb_found else '~lldb') + + return ''.join(variants), {'compilers': compilers} + + @classmethod + def validate_detected_spec(cls, spec, extra_attributes): + # For LLVM 'compilers' is a mandatory attribute + msg = ('the extra attribute "compilers" must be set for ' + 'the detected spec "{0}"'.format(spec)) + assert 'compilers' in extra_attributes, msg + compilers = extra_attributes['compilers'] + for key in ('c', 'cxx'): + msg = '{0} compiler not found for {1}' + assert key in compilers, msg.format(key, spec) + + @property + def cc(self): + msg = "cannot retrieve C compiler [spec is not concrete]" + assert self.spec.concrete, msg + if self.spec.external: + return self.spec.extra_attributes['compilers'].get('c', None) + result = None + if '+clang' in self.spec: + result = os.path.join(self.spec.prefix.bin, 'clang') + return result + + @property + def cxx(self): + msg = "cannot retrieve C++ compiler [spec is not concrete]" + assert self.spec.concrete, msg + if self.spec.external: + return self.spec.extra_attributes['compilers'].get('cxx', None) + result = None + if '+clang' in self.spec: + result = os.path.join(self.spec.prefix.bin, 'clang++') + return result + + @property + def fc(self): + msg = "cannot retrieve Fortran compiler [spec is not concrete]" + assert self.spec.concrete, msg + if self.spec.external: + return self.spec.extra_attributes['compilers'].get('fc', None) + result = None + if '+flang' in self.spec: + result = os.path.join(self.spec.prefix.bin, 'flang') + return result + + @property + def f77(self): + msg = "cannot retrieve Fortran 77 compiler [spec is not concrete]" + assert self.spec.concrete, msg + if self.spec.external: + return self.spec.extra_attributes['compilers'].get('f77', None) + result = None + if '+flang' in self.spec: + result = os.path.join(self.spec.prefix.bin, 'flang') + return result + + @run_before('cmake') + def codesign_check(self): + if self.spec.satisfies("+code_signing"): + codesign = which('codesign') + mkdir('tmp') + llvm_check_file = join_path('tmp', 'llvm_check') + copy('/usr/bin/false', llvm_check_file) + try: + codesign('-f', '-s', 'lldb_codesign', '--dryrun', + llvm_check_file) + + except ProcessError: + # Newer LLVM versions have a simple script that sets up + # automatically when run with sudo priviliges + setup = Executable("./lldb/scripts/macos-setup-codesign.sh") + try: + setup() + except Exception: + raise RuntimeError( + 'spack was unable to either find or set up' + 'code-signing on your system. Please refer to' + 'https://lldb.llvm.org/resources/build.html#' + 'code-signing-on-macos for details on how to' + 'create this identity.' + ) + + def flag_handler(self, name, flags): + if name == 'cxxflags': + flags.append(self.compiler.cxx11_flag) + return(None, flags, None) + elif name == 'ldflags' and self.spec.satisfies('%intel'): + flags.append('-shared-intel') + return(None, flags, None) + return(flags, None, None) + + def setup_run_environment(self, env): + if "+clang" in self.spec: + env.set("CC", join_path(self.spec.prefix.bin, "clang")) + env.set("CXX", join_path(self.spec.prefix.bin, "clang++")) + if "+flang" in self.spec: + env.set("FC", join_path(self.spec.prefix.bin, "flang")) + env.set("F77", join_path(self.spec.prefix.bin, "flang")) + + root_cmakelists_dir = "llvm" + + def cmake_args(self): + spec = self.spec + python = spec['python'] + cmake_args = [ + "-DLLVM_REQUIRES_RTTI:BOOL=ON", + "-DLLVM_ENABLE_RTTI:BOOL=ON", + "-DLLVM_ENABLE_EH:BOOL=ON", + "-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp", + "-DPYTHON_EXECUTABLE:PATH={0}".format(python.command.path), + "-DLIBOMP_USE_HWLOC:BOOL=ON", + "-DLIBOMP_HWLOC_INSTALL_DIR={0}".format(spec["hwloc"].prefix), + ] + + if python.version >= Version("3.0.0"): + cmake_args.append("-DPython3_EXECUTABLE={0}".format( + python.command.path)) + else: + cmake_args.append("-DPython2_EXECUTABLE={0}".format( + python.command.path)) + + projects = [] + + if "+cuda" in spec: + cmake_args.extend( + [ + "-DCUDA_TOOLKIT_ROOT_DIR:PATH=" + spec["cuda"].prefix, + "-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES={0}".format( + ",".join(spec.variants["cuda_arch"].value) + ), + "-DCLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_{0}".format( + spec.variants["cuda_arch"].value[-1] + ), + ] + ) + else: + # still build libomptarget but disable cuda + cmake_args.extend( + [ + "-DCUDA_TOOLKIT_ROOT_DIR:PATH=IGNORE", + "-DCUDA_SDK_ROOT_DIR:PATH=IGNORE", + "-DCUDA_NVCC_EXECUTABLE:FILEPATH=IGNORE", + "-DLIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES:STRING=IGNORE", + ] + ) + + if "+omp_debug" in spec: + cmake_args.append("-DLIBOMPTARGET_ENABLE_DEBUG:Bool=ON") + + if "+python" in spec and "+lldb" in spec and spec.satisfies("@5.0.0:"): + cmake_args.append("-DLLDB_USE_SYSTEM_SIX:Bool=TRUE") + + if "+lldb" in spec and spec.satisfies("@:9.9.9"): + cmake_args.append("-DLLDB_DISABLE_PYTHON:Bool={0}".format( + 'ON' if '~python' in spec else 'OFF')) + if "+lldb" in spec and spec.satisfies("@10.0.0:"): + cmake_args.append("-DLLDB_ENABLE_PYTHON:Bool={0}".format( + 'ON' if '+python' in spec else 'OFF')) + + if "+gold" in spec: + cmake_args.append( + "-DLLVM_BINUTILS_INCDIR=" + spec["binutils"].prefix.include + ) + + if "+clang" in spec: + projects.append("clang") + projects.append("clang-tools-extra") + projects.append("openmp") + if "+flang" in spec: + projects.append("flang") + if "+lldb" in spec: + projects.append("lldb") + if "+lld" in spec: + projects.append("lld") + if "+compiler-rt" in spec: + projects.append("compiler-rt") + if "+libcxx" in spec: + projects.append("libcxx") + projects.append("libcxxabi") + if "+mlir" in spec: + projects.append("mlir") + if "+internal_unwind" in spec: + projects.append("libunwind") + if "+polly" in spec: + projects.append("polly") + cmake_args.append("-DLINK_POLLY_INTO_TOOLS:Bool=ON") + + if "+shared_libs" in spec: + cmake_args.append("-DBUILD_SHARED_LIBS:Bool=ON") + if "+llvm_dylib" in spec: + cmake_args.append("-DLLVM_BUILD_LLVM_DYLIB:Bool=ON") + if "+omp_debug" in spec: + cmake_args.append("-DLIBOMPTARGET_ENABLE_DEBUG:Bool=ON") + + if "+split_dwarf" in spec: + cmake_args.append("-DLLVM_USE_SPLIT_DWARF:Bool=ON") + + if "+all_targets" not in spec: # all is default on cmake + + targets = ["NVPTX", "AMDGPU"] + if spec.version < Version("3.9.0"): + # Starting in 3.9.0 CppBackend is no longer a target (see + # LLVM_ALL_TARGETS in llvm's top-level CMakeLists.txt for + # the complete list of targets) + targets.append("CppBackend") + + if spec.target.family == "x86" or spec.target.family == "x86_64": + targets.append("X86") + elif spec.target.family == "arm": + targets.append("ARM") + elif spec.target.family == "aarch64": + targets.append("AArch64") + elif ( + spec.target.family == "sparc" + or spec.target.family == "sparc64" + ): + targets.append("Sparc") + elif ( + spec.target.family == "ppc64" + or spec.target.family == "ppc64le" + or spec.target.family == "ppc" + or spec.target.family == "ppcle" + ): + targets.append("PowerPC") + + cmake_args.append( + "-DLLVM_TARGETS_TO_BUILD:STRING=" + ";".join(targets) + ) + + if "+omp_tsan" in spec: + cmake_args.append("-DLIBOMP_TSAN_SUPPORT=ON") + + if self.compiler.name == "gcc": + compiler = Executable(self.compiler.cc) + gcc_output = compiler('-print-search-dirs', output=str, error=str) + + for line in gcc_output.splitlines(): + if line.startswith("install:"): + # Get path and strip any whitespace + # (causes oddity with ancestor) + gcc_prefix = line.split(":")[1].strip() + gcc_prefix = ancestor(gcc_prefix, 4) + break + cmake_args.append("-DGCC_INSTALL_PREFIX=" + gcc_prefix) + + if spec.satisfies("@4.0.0:"): + if spec.satisfies("platform=cray") or spec.satisfies( + "platform=linux" + ): + cmake_args.append("-DCMAKE_BUILD_WITH_INSTALL_RPATH=1") + + if self.spec.satisfies("~code_signing platform=darwin"): + cmake_args.append('-DLLDB_USE_SYSTEM_DEBUGSERVER=ON') + + # Semicolon seperated list of projects to enable + cmake_args.append( + "-DLLVM_ENABLE_PROJECTS:STRING={0}".format(";".join(projects)) + ) + + return cmake_args + + @run_before("build") + def pre_install(self): + with working_dir(self.build_directory): + # When building shared libraries these need to be installed first + make("install-LLVMTableGen") + if self.spec.version >= Version("4.0.0"): + # LLVMDemangle target was added in 4.0.0 + make("install-LLVMDemangle") + make("install-LLVMSupport") + + @run_after("install") + def post_install(self): + spec = self.spec + + # unnecessary if we get bootstrap builds in here + if "+cuda" in self.spec: + ompdir = "build-bootstrapped-omp" + # rebuild libomptarget to get bytecode runtime library files + with working_dir(ompdir, create=True): + cmake_args = [ + self.stage.source_path + "/openmp", + "-DCMAKE_C_COMPILER:PATH={0}".format( + spec.prefix.bin + "/clang" + ), + "-DCMAKE_CXX_COMPILER:PATH={0}".format( + spec.prefix.bin + "/clang++" + ), + "-DCMAKE_INSTALL_PREFIX:PATH={0}".format(spec.prefix), + ] + cmake_args.extend(self.cmake_args()) + cmake_args.append( + "-DLIBOMPTARGET_NVPTX_ENABLE_BCLIB:BOOL=TRUE" + ) + + # work around bad libelf detection in libomptarget + cmake_args.append( + "-DLIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR:String={0}".format( + spec["libelf"].prefix.include + ) + ) + + cmake(*cmake_args) + make() + make("install") + if "+python" in self.spec: + install_tree("llvm/bindings/python", site_packages_dir) + + if "+clang" in self.spec: + install_tree("clang/bindings/python", site_packages_dir) + + with working_dir(self.build_directory): + install_tree("bin", join_path(self.prefix, "libexec", "llvm")) + + def add_files_to_view(self, view, merge_map): + # we remove libgomp-related files from views as they conflict with + # gcc-ones + ignore_file_paths = [ + join_path(self.prefix, "lib", "libgomp.so"), + ] + + if self.spec.satisfies('~force_full_view'): + for path in ignore_file_paths: + if path in merge_map: + del merge_map[path] + + super(Llvm, self).add_files_to_view(view, merge_map) diff --git a/packages/llvm/thread-p9.patch b/packages/llvm/thread-p9.patch new file mode 100644 index 0000000000000000000000000000000000000000..140473a8508ada02fd50e06b18f3301bd64072d4 --- /dev/null +++ b/packages/llvm/thread-p9.patch @@ -0,0 +1,16 @@ +diff --git a/libcxx/include/thread b/libcxx/include/thread +index 02da703..d1677a1 100644 +--- a/projects/libcxx/include/thread ++++ b/projects/libcxx/include/thread +@@ -368,9 +368,9 @@ sleep_for(const chrono::duration<_Rep, _Period>& __d) + { + #if defined(_LIBCPP_COMPILER_GCC) && (__powerpc__ || __POWERPC__) + // GCC's long double const folding is incomplete for IBM128 long doubles. +- _LIBCPP_CONSTEXPR duration<long double> _Max = nanoseconds::max(); +-#else + _LIBCPP_CONSTEXPR duration<long double> _Max = duration<long double>(ULLONG_MAX/1000000000ULL) ; ++#else ++ _LIBCPP_CONSTEXPR duration<long double> _Max = nanoseconds::max(); + #endif + nanoseconds __ns; + if (__d < _Max) diff --git a/packages/visionary-dls-core/package.py b/packages/meta-brainscales/package.py similarity index 64% rename from packages/visionary-dls-core/package.py rename to packages/meta-brainscales/package.py index 6fc996810cbb1686824e25cde5f91e7a0a42fb91..a99210c9d1ffaa8d3dc3e61c48d1b43e75d25f5b 100644 --- a/packages/visionary-dls-core/package.py +++ b/packages/meta-brainscales/package.py @@ -6,7 +6,7 @@ from spack import * -class VisionaryDlsCore(Package): +class MetaBrainscales(Package): """Core package that contains dependencies of the core DLS software ONLY!""" @@ -18,58 +18,51 @@ class VisionaryDlsCore(Package): # TODO: as soon as a MetaPackage-concept has been merged, please update this package version('1.0', '372ce038842f20bf0ae02de50c26e85d', url='https://github.com/electronicvisions/spack/archive/v0.8.tar.gz') - # Depend on visionary-nux to enable joint developement of host and PPU code with one meta package - depends_on('visionary-nux ~dev') + # PPU compiler dependencies + depends_on('gettext') + depends_on('zlib') + depends_on('bison') + depends_on('flex') + depends_on('m4') + depends_on('texinfo') + depends_on('wget') + conflicts('flex', when='@2.6.3', msg='Binutils 2.25 for Nux doesn\'t build with flex 2.6.3.') - # depends_on('libusb-1.0') external dependency - depends_on('bear') + # host software dependencies depends_on('bitsery') - depends_on('boost@1.69.0: +graph+icu+mpi+python+numpy+coroutine+context+valgrind cxxstd=17') + depends_on('binutils+gold+ld+plugins') # specialize + depends_on('boost@1.69.0: +graph+icu+mpi+python+numpy+coroutine+context cxxstd=17') # specialize boost (non-clingo) depends_on('cereal') depends_on('cppcheck') depends_on('doxygen+graphviz') - depends_on('genpybind@visions') + depends_on('genpybind@ebrains') depends_on('gflags') depends_on('googletest+gmock') - # depends_on('icarus') depends_on('intel-tbb') # ppu gdbserver depends_on('libelf') depends_on('liblockfile') depends_on('llvm') depends_on('log4cxx') - depends_on('munge') depends_on('pkg-config') + depends_on('python@3.7.0:') # BrainScaleS(-2) only supports Python >= 3.7 + depends_on('py-h5py') # PyNN tests need it depends_on('py-matplotlib') depends_on('py-nose') depends_on('py-numpy') - depends_on('py-pybind11') + depends_on('py-pybind11@2.6.0:2.6.999') # workaround concretization error (py-scipy doesn't like 2.7) depends_on('py-pybind11-stubgen') depends_on('py-pycodestyle') depends_on('py-pyelftools') depends_on('py-pylint') depends_on('py-pynn') - depends_on('py-python-usbtmc') + depends_on('py-pyyaml') depends_on('py-scipy') depends_on('py-sqlalchemy') - depends_on('util-linux') # from lib-rcf + depends_on('util-linux') depends_on('yaml-cpp+shared') - ################## - # Current fixups # - ################## - # intel-mkldnn depends on intel-mkl which also provides blas -> - # concretization error -> reinvestigate when needed - depends_on('py-torch ~mkldnn') - - # we only support Python 3.7+! - depends_on('python@3.7.0:') - - # xilinx runtime dependencies - depends_on('visionary-xilinx') + # dummy installer; it's a "meta" package def install(self, spec, prefix): mkdirp(prefix.etc) - # store a copy of this package. install(__file__, join_path(prefix.etc, spec.name + '.py')) - - # we could create some filesystem view here? diff --git a/packages/nest/package.py b/packages/nest/package.py index 13543f6f9a824c7674d96b2ba4853b7966133107..f2c7cbcbcee4be7b0efced2ebd00f1bb1f9c6997 100644 --- a/packages/nest/package.py +++ b/packages/nest/package.py @@ -75,6 +75,7 @@ class Nest(CMakePackage): depends_on('doxygen', type='build') depends_on('gsl', when='+gsl') depends_on('readline') + depends_on('ncurses') depends_on('libtool') depends_on('pkgconfig', type='build') @@ -114,14 +115,12 @@ class Nest(CMakePackage): env['F77'] = spec['mpi'].mpif77 env['FC'] = spec['mpi'].mpifc - configure(*self.cmake_args(spec, prefix)) + configure(*configure_args) make() make("install") def cmake_args(self): - args = [ - "--prefix=" + prefix, - ] + args = [] if '+mpi' in self.spec: args.append('-Dwith-mpi=ON') diff --git a/packages/py-pybind11/package.py b/packages/py-pybind11/package.py new file mode 100644 index 0000000000000000000000000000000000000000..a353ce632ee5a6285bf02e822714e071becf18b2 --- /dev/null +++ b/packages/py-pybind11/package.py @@ -0,0 +1,91 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack import * + + +class PyPybind11(CMakePackage): + """pybind11 -- Seamless operability between C++11 and Python. + + pybind11 is a lightweight header-only library that exposes C++ types in + Python and vice versa, mainly to create Python bindings of existing C++ + code. Its goals and syntax are similar to the excellent Boost.Python + library by David Abrahams: to minimize boilerplate code in traditional + extension modules by inferring type information using compile-time + introspection.""" + + homepage = "https://pybind11.readthedocs.io" + url = "https://github.com/pybind/pybind11/archive/v2.6.2.tar.gz" + git = "https://github.com/pybind/pybind11.git" + + maintainers = ['ax3l'] + + version('master', branch='master') + version('2.7.0', sha256='6cd73b3d0bf3daf415b5f9b87ca8817cc2e2b64c275d65f9500250f9fee1677e') + version('2.6.2', sha256='8ff2fff22df038f5cd02cea8af56622bc67f5b64534f1b83b9f133b8366acff2') + version('2.6.1', sha256='cdbe326d357f18b83d10322ba202d69f11b2f49e2d87ade0dc2be0c5c34f8e2a') + version('2.5.0', sha256='97504db65640570f32d3fdf701c25a340c8643037c3b69aec469c10c93dc8504') + version('2.4.3', sha256='1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d') + version('2.3.0', sha256='0f34838f2c8024a6765168227ba587b3687729ebf03dc912f88ff75c7aa9cfe8') + version('2.2.4', sha256='b69e83658513215b8d1443544d0549b7d231b9f201f6fc787a2b2218b408181e') + version('2.2.3', sha256='3a3b7b651afab1c5ba557f4c37d785a522b8030dfc765da26adc2ecd1de940ea') + version('2.2.2', sha256='b639a2b2cbf1c467849660801c4665ffc1a4d0a9e153ae1996ed6f21c492064e') + version('2.2.1', sha256='f8bd1509578b2a1e7407d52e6ee8afe64268909a1bbda620ca407318598927e7') + version('2.2.0', sha256='1b0fda17c650c493f5862902e90f426df6751da8c0b58c05983ab009951ed769') + version('2.1.1', sha256='f2c6874f1ea5b4ad4ffffe352413f7d2cd1a49f9050940805c2a082348621540') + version('2.1.0', sha256='2860f2b8d0c9f65f0698289a161385f59d099b7ead1bf64e8993c486f2b93ee0') + + depends_on('py-setuptools', type='build') + depends_on('py-pytest', type='test') + + extends('python') + + # compiler support + conflicts('%gcc@:4.7') + conflicts('%clang@:3.2') + conflicts('%intel@:16') + + def cmake_args(self): + args = [] + args.append('-DPYTHON_EXECUTABLE:FILEPATH=%s' + % self.spec['python'].command.path) + args += [ + self.define('PYBIND11_TEST', self.run_tests) + ] + return args + + def setup_build_environment(self, env): + env.set('PYBIND11_USE_CMAKE', 1) + + # https://github.com/pybind/pybind11/pull/1995 + @when('@:2.4.99') + def patch(self): + """ see https://github.com/spack/spack/issues/13559 """ + filter_file('import sys', + 'import sys; return "{0}"'.format(self.prefix.include), + 'pybind11/__init__.py', + string=True) + + def install(self, spec, prefix): + super(PyPybind11, self).install(spec, prefix) + setup_py('install', '--single-version-externally-managed', '--root=/', + '--prefix={0}'.format(prefix)) + + @run_after('install') + @on_package_attributes(run_tests=True) + def install_test(self): + with working_dir('spack-test', create=True): + # test include helper points to right location + python = self.spec['python'].command + py_inc = python( + '-c', + 'import pybind11 as py; ' + + self.spec['python'].package.print_string('py.get_include()'), + output=str).strip() + for inc in [py_inc, self.prefix.include]: + inc_file = join_path(inc, 'pybind11', 'pybind11.h') + assert os.path.isfile(inc_file) diff --git a/packages/pynn-brainscales/package.py b/packages/pynn-brainscales/package.py new file mode 100644 index 0000000000000000000000000000000000000000..4f9c40a283ad05c39ad616933dad77c9e3e360d1 --- /dev/null +++ b/packages/pynn-brainscales/package.py @@ -0,0 +1,75 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack import * + + +class PynnBrainscales(WafPackage): + """PyNN toplevel for the BrainScaleS-2 neuromorphic hardware systems""" + + homepage = "https://github.com/electronicvisions/pynn-brainscales" + #url = "https://github.com/electronicvisions/pynn-brainscales/archive/v0.1.0.tar.gz" + git = "https://github.com/electronicvisions/pynn-brainscales.git" + + version('waf', branch='waf') + depends_on('meta-brainscales', type=('build', 'link', 'run')) + extends('python') + + def setup_build_environment(self, env): + """waf needs to find headers and libraries by itself (mostly `boost` + tool, but also `gtest`); it also needs to run executables during + configuration.""" + + include = [] + for dep in self.spec.traverse(deptype='build'): + query = self.spec[dep.name] + try: + include.extend(query.headers.directories) + except: + pass + + library = [] + for dep in self.spec.traverse(deptype=('link', 'run')): + query = self.spec[dep.name] + try: + library.extend(query.libs.directories) + print(query.libs.directories, "\n") + except: + pass + + path = [] + for dep in self.spec.traverse(deptype=('build', 'link', 'run')): + query = self.spec[dep.name] + try: + path.append(query.prefix.bin) + except: + pass + + # llvm might be built with ~shared_libs but still builds shared libs + if not any('llvm' in lib for lib in library): + library.append(self.spec['llvm'].prefix.lib) + + env.set('CPATH', ':'.join(include)) + env.set('C_INCLUDE_PATH', ':'.join(include)) + env.set('CPLUS_INCLUDE_PATH', ':'.join(include)) + env.set('LIBRARY_PATH', ':'.join(library)) + env.set('LD_LIBRARY_PATH', ':'.join(library)) + env.prepend_path('PATH', ':'.join(path)) + + # override configure step as we perform a project setup first + def configure(self, spec, prefix): + """Setup and configure the project.""" + + self.waf('setup', '--project=pynn-brainscales', '--without-grenade-ppu-support', '--without-munge', + '--project=grenade@ebrains_release-1-rc1', + '--project=hate@ebrains_release-1-rc1', + '--project=hxcomm@ebrains_release-1-rc1', + '--project=lib-rcf@ebrains_release-1-rc1', + '--project=visions-slurm@ebrains_release-1-rc1' + ) + + args = ['--prefix={0}'.format(self.prefix)] + args += self.configure_args() + + self.waf('configure', *args) diff --git a/packages/tvb-data/package.py b/packages/tvb-data/package.py new file mode 100644 index 0000000000000000000000000000000000000000..2c2a36892cdff8d49df799371f916e62c8a829b8 --- /dev/null +++ b/packages/tvb-data/package.py @@ -0,0 +1,25 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class TvbData(PythonPackage): + """ + Various demonstration datasets for use with The Virtual Brain are provided here. + """ + + homepage = "https://zenodo.org/record/4263723" + url = 'https://zenodo.org/record/4263723/files/tvb_data.zip' + + maintainers = ['paulapopa'] + + version('2.0.3', '1e02cdc21147f46644c57b14429f564f') + + # python_requires + depends_on('python@3.8:', type=('build', 'run')) + + # setup_requires + depends_on('py-setuptools', type='build') diff --git a/packages/tvb-framework/package.py b/packages/tvb-framework/package.py new file mode 100644 index 0000000000000000000000000000000000000000..9d7d7197626ae69d93f941e4dec77e7b0e9644ec --- /dev/null +++ b/packages/tvb-framework/package.py @@ -0,0 +1,55 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class TvbFramework(PythonPackage): + """ + "The Virtual Brain" Project (TVB Project) has the purpose of offering modern tools to the Neurosciences community, + for computing, simulating and analyzing functional and structural data of human brains, brains modeled at the level + of population of neurons. + """ + + homepage = "https://www.thevirtualbrain.org/" + pypi = 'tvb-framework/tvb-framework-2.3.tar.gz' + + maintainers = ['paulapopa'] + + version('2.3', '0f3386135cdbd80bfd7d31f2e056b015a27ff6d081492db16274deed581b0aac') + version('2.2', 'dfcdecee7325dd9b75eb63ddca56d046a4a6f3a20a9bc71a609315b6f151d68b') + version('2.0.10', '082c8c742680804a62fd20b80d0ac0fa7421b0e1cd3a54293ff2fec2abc4f15d') + version('2.0.9', '8a3af6d52e057db901d38c524d4bf1c7f78e793b990b27950d0765b8edf03c61') + version('2.0.8', '95a8585b3095eecbdc8ca389269471a350b6876e732f62c260d75b615d10a237') + + # python_requires + depends_on('python@3.8:', type=('build', 'run')) + + # setup_requires + depends_on('py-setuptools', type='build') + + # install_requires + depends_on('py-alembic', type=('build', 'run')) + depends_on('py-cherrypy', type=('build', 'run')) + # depends_on('py-formencode', type=('build', 'run')) + # depends_on('py-gevent', type=('build', 'run')) + depends_on('py-h5py', type=('build', 'run')) + depends_on('py-jinja2', type=('build', 'run')) + depends_on('py-nibabel', type=('build', 'run')) + depends_on('py-numpy', type=('build', 'run')) + depends_on('py-pandas', type=('build', 'run')) + depends_on('py-pillow', type=('build', 'run')) + depends_on('py-psutil', type=('build', 'run')) + depends_on('py-requests', type=('build', 'run')) + depends_on('py-scikit-learn', type=('build', 'run')) + depends_on('py-scipy', type=('build', 'run')) + depends_on('py-simplejson', type=('build', 'run')) + depends_on('py-six', type=('build', 'run')) + depends_on('py-sqlalchemy', type=('build', 'run')) + depends_on('tvb-data', type='run') + depends_on('tvb-library', type=('build', 'run')) + depends_on('tvb-storage', type=('build', 'run'), when='@2.3') + depends_on('py-werkzeug', type=('build', 'run')) + diff --git a/packages/tvb-library/package.py b/packages/tvb-library/package.py new file mode 100644 index 0000000000000000000000000000000000000000..d11688e6babe092c33a2f25011b4388355b0d545 --- /dev/null +++ b/packages/tvb-library/package.py @@ -0,0 +1,56 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class TvbLibrary(PythonPackage): + """ + "The Virtual Brain" Project (TVB Project) has the purpose of offering modern tools to the Neurosciences community, + for computing, simulating and analyzing functional and structural data of human brains, brains modeled at the level + of population of neurons. + """ + + homepage = "https://www.thevirtualbrain.org/" + pypi = 'tvb-library/tvb-library-2.3.tar.gz' + + maintainers = ['paulapopa'] + + version('2.3', '0595f2eca95e5ed4c7a1c88425914cc71d0ea7a9f4ec575b6a315ca2408ea801') + version('2.2', 'de70440b0cfd517e49a8ef52aa01f3bfde87a907abbf68ae5d85fef9a28000dd') + version('2.1', '99c2817d9d341abd6d7ff07c4a827f58462d85a4dbb975e514840f362b3ca5cc') + version('2.0.10', '27ece9ec3a79125b037fdd67963da23dc4fad7bd7154b884faa6c26c2775a1b8') + version('2.0.9', '0c3109c03665e3dd516fda12ba2155d718cd9933fb25b9f7bd0906851e937f39') + version('2.0.8', '41e912723b66fe7beeff79f6b760d8ae8c34b0e80e668e81ce59c380ad00506d') + version('2.0.7', 'b9a6c03b8b7c55e512b0a601260934c984870c875f958423c360e3813e70100b') + version('2.0.6', 'f1ee168939e522f698b2fe18c2b7013b827ace198d4af777b38cf80fc2ab5db3') + version('2.0.5', 'a1af85c3a376b52daa140416320f59794263fac99796a3a9e47ec1db46bda160') + version('2.0.3', 'f4ebe3f3bba13dd6b32568cc016cb218251002a8c20acb992e030ab2ca5b30c8') + version('2.0.2', '1893e641108c8fdbafc7c3400ac95b0f5f0a714fd0a378258ac065d11d2de071') + version('2.0', 'a89bcd1949788d35722a1dc1e3bb8d5e32fa02a4eef977fd476ab6df18285e9b') + + # python_requires + depends_on('python@3.8:', type=('build', 'run')) + + # setup_requires + depends_on('py-setuptools', type='build') + + # install_requires + depends_on('py-autopep8', type=('build', 'run')) + depends_on('py-mako', type=('build', 'run')) + depends_on('py-matplotlib', type=('build', 'run')) + depends_on('py-numpy', type=('build', 'run')) + depends_on('py-numba', type=('build', 'run')) + depends_on('py-numexpr', type=('build', 'run')) + depends_on('py-requests', type=('build', 'run')) + depends_on('py-scipy', type=('build', 'run')) + depends_on('py-six', type=('build', 'run')) + depends_on('tvb-data', type=('run')) + + # extra_requires + # ["h5py", "mpl_toolkits", "tvb-gdist"] + + # test_requires + # ["pytest", "pytest-benchmark"] diff --git a/packages/tvb-storage/package.py b/packages/tvb-storage/package.py new file mode 100644 index 0000000000000000000000000000000000000000..b6af7e09ff722d745374943a5b1423582fda2d85 --- /dev/null +++ b/packages/tvb-storage/package.py @@ -0,0 +1,33 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class TvbStorage(PythonPackage): + """ + "The Virtual Brain" Project (TVB Project) has the purpose of offering modern tools to the Neurosciences community, + for computing, simulating and analyzing functional and structural data of human brains, brains modeled at the level + of population of neurons. + """ + + homepage = "https://www.thevirtualbrain.org/" + pypi = 'tvb-storage/tvb-storage-2.3.tar.gz' + + maintainers = ['paulapopa'] + + version('2.3', 'e10bb40a486771703ba2776555ea5a453e619b07f10e4dea0d347043dee8f54b') + + # python_requires + depends_on('python@3.8:', type=('build', 'run')) + + # setup_requires + depends_on('py-setuptools', type='build') + + # install_requires + depends_on('py-h5py', type=('build', 'run')) + depends_on('py-numpy', type=('build', 'run')) + depends_on('py-scipy', type=('build', 'run')) + depends_on('tvb-library', type=('build', 'run')) diff --git a/packages/visionary-dls-core/.gitkeep b/packages/visionary-dls-core/.gitkeep deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/packages/visionary-nux/.gitkeep b/packages/visionary-nux/.gitkeep deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/packages/visionary-nux/package.py b/packages/visionary-nux/package.py deleted file mode 100644 index 15a6a87d369bd930dac52a839c4aae091958e3e2..0000000000000000000000000000000000000000 --- a/packages/visionary-nux/package.py +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - - -class VisionaryNux(Package): - """Visionary Meta Package""" - - homepage = '' - # some random tarball, to make `spack fetch --dependencies visionary-defaults` work - url = 'https://github.com/electronicvisions/spack/archive/v0.8.tar.gz' - - # This is only a dummy tarball (see difference between version numbers) - # TODO: as soon as a MetaPackage-concept has been merged, please update this package - version('1.0', '372ce038842f20bf0ae02de50c26e85d', url='https://github.com/electronicvisions/spack/archive/v0.8.tar.gz') - - variant('dev', default=True) - - depends_on('visionary-dev-tools', when='+dev') - - depends_on('gettext') - depends_on('zlib') - - # was bison 3.0.4 in the past - depends_on('bison') - depends_on('flex') - depends_on('m4') - depends_on('texinfo') - depends_on('wget') - - conflicts('flex', when='@2.6.3', msg='Binutils 2.25 for Nux doesn\'t build with flex 2.6.3.') - - def install(self, spec, prefix): - mkdirp(prefix.etc) - # store a copy of this package. - install(__file__, join_path(prefix.etc, 'visionary-nux.py')) - - # we could create some filesystem view here? diff --git a/spack.yaml b/spack.yaml index 33b81372ef4fa93867a58b12ee9982c97ef864cb..a8f3e18f090aad71d3b94a0ff14e660caa2a475a 100644 --- a/spack.yaml +++ b/spack.yaml @@ -7,10 +7,19 @@ spack: - py-notebook %gcc@10.3.0 # General - py-pip %gcc@10.3.0 + - py-numpy %gcc@10.3.0 + - py-scipy %gcc@10.3.0 + - py-pandas %gcc@10.3.0 + - py-seaborn %gcc@10.3.0 + - py-matplotlib %gcc@10.3.0 # EBRAINS simulators - - nest@2.20.0 +python +gsl +mpi %gcc@10.3.0 - - arbor +mpi %gcc@10.3.0 + - nest@3.0 +python +gsl +mpi %gcc@10.3.0 + - arbor +mpi ^python@3:3.9 %gcc@10.3.0 - neuron +mpi %gcc@10.3.0 - py-pynn %gcc@10.3.0 - # BrainScaleS (plus some constraining) - - visionary-dls-core ^py-setuptools-scm+toml ^python@3.8.0:3.8.999 ^binutils+ld ^py-astroid@2.5.6:2.6.999 ^py-scipy@:1.6.999 %gcc@10.3.0 + - tvb-data %gcc@10.3.0 + - tvb-library ^binutils+ld+gold %gcc@10.3.0 + - tvb-storage ^binutils+ld+gold %gcc@10.3.0 + - tvb-framework ^binutils+ld+gold %gcc@10.3.0 + - meta-brainscales %gcc@10.3.0 + - pynn-brainscales %gcc@10.3.0