diff --git a/.gitignore b/.gitignore index 770d47e25d61f..58429d093bfe5 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,7 @@ all.config # Kdevelop4 *.kdev4 + +# dtb objects +*.dtb +*.dtbo diff --git a/Documentation/ABI/testing/debugfs-aufs b/Documentation/ABI/testing/debugfs-aufs new file mode 100644 index 0000000000000..4a6694194ba69 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-aufs @@ -0,0 +1,55 @@ +What: /debug/aufs/si_/ +Date: March 2009 +Contact: J. R. Okajima +Description: + Under /debug/aufs, a directory named si_ is created + per aufs mount, where is a unique id generated + internally. + +What: /debug/aufs/si_/plink +Date: Apr 2013 +Contact: J. R. Okajima +Description: + It has three lines and shows the information about the + pseudo-link. The first line is a single number + representing a number of buckets. The second line is a + number of pseudo-links per buckets (separated by a + blank). The last line is a single number representing a + total number of psedo-links. + When the aufs mount option 'noplink' is specified, it + will show "1\n0\n0\n". + +What: /debug/aufs/si_/xib +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xib (External Inode Number + Bitmap), its block size and file size. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. + +What: /debug/aufs/si_/xi0, xi1 ... xiN and xiN-N +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xino (External Inode Number + Translation Table), its link count, block size and file + size. + Due to the file size limit, there may exist multiple + xino files per branch. In this case, "-N" is added to + the filename and it corresponds to the index of the + internal xino array. "-0" is omitted. + When the aufs mount option 'noxino' is specified, Those + entries won't exist. About XINO files, see the aufs + manual. + +What: /debug/aufs/si_/xigen +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xigen (External Inode + Generation Table), its block size and file size. + If CONFIG_AUFS_EXPORT is disabled, this entry will not + be created. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. diff --git a/Documentation/ABI/testing/sysfs-aufs b/Documentation/ABI/testing/sysfs-aufs new file mode 100644 index 0000000000000..82f9518495eae --- /dev/null +++ b/Documentation/ABI/testing/sysfs-aufs @@ -0,0 +1,31 @@ +What: /sys/fs/aufs/si_/ +Date: March 2009 +Contact: J. R. Okajima +Description: + Under /sys/fs/aufs, a directory named si_ is created + per aufs mount, where is a unique id generated + internally. + +What: /sys/fs/aufs/si_/br0, br1 ... brN +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the abolute path of a member directory (which + is called branch) in aufs, and its permission. + +What: /sys/fs/aufs/si_/brid0, brid1 ... bridN +Date: July 2013 +Contact: J. R. Okajima +Description: + It shows the id of a member directory (which is called + branch) in aufs. + +What: /sys/fs/aufs/si_/xi_path +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the abolute path of XINO (External Inode Number + Bitmap, Translation Table and Generation Table) file + even if it is the default path. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. diff --git a/Documentation/ABI/testing/sysfs-devices-platform-bone_capemgr b/Documentation/ABI/testing/sysfs-devices-platform-bone_capemgr new file mode 100644 index 0000000000000..e2df613581234 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-platform-bone_capemgr @@ -0,0 +1,63 @@ +What: /sys/devices/platform/bone_capemgr/slots +Date: May 2015 +KernelVersion: 4.0 +Contact: Pantelis Antoniou +Description: + READ: + Describe the state of all the slots of the beaglebone capemgr. + Each line of the output describes a slot: + The slot format is as following: + : [P-][F-][O-][l-][L-][D-] \ + ,, + , + + Where the flags are: + P: Slot has been probed + F: Slot has failed probing (i.e. no EEPROM detected) + O: Slot has been overridden by the user + l: Slot is current loading + L: Slot has completed loading and is ready + D: Slot has been disabled + + Example: + 0: P---L- -1 BeagleBone RS232 CAPE,00A1,Beagleboardtoys,BB-BONE-SERL-03 + 1: PF---- -1 + 2: PF---- -1 + 3: PF---- -1 + + WRITE: + Writing a string of the form [:version] issues a request to + load a firmware blob containing an overlay. The name of the firmware blob + is -[version|00A0].dtbo. This act is defined as a slot override. + + Writing a negative slot id removes the slot if it was an overridden one, or + unloads a slot that was probed. + +What: /sys/devices/platform/bone_capemgr/baseboard/ +Date: May 2015 +KernelVersion: 4.0 +Contact: Pantelis Antoniou +Description: Contains the probed base board EEPROM field; one of: + board-name - board-name as stored in cape EEPROM + dc-supplied - whether the cape draws or supplies DC + eeprom-format-revision - EEPROM format rev, only 00A0 supported + header - header; should be 'aa 55 33 ee' + manufacturer - manufacturer string + part-number - part-number of the cape + serial-number - serial number of the cape + version - version of the cape, i.e. 00A0 + number-of-pins - displayed but ignored + pin-usage - displayed but ignored + sys-5v - displayed but ignored + vdd-3v3exp - displayed but ignored + vdd-5v - displayed but ignored +What: /sys/devices/platform/bone_capemgr/slot-/ +Date: May 2015 +KernelVersion: 4.0 +Contact: Pantelis Antoniou +Description: Contains the probed cape's EEPROM field; the field is one of: + board-name - baseboard name i.e. A335BNLT + header - header; should be 'aa 55 33 ee' + revision - baseboard revision + serial-number - baseboard serial number + config-option - displayed but ignored diff --git a/Documentation/ABI/testing/sysfs-firmware-devicetree-overlays b/Documentation/ABI/testing/sysfs-firmware-devicetree-overlays new file mode 100644 index 0000000000000..88d15498a21be --- /dev/null +++ b/Documentation/ABI/testing/sysfs-firmware-devicetree-overlays @@ -0,0 +1,52 @@ +What: /sys/firmware/devicetree/overlays/ +Date: October 2015 +Contact: Pantelis Antoniou +Description: + This directory contains the applied device tree overlays of + the running system, as directories of the overlay id. + +What: /sys/firmware/devicetree/overlays/enable +Date: October 2015 +Contact: Pantelis Antoniou +Description: + The master enable switch, by default is 1, and when + set to 0 it cannot be re-enabled for security reasons. + + The discussion about this switch takes place in: + http://comments.gmane.org/gmane.linux.drivers.devicetree/101871 + + Kees Cook: + "Coming from the perspective of drawing a bright line between + kernel and the root user (which tends to start with disabling + kernel module loading), I would say that there at least needs + to be a high-level one-way "off" switch for the interface so + that systems that have this interface can choose to turn it off + during initial boot, etc." + +What: /sys/firmware/devicetree/overlays/ +Date: October 2015 +Contact: Pantelis Antoniou +Description: + Each directory represents an applied overlay, containing + the following attribute files. + +What: /sys/firmware/devicetree/overlays//can_remove +Date: October 2015 +Contact: Pantelis Antoniou +Description: + The attribute set to 1 means that the overlay can be removed, + while 0 means that the overlay is being overlapped therefore + removal is prohibited. + +What: /sys/firmware/devicetree/overlays/// +Date: October 2015 +Contact: Pantelis Antoniou +Description: + Each of these directories contain information about of the + particular overlay fragment. + +What: /sys/firmware/devicetree/overlays///target +Date: October 2015 +Contact: Pantelis Antoniou +Description: + The full-path of the target of the fragment diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 7d8b17ce88047..f50cfae0c7927 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2883,6 +2883,8 @@ This can be set from sysctl after boot. See Documentation/sysctl/vm.txt for details. + of_overlay_disable [OF] Disable device tree overlays at boot time. + ohci1394_dma=early [HW] enable debugging via the ohci1394 driver. See Documentation/debugging-via-ohci1394.txt for more info. diff --git a/Documentation/devicetree/bindings/arm/omap/omap.txt b/Documentation/devicetree/bindings/arm/omap/omap.txt index db971802378cc..63d410a0aece5 100644 --- a/Documentation/devicetree/bindings/arm/omap/omap.txt +++ b/Documentation/devicetree/bindings/arm/omap/omap.txt @@ -24,6 +24,8 @@ Optional properties: - ti,no-reset-on-init: When present, the module should not be reset at init - ti,no-idle-on-init: When present, the module should not be idled at init - ti,no-idle: When present, the module is never allowed to idle. +- ti,deassert-hard-reset: list of hwmod and hardware reset line name pairs + (ascii strings) to be deasserted upon device instantiation. Example: diff --git a/Documentation/devicetree/bindings/misc/bone_capemgr.txt b/Documentation/devicetree/bindings/misc/bone_capemgr.txt new file mode 100644 index 0000000000000..7e4fbc982ceab --- /dev/null +++ b/Documentation/devicetree/bindings/misc/bone_capemgr.txt @@ -0,0 +1,111 @@ +* Beaglebone cape manager driver + +Required properties: +- compatible: "ti,bone-capemgr" +- eeprom: phandle to the EEPROM baseboard. + The EEPROM framework interface is use to obtain the data. + +Required children nodes: + +- baseboardmaps: Contains nodes, which each of the them defines a mapping from + the baseboard EEPROM board-name ID to a DT friendly compatible + string. + + - board-name: The baseboard EEPROM board name, i.e. A335BONE for the + original beaglebone white. + - compatible-name: The DT friendly compatible string to be used for matching + compatible capes, i.e. "ti,beaglebone" + + + - nvmem-cells: Defines the phandles of the nvmem cells of the baseboard and the + slots. + - nvmem-cells: Defines the names of the nvmem cells. Required to have at + least a baseboard cell name. + + - #slots: Defines how many slots are there. + +- Example of a beaglebone cape-manager: + +bone_capemgr { + compatible = "ti,bone-capemgr"; + status = "okay"; + + nvmem-cell = <&baseboard_data + &cape0_data &cape1_data &cape2_data &cape3_data>; + nvmem-cell-names = "baseboard", "slot0", "slot1", "slot2", "slot3"; + + #slots = <4>; + + /* map board revisions to compatible definitions */ + baseboardmaps { + baseboard_beaglebone: board@0 { + board-name = "A335BONE"; + compatible-name = "ti,beaglebone"; + }; + + baseboard_beaglebone_black: board@1 { + board-name = "A335BNLT"; + compatible-name = "ti,beaglebone-black"; + }; + }; +}; + +The format of the cape to be loaded is in a standard overlay format with +the following root properties that are interpreted by the cape manager: + +Required properties: + - compatible: Should be compatible to the baseboard according to the + baseboard map value, i.e. "ti,beaglebone". + - part-numer: Should contain the part-number as stored in the EEPROM. + - version: Should contain a list of all the version that are supported + by the single cape dtbo, i.e. "00A1". + +Optional properties: + - exclusive-use: A string list which state the resources this cape requires. + No processing or matching to anything regarding the internal + kernel state is performed; it's purpose is to guard against + conflicts with other capes. + - priority: A priority to be assigned when loading a cape. A lower value + has higher priority. The purpose of the priority is to control + which cape is loaded first in case of a conflict. + +- Example of a serial cape: + +/dts-v1/; +/plugin/; +/ { + compatible = "ti,beaglebone", "ti,beaglebone-black"; + + /* identification */ + part-number = "BB-BONE-SERL-03"; + version = "00A1"; + + /* state the resources this cape uses */ + exclusive-use = + /* the pin header uses */ + "P9.21", /* uart2_txd */ + "P9.22", /* uart2_rxd */ + /* the hardware ip uses */ + "uart2"; + + fragment@0 { + target = <&am33xx_pinmux>; + __overlay__ { + bb_uart2_pins: pinmux_bb_uart2_pins { + pinctrl-single,pins = < + 0x150 0x21 /* spi0_sclk.uart2_rxd | MODE1 */ + 0x154 0x01 /* spi0_d0.uart2_txd | MODE1 */ + >; + }; + }; + }; + + fragment@1 { + target = <&uart2>; + __overlay__ { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&bb_uart2_pins>; + }; + }; +}; diff --git a/Documentation/devicetree/configfs-overlays.txt b/Documentation/devicetree/configfs-overlays.txt new file mode 100644 index 0000000000000..5fa43e0643072 --- /dev/null +++ b/Documentation/devicetree/configfs-overlays.txt @@ -0,0 +1,31 @@ +Howto use the configfs overlay interface. + +A device-tree configfs entry is created in /config/device-tree/overlays +and and it is manipulated using standard file system I/O. +Note that this is a debug level interface, for use by developers and +not necessarily something accessed by normal users due to the +security implications of having direct access to the kernel's device tree. + +* To create an overlay you mkdir the directory: + + # mkdir /config/device-tree/overlays/foo + +* Either you echo the overlay firmware file to the path property file. + + # echo foo.dtbo >/config/device-tree/overlays/foo/path + +* Or you cat the contents of the overlay to the dtbo file + + # cat foo.dtbo >/config/device-tree/overlays/foo/dtbo + +The overlay file will be applied, and devices will be created/destroyed +as required. + +To remove it simply rmdir the directory. + + # rmdir /config/device-tree/overlays/foo + +The rationalle of the dual interface (firmware & direct copy) is that each is +better suited to different use patterns. The firmware interface is what's +intended to be used by hardware managers in the kernel, while the copy interface +make sense for developers (since it avoids problems with namespaces). diff --git a/Documentation/devicetree/overlay-notes.txt b/Documentation/devicetree/overlay-notes.txt index eb7f2685fda1f..d4ebb42ab3c83 100644 --- a/Documentation/devicetree/overlay-notes.txt +++ b/Documentation/devicetree/overlay-notes.txt @@ -98,6 +98,14 @@ Finally, if you need to remove all overlays in one-go, just call of_overlay_destroy_all() which will remove every single one in the correct order. +If your board has multiple slots/places where a single overlay can work +and each slot is defined by a node, you can use the +of_overlay_create_target_index() method to select the target. + +For overlays on probeable busses, use the of_overlay_create_target_root() method +in which you supply a device node as a target root, and which all target +references in the overlay are performed relative to that node. + Overlay DTS Format ------------------ @@ -108,9 +116,11 @@ The DTS of an overlay should have the following format: fragment@0 { /* first child node */ - target=; /* phandle target of the overlay */ + /* phandle target of the overlay */ + target= [, , ...]; or - target-path="/path"; /* target path of the overlay */ + /* target path of the overlay */ + target-path="/path" [ , "/path", ...]; __overlay__ { property-a; /* add property-a to the target */ @@ -129,3 +139,11 @@ Using the non-phandle based target method allows one to use a base DT which does not contain a __symbols__ node, i.e. it was not compiled with the -@ option. The __symbols__ node is only required for the target= method, since it contains the information required to map from a phandle to a tree location. + +Using a target index requires the use of a selector target on the call to +of_overlay_create_target_index(). I.e. passing an index of 0 will select the +target in the foo node, an index of 1 the bar node, etc. + +Note that when using the target root create method all target references must +lie under the target root node. I.e. the overlay is not allowed to 'break' out +of the root. diff --git a/Documentation/filesystems/aufs/README b/Documentation/filesystems/aufs/README new file mode 100644 index 0000000000000..029c3e5df231b --- /dev/null +++ b/Documentation/filesystems/aufs/README @@ -0,0 +1,402 @@ + +Aufs4 -- advanced multi layered unification filesystem version 4.x +http://aufs.sf.net +Junjiro R. Okajima + + +0. Introduction +---------------------------------------- +In the early days, aufs was entirely re-designed and re-implemented +Unionfs Version 1.x series. Adding many original ideas, approaches, +improvements and implementations, it becomes totally different from +Unionfs while keeping the basic features. +Recently, Unionfs Version 2.x series begin taking some of the same +approaches to aufs1's. +Unionfs is being developed by Professor Erez Zadok at Stony Brook +University and his team. + +Aufs4 supports linux-4.0 and later, and for linux-3.x series try aufs3. +If you want older kernel version support, try aufs2-2.6.git or +aufs2-standalone.git repository, aufs1 from CVS on SourceForge. + +Note: it becomes clear that "Aufs was rejected. Let's give it up." + According to Christoph Hellwig, linux rejects all union-type + filesystems but UnionMount. + + +PS. Al Viro seems have a plan to merge aufs as well as overlayfs and + UnionMount, and he pointed out an issue around a directory mutex + lock and aufs addressed it. But it is still unsure whether aufs will + be merged (or any other union solution). + + + +1. Features +---------------------------------------- +- unite several directories into a single virtual filesystem. The member + directory is called as a branch. +- you can specify the permission flags to the branch, which are 'readonly', + 'readwrite' and 'whiteout-able.' +- by upper writable branch, internal copyup and whiteout, files/dirs on + readonly branch are modifiable logically. +- dynamic branch manipulation, add, del. +- etc... + +Also there are many enhancements in aufs, such as: +- test only the highest one for the directory permission (dirperm1) +- copyup on open (coo=) +- 'move' policy for copy-up between two writable branches, after + checking free space. +- xattr, acl +- readdir(3) in userspace. +- keep inode number by external inode number table +- keep the timestamps of file/dir in internal copyup operation +- seekable directory, supporting NFS readdir. +- whiteout is hardlinked in order to reduce the consumption of inodes + on branch +- do not copyup, nor create a whiteout when it is unnecessary +- revert a single systemcall when an error occurs in aufs +- remount interface instead of ioctl +- maintain /etc/mtab by an external command, /sbin/mount.aufs. +- loopback mounted filesystem as a branch +- kernel thread for removing the dir who has a plenty of whiteouts +- support copyup sparse file (a file which has a 'hole' in it) +- default permission flags for branches +- selectable permission flags for ro branch, whether whiteout can + exist or not +- export via NFS. +- support /fs/aufs and /aufs. +- support multiple writable branches, some policies to select one + among multiple writable branches. +- a new semantics for link(2) and rename(2) to support multiple + writable branches. +- no glibc changes are required. +- pseudo hardlink (hardlink over branches) +- allow a direct access manually to a file on branch, e.g. bypassing aufs. + including NFS or remote filesystem branch. +- userspace wrapper for pathconf(3)/fpathconf(3) with _PC_LINK_MAX. +- and more... + +Currently these features are dropped temporary from aufs4. +See design/08plan.txt in detail. +- nested mount, i.e. aufs as readonly no-whiteout branch of another aufs + (robr) +- statistics of aufs thread (/sys/fs/aufs/stat) + +Features or just an idea in the future (see also design/*.txt), +- reorder the branch index without del/re-add. +- permanent xino files for NFSD +- an option for refreshing the opened files after add/del branches +- light version, without branch manipulation. (unnecessary?) +- copyup in userspace +- inotify in userspace +- readv/writev + + +2. Download +---------------------------------------- +There are three GIT trees for aufs4, aufs4-linux.git, +aufs4-standalone.git, and aufs-util.git. Note that there is no "4" in +"aufs-util.git." +While the aufs-util is always necessary, you need either of aufs4-linux +or aufs4-standalone. + +The aufs4-linux tree includes the whole linux mainline GIT tree, +git://git.kernel.org/.../torvalds/linux.git. +And you cannot select CONFIG_AUFS_FS=m for this version, eg. you cannot +build aufs4 as an external kernel module. +Several extra patches are not included in this tree. Only +aufs4-standalone tree contains them. They are described in the later +section "Configuration and Compilation." + +On the other hand, the aufs4-standalone tree has only aufs source files +and necessary patches, and you can select CONFIG_AUFS_FS=m. +But you need to apply all aufs patches manually. + +You will find GIT branches whose name is in form of "aufs4.x" where "x" +represents the linux kernel version, "linux-4.x". For instance, +"aufs4.0" is for linux-4.0. For latest "linux-4.x-rcN", use +"aufs4.x-rcN" branch. + +o aufs4-linux tree +$ git clone --reference /your/linux/git/tree \ + git://github.com/sfjro/aufs4-linux.git aufs4-linux.git +- if you don't have linux GIT tree, then remove "--reference ..." +$ cd aufs4-linux.git +$ git checkout origin/aufs4.0 + +Or You may want to directly git-pull aufs into your linux GIT tree, and +leave the patch-work to GIT. +$ cd /your/linux/git/tree +$ git remote add aufs4 git://github.com/sfjro/aufs4-linux.git +$ git fetch aufs4 +$ git checkout -b my4.0 v4.0 +$ (add your local change...) +$ git pull aufs4 aufs4.0 +- now you have v4.0 + your_changes + aufs4.0 in you my4.0 branch. +- you may need to solve some conflicts between your_changes and + aufs4.0. in this case, git-rerere is recommended so that you can + solve the similar conflicts automatically when you upgrade to 4.1 or + later in the future. + +o aufs4-standalone tree +$ git clone git://github.com/sfjro/aufs4-standalone.git aufs4-standalone.git +$ cd aufs4-standalone.git +$ git checkout origin/aufs4.0 + +o aufs-util tree +$ git clone git://git.code.sf.net/p/aufs/aufs-util aufs-util.git +- note that the public aufs-util.git is on SourceForge instead of + GitHUB. +$ cd aufs-util.git +$ git checkout origin/aufs4.0 + +Note: The 4.x-rcN branch is to be used with `rc' kernel versions ONLY. +The minor version number, 'x' in '4.x', of aufs may not always +follow the minor version number of the kernel. +Because changes in the kernel that cause the use of a new +minor version number do not always require changes to aufs-util. + +Since aufs-util has its own minor version number, you may not be +able to find a GIT branch in aufs-util for your kernel's +exact minor version number. +In this case, you should git-checkout the branch for the +nearest lower number. + +For (an unreleased) example: +If you are using "linux-4.10" and the "aufs4.10" branch +does not exist in aufs-util repository, then "aufs4.9", "aufs4.8" +or something numerically smaller is the branch for your kernel. + +Also you can view all branches by + $ git branch -a + + +3. Configuration and Compilation +---------------------------------------- +Make sure you have git-checkout'ed the correct branch. + +For aufs4-linux tree, +- enable CONFIG_AUFS_FS. +- set other aufs configurations if necessary. + +For aufs4-standalone tree, +There are several ways to build. + +1. +- apply ./aufs4-kbuild.patch to your kernel source files. +- apply ./aufs4-base.patch too. +- apply ./aufs4-mmap.patch too. +- apply ./aufs4-standalone.patch too, if you have a plan to set + CONFIG_AUFS_FS=m. otherwise you don't need ./aufs4-standalone.patch. +- copy ./{Documentation,fs,include/uapi/linux/aufs_type.h} files to your + kernel source tree. Never copy $PWD/include/uapi/linux/Kbuild. +- enable CONFIG_AUFS_FS, you can select either + =m or =y. +- and build your kernel as usual. +- install the built kernel. + Note: Since linux-3.9, every filesystem module requires an alias + "fs-". You should make sure that "fs-aufs" is listed in your + modules.aliases file if you set CONFIG_AUFS_FS=m. +- install the header files too by "make headers_install" to the + directory where you specify. By default, it is $PWD/usr. + "make help" shows a brief note for headers_install. +- and reboot your system. + +2. +- module only (CONFIG_AUFS_FS=m). +- apply ./aufs4-base.patch to your kernel source files. +- apply ./aufs4-mmap.patch too. +- apply ./aufs4-standalone.patch too. +- build your kernel, don't forget "make headers_install", and reboot. +- edit ./config.mk and set other aufs configurations if necessary. + Note: You should read $PWD/fs/aufs/Kconfig carefully which describes + every aufs configurations. +- build the module by simple "make". + Note: Since linux-3.9, every filesystem module requires an alias + "fs-". You should make sure that "fs-aufs" is listed in your + modules.aliases file. +- you can specify ${KDIR} make variable which points to your kernel + source tree. +- install the files + + run "make install" to install the aufs module, or copy the built + $PWD/aufs.ko to /lib/modules/... and run depmod -a (or reboot simply). + + run "make install_headers" (instead of headers_install) to install + the modified aufs header file (you can specify DESTDIR which is + available in aufs standalone version's Makefile only), or copy + $PWD/usr/include/linux/aufs_type.h to /usr/include/linux or wherever + you like manually. By default, the target directory is $PWD/usr. +- no need to apply aufs4-kbuild.patch, nor copying source files to your + kernel source tree. + +Note: The header file aufs_type.h is necessary to build aufs-util + as well as "make headers_install" in the kernel source tree. + headers_install is subject to be forgotten, but it is essentially + necessary, not only for building aufs-util. + You may not meet problems without headers_install in some older + version though. + +And then, +- read README in aufs-util, build and install it +- note that your distribution may contain an obsoleted version of + aufs_type.h in /usr/include/linux or something. When you build aufs + utilities, make sure that your compiler refers the correct aufs header + file which is built by "make headers_install." +- if you want to use readdir(3) in userspace or pathconf(3) wrapper, + then run "make install_ulib" too. And refer to the aufs manual in + detail. + +There several other patches in aufs4-standalone.git. They are all +optional. When you meet some problems, they will help you. +- aufs4-loopback.patch + Supports a nested loopback mount in a branch-fs. This patch is + unnecessary until aufs produces a message like "you may want to try + another patch for loopback file". +- proc_mounts.patch + When there are many mountpoints and many mount(2)/umount(2) are + running, then /proc/mounts may not show the all mountpoints. This + patch makes /proc/mounts always show the full mountpoints list. + If you don't want to apply this patch and meet such problem, then you + need to increase the value of 'ProcMounts_Times' make-variable in + aufs-util.git as a second best solution. +- vfs-ino.patch + Modifies a system global kernel internal function get_next_ino() in + order to stop assigning 0 for an inode-number. Not directly related to + aufs, but recommended generally. +- tmpfs-idr.patch + Keeps the tmpfs inode number as the lowest value. Effective to reduce + the size of aufs XINO files for tmpfs branch. Also it prevents the + duplication of inode number, which is important for backup tools and + other utilities. When you find aufs XINO files for tmpfs branch + growing too much, try this patch. +- lockdep-debug.patch + Because aufs is not only an ordinary filesystem (callee of VFS), but + also a caller of VFS functions for branch filesystems, subclassing of + the internal locks for LOCKDEP is necessary. LOCKDEP is a debugging + feature of linux kernel. If you enable CONFIG_LOCKDEP, then you will + need to apply this debug patch to expand several constant values. + If don't know what LOCKDEP, then you don't have apply this patch. + + +4. Usage +---------------------------------------- +At first, make sure aufs-util are installed, and please read the aufs +manual, aufs.5 in aufs-util.git tree. +$ man -l aufs.5 + +And then, +$ mkdir /tmp/rw /tmp/aufs +# mount -t aufs -o br=/tmp/rw:${HOME} none /tmp/aufs + +Here is another example. The result is equivalent. +# mount -t aufs -o br=/tmp/rw=rw:${HOME}=ro none /tmp/aufs + Or +# mount -t aufs -o br:/tmp/rw none /tmp/aufs +# mount -o remount,append:${HOME} /tmp/aufs + +Then, you can see whole tree of your home dir through /tmp/aufs. If +you modify a file under /tmp/aufs, the one on your home directory is +not affected, instead the same named file will be newly created under +/tmp/rw. And all of your modification to a file will be applied to +the one under /tmp/rw. This is called the file based Copy on Write +(COW) method. +Aufs mount options are described in aufs.5. +If you run chroot or something and make your aufs as a root directory, +then you need to customize the shutdown script. See the aufs manual in +detail. + +Additionally, there are some sample usages of aufs which are a +diskless system with network booting, and LiveCD over NFS. +See sample dir in CVS tree on SourceForge. + + +5. Contact +---------------------------------------- +When you have any problems or strange behaviour in aufs, please let me +know with: +- /proc/mounts (instead of the output of mount(8)) +- /sys/module/aufs/* +- /sys/fs/aufs/* (if you have them) +- /debug/aufs/* (if you have them) +- linux kernel version + if your kernel is not plain, for example modified by distributor, + the url where i can download its source is necessary too. +- aufs version which was printed at loading the module or booting the + system, instead of the date you downloaded. +- configuration (define/undefine CONFIG_AUFS_xxx) +- kernel configuration or /proc/config.gz (if you have it) +- behaviour which you think to be incorrect +- actual operation, reproducible one is better +- mailto: aufs-users at lists.sourceforge.net + +Usually, I don't watch the Public Areas(Bugs, Support Requests, Patches, +and Feature Requests) on SourceForge. Please join and write to +aufs-users ML. + + +6. Acknowledgements +---------------------------------------- +Thanks to everyone who have tried and are using aufs, whoever +have reported a bug or any feedback. + +Especially donators: +Tomas Matejicek(slax.org) made a donation (much more than once). + Since Apr 2010, Tomas M (the author of Slax and Linux Live + scripts) is making "doubling" donations. + Unfortunately I cannot list all of the donators, but I really + appreciate. + It ends Aug 2010, but the ordinary donation URL is still available. + +Dai Itasaka made a donation (2007/8). +Chuck Smith made a donation (2008/4, 10 and 12). +Henk Schoneveld made a donation (2008/9). +Chih-Wei Huang, ASUS, CTC donated Eee PC 4G (2008/10). +Francois Dupoux made a donation (2008/11). +Bruno Cesar Ribas and Luis Carlos Erpen de Bona, C3SL serves public + aufs2 GIT tree (2009/2). +William Grant made a donation (2009/3). +Patrick Lane made a donation (2009/4). +The Mail Archive (mail-archive.com) made donations (2009/5). +Nippy Networks (Ed Wildgoose) made a donation (2009/7). +New Dream Network, LLC (www.dreamhost.com) made a donation (2009/11). +Pavel Pronskiy made a donation (2011/2). +Iridium and Inmarsat satellite phone retailer (www.mailasail.com), Nippy + Networks (Ed Wildgoose) made a donation for hardware (2011/3). +Max Lekomcev (DOM-TV project) made a donation (2011/7, 12, 2012/3, 6 and +11). +Sam Liddicott made a donation (2011/9). +Era Scarecrow made a donation (2013/4). +Bor Ratajc made a donation (2013/4). +Alessandro Gorreta made a donation (2013/4). +POIRETTE Marc made a donation (2013/4). +Alessandro Gorreta made a donation (2013/4). +lauri kasvandik made a donation (2013/5). +"pemasu from Finland" made a donation (2013/7). +The Parted Magic Project made a donation (2013/9 and 11). +Pavel Barta made a donation (2013/10). +Nikolay Pertsev made a donation (2014/5). +James B made a donation (2014/7 and 2015/7). +Stefano Di Biase made a donation (2014/8). +Daniel Epellei made a donation (2015/1). +OmegaPhil made a donation (2016/1, 2018/4). +Tomasz Szewczyk made a donation (2016/4). +James Burry made a donation (2016/12). +Carsten Rose made a donation (2018/9). +Porteus Kiosk made a donation (2018/10). + +Thank you very much. +Donations are always, including future donations, very important and +helpful for me to keep on developing aufs. + + +7. +---------------------------------------- +If you are an experienced user, no explanation is needed. Aufs is +just a linux filesystem. + + +Enjoy! + +# Local variables: ; +# mode: text; +# End: ; diff --git a/Documentation/filesystems/aufs/design/01intro.txt b/Documentation/filesystems/aufs/design/01intro.txt new file mode 100644 index 0000000000000..7b2a062788515 --- /dev/null +++ b/Documentation/filesystems/aufs/design/01intro.txt @@ -0,0 +1,171 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Introduction +---------------------------------------- + +aufs [ei ju: ef es] | /ey-yoo-ef-es/ | [a u f s] +1. abbrev. for "advanced multi-layered unification filesystem". +2. abbrev. for "another unionfs". +3. abbrev. for "auf das" in German which means "on the" in English. + Ex. "Butter aufs Brot"(G) means "butter onto bread"(E). + But "Filesystem aufs Filesystem" is hard to understand. +4. abbrev. for "African Urban Fashion Show". + +AUFS is a filesystem with features: +- multi layered stackable unification filesystem, the member directory + is called as a branch. +- branch permission and attribute, 'readonly', 'real-readonly', + 'readwrite', 'whiteout-able', 'link-able whiteout', etc. and their + combination. +- internal "file copy-on-write". +- logical deletion, whiteout. +- dynamic branch manipulation, adding, deleting and changing permission. +- allow bypassing aufs, user's direct branch access. +- external inode number translation table and bitmap which maintains the + persistent aufs inode number. +- seekable directory, including NFS readdir. +- file mapping, mmap and sharing pages. +- pseudo-link, hardlink over branches. +- loopback mounted filesystem as a branch. +- several policies to select one among multiple writable branches. +- revert a single systemcall when an error occurs in aufs. +- and more... + + +Multi Layered Stackable Unification Filesystem +---------------------------------------------------------------------- +Most people already knows what it is. +It is a filesystem which unifies several directories and provides a +merged single directory. When users access a file, the access will be +passed/re-directed/converted (sorry, I am not sure which English word is +correct) to the real file on the member filesystem. The member +filesystem is called 'lower filesystem' or 'branch' and has a mode +'readonly' and 'readwrite.' And the deletion for a file on the lower +readonly branch is handled by creating 'whiteout' on the upper writable +branch. + +On LKML, there have been discussions about UnionMount (Jan Blunck, +Bharata B Rao and Valerie Aurora) and Unionfs (Erez Zadok). They took +different approaches to implement the merged-view. +The former tries putting it into VFS, and the latter implements as a +separate filesystem. +(If I misunderstand about these implementations, please let me know and +I shall correct it. Because it is a long time ago when I read their +source files last time). + +UnionMount's approach will be able to small, but may be hard to share +branches between several UnionMount since the whiteout in it is +implemented in the inode on branch filesystem and always +shared. According to Bharata's post, readdir does not seems to be +finished yet. +There are several missing features known in this implementations such as +- for users, the inode number may change silently. eg. copy-up. +- link(2) may break by copy-up. +- read(2) may get an obsoleted filedata (fstat(2) too). +- fcntl(F_SETLK) may be broken by copy-up. +- unnecessary copy-up may happen, for example mmap(MAP_PRIVATE) after + open(O_RDWR). + +In linux-3.18, "overlay" filesystem (formerly known as "overlayfs") was +merged into mainline. This is another implementation of UnionMount as a +separated filesystem. All the limitations and known problems which +UnionMount are equally inherited to "overlay" filesystem. + +Unionfs has a longer history. When I started implementing a stackable +filesystem (Aug 2005), it already existed. It has virtual super_block, +inode, dentry and file objects and they have an array pointing lower +same kind objects. After contributing many patches for Unionfs, I +re-started my project AUFS (Jun 2006). + +In AUFS, the structure of filesystem resembles to Unionfs, but I +implemented my own ideas, approaches and enhancements and it became +totally different one. + +Comparing DM snapshot and fs based implementation +- the number of bytes to be copied between devices is much smaller. +- the type of filesystem must be one and only. +- the fs must be writable, no readonly fs, even for the lower original + device. so the compression fs will not be usable. but if we use + loopback mount, we may address this issue. + for instance, + mount /cdrom/squashfs.img /sq + losetup /sq/ext2.img + losetup /somewhere/cow + dmsetup "snapshot /dev/loop0 /dev/loop1 ..." +- it will be difficult (or needs more operations) to extract the + difference between the original device and COW. +- DM snapshot-merge may help a lot when users try merging. in the + fs-layer union, users will use rsync(1). + +You may want to read my old paper "Filesystems in LiveCD" +(http://aufs.sourceforge.net/aufs2/report/sq/sq.pdf). + + +Several characters/aspects/persona of aufs +---------------------------------------------------------------------- + +Aufs has several characters, aspects or persona. +1. a filesystem, callee of VFS helper +2. sub-VFS, caller of VFS helper for branches +3. a virtual filesystem which maintains persistent inode number +4. reader/writer of files on branches such like an application + +1. Callee of VFS Helper +As an ordinary linux filesystem, aufs is a callee of VFS. For instance, +unlink(2) from an application reaches sys_unlink() kernel function and +then vfs_unlink() is called. vfs_unlink() is one of VFS helper and it +calls filesystem specific unlink operation. Actually aufs implements the +unlink operation but it behaves like a redirector. + +2. Caller of VFS Helper for Branches +aufs_unlink() passes the unlink request to the branch filesystem as if +it were called from VFS. So the called unlink operation of the branch +filesystem acts as usual. As a caller of VFS helper, aufs should handle +every necessary pre/post operation for the branch filesystem. +- acquire the lock for the parent dir on a branch +- lookup in a branch +- revalidate dentry on a branch +- mnt_want_write() for a branch +- vfs_unlink() for a branch +- mnt_drop_write() for a branch +- release the lock on a branch + +3. Persistent Inode Number +One of the most important issue for a filesystem is to maintain inode +numbers. This is particularly important to support exporting a +filesystem via NFS. Aufs is a virtual filesystem which doesn't have a +backend block device for its own. But some storage is necessary to +keep and maintain the inode numbers. It may be a large space and may not +suit to keep in memory. Aufs rents some space from its first writable +branch filesystem (by default) and creates file(s) on it. These files +are created by aufs internally and removed soon (currently) keeping +opened. +Note: Because these files are removed, they are totally gone after + unmounting aufs. It means the inode numbers are not persistent + across unmount or reboot. I have a plan to make them really + persistent which will be important for aufs on NFS server. + +4. Read/Write Files Internally (copy-on-write) +Because a branch can be readonly, when you write a file on it, aufs will +"copy-up" it to the upper writable branch internally. And then write the +originally requested thing to the file. Generally kernel doesn't +open/read/write file actively. In aufs, even a single write may cause a +internal "file copy". This behaviour is very similar to cp(1) command. + +Some people may think it is better to pass such work to user space +helper, instead of doing in kernel space. Actually I am still thinking +about it. But currently I have implemented it in kernel space. diff --git a/Documentation/filesystems/aufs/design/02struct.txt b/Documentation/filesystems/aufs/design/02struct.txt new file mode 100644 index 0000000000000..59ca21ae17371 --- /dev/null +++ b/Documentation/filesystems/aufs/design/02struct.txt @@ -0,0 +1,258 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Basic Aufs Internal Structure + +Superblock/Inode/Dentry/File Objects +---------------------------------------------------------------------- +As like an ordinary filesystem, aufs has its own +superblock/inode/dentry/file objects. All these objects have a +dynamically allocated array and store the same kind of pointers to the +lower filesystem, branch. +For example, when you build a union with one readwrite branch and one +readonly, mounted /au, /rw and /ro respectively. +- /au = /rw + /ro +- /ro/fileA exists but /rw/fileA + +Aufs lookup operation finds /ro/fileA and gets dentry for that. These +pointers are stored in a aufs dentry. The array in aufs dentry will be, +- [0] = NULL (because /rw/fileA doesn't exist) +- [1] = /ro/fileA + +This style of an array is essentially same to the aufs +superblock/inode/dentry/file objects. + +Because aufs supports manipulating branches, ie. add/delete/change +branches dynamically, these objects has its own generation. When +branches are changed, the generation in aufs superblock is +incremented. And a generation in other object are compared when it is +accessed. When a generation in other objects are obsoleted, aufs +refreshes the internal array. + + +Superblock +---------------------------------------------------------------------- +Additionally aufs superblock has some data for policies to select one +among multiple writable branches, XIB files, pseudo-links and kobject. +See below in detail. +About the policies which supports copy-down a directory, see +wbr_policy.txt too. + + +Branch and XINO(External Inode Number Translation Table) +---------------------------------------------------------------------- +Every branch has its own xino (external inode number translation table) +file. The xino file is created and unlinked by aufs internally. When two +members of a union exist on the same filesystem, they share the single +xino file. +The struct of a xino file is simple, just a sequence of aufs inode +numbers which is indexed by the lower inode number. +In the above sample, assume the inode number of /ro/fileA is i111 and +aufs assigns the inode number i999 for fileA. Then aufs writes 999 as +4(8) bytes at 111 * 4(8) bytes offset in the xino file. + +When the inode numbers are not contiguous, the xino file will be sparse +which has a hole in it and doesn't consume as much disk space as it +might appear. If your branch filesystem consumes disk space for such +holes, then you should specify 'xino=' option at mounting aufs. + +Aufs has a mount option to free the disk blocks for such holes in XINO +files on tmpfs or ramdisk. But it is not so effective actually. If you +meet a problem of disk shortage due to XINO files, then you should try +"tmpfs-ino.patch" (and "vfs-ino.patch" too) in aufs4-standalone.git. +The patch localizes the assignment inumbers per tmpfs-mount and avoid +the holes in XINO files. + +Also a writable branch has three kinds of "whiteout bases". All these +are existed when the branch is joined to aufs, and their names are +whiteout-ed doubly, so that users will never see their names in aufs +hierarchy. +1. a regular file which will be hardlinked to all whiteouts. +2. a directory to store a pseudo-link. +3. a directory to store an "orphan"-ed file temporary. + +1. Whiteout Base + When you remove a file on a readonly branch, aufs handles it as a + logical deletion and creates a whiteout on the upper writable branch + as a hardlink of this file in order not to consume inode on the + writable branch. +2. Pseudo-link Dir + See below, Pseudo-link. +3. Step-Parent Dir + When "fileC" exists on the lower readonly branch only and it is + opened and removed with its parent dir, and then user writes + something into it, then aufs copies-up fileC to this + directory. Because there is no other dir to store fileC. After + creating a file under this dir, the file is unlinked. + +Because aufs supports manipulating branches, ie. add/delete/change +dynamically, a branch has its own id. When the branch order changes, +aufs finds the new index by searching the branch id. + + +Pseudo-link +---------------------------------------------------------------------- +Assume "fileA" exists on the lower readonly branch only and it is +hardlinked to "fileB" on the branch. When you write something to fileA, +aufs copies-up it to the upper writable branch. Additionally aufs +creates a hardlink under the Pseudo-link Directory of the writable +branch. The inode of a pseudo-link is kept in aufs super_block as a +simple list. If fileB is read after unlinking fileA, aufs returns +filedata from the pseudo-link instead of the lower readonly +branch. Because the pseudo-link is based upon the inode, to keep the +inode number by xino (see above) is essentially necessary. + +All the hardlinks under the Pseudo-link Directory of the writable branch +should be restored in a proper location later. Aufs provides a utility +to do this. The userspace helpers executed at remounting and unmounting +aufs by default. +During this utility is running, it puts aufs into the pseudo-link +maintenance mode. In this mode, only the process which began the +maintenance mode (and its child processes) is allowed to operate in +aufs. Some other processes which are not related to the pseudo-link will +be allowed to run too, but the rest have to return an error or wait +until the maintenance mode ends. If a process already acquires an inode +mutex (in VFS), it has to return an error. + + +XIB(external inode number bitmap) +---------------------------------------------------------------------- +Addition to the xino file per a branch, aufs has an external inode number +bitmap in a superblock object. It is also an internal file such like a +xino file. +It is a simple bitmap to mark whether the aufs inode number is in-use or +not. +To reduce the file I/O, aufs prepares a single memory page to cache xib. + +As well as XINO files, aufs has a feature to truncate/refresh XIB to +reduce the number of consumed disk blocks for these files. + + +Virtual or Vertical Dir, and Readdir in Userspace +---------------------------------------------------------------------- +In order to support multiple layers (branches), aufs readdir operation +constructs a virtual dir block on memory. For readdir, aufs calls +vfs_readdir() internally for each dir on branches, merges their entries +with eliminating the whiteout-ed ones, and sets it to file (dir) +object. So the file object has its entry list until it is closed. The +entry list will be updated when the file position is zero and becomes +obsoleted. This decision is made in aufs automatically. + +The dynamically allocated memory block for the name of entries has a +unit of 512 bytes (by default) and stores the names contiguously (no +padding). Another block for each entry is handled by kmem_cache too. +During building dir blocks, aufs creates hash list and judging whether +the entry is whiteouted by its upper branch or already listed. +The merged result is cached in the corresponding inode object and +maintained by a customizable life-time option. + +Some people may call it can be a security hole or invite DoS attack +since the opened and once readdir-ed dir (file object) holds its entry +list and becomes a pressure for system memory. But I'd say it is similar +to files under /proc or /sys. The virtual files in them also holds a +memory page (generally) while they are opened. When an idea to reduce +memory for them is introduced, it will be applied to aufs too. +For those who really hate this situation, I've developed readdir(3) +library which operates this merging in userspace. You just need to set +LD_PRELOAD environment variable, and aufs will not consume no memory in +kernel space for readdir(3). + + +Workqueue +---------------------------------------------------------------------- +Aufs sometimes requires privilege access to a branch. For instance, +in copy-up/down operation. When a user process is going to make changes +to a file which exists in the lower readonly branch only, and the mode +of one of ancestor directories may not be writable by a user +process. Here aufs copy-up the file with its ancestors and they may +require privilege to set its owner/group/mode/etc. +This is a typical case of a application character of aufs (see +Introduction). + +Aufs uses workqueue synchronously for this case. It creates its own +workqueue. The workqueue is a kernel thread and has privilege. Aufs +passes the request to call mkdir or write (for example), and wait for +its completion. This approach solves a problem of a signal handler +simply. +If aufs didn't adopt the workqueue and changed the privilege of the +process, then the process may receive the unexpected SIGXFSZ or other +signals. + +Also aufs uses the system global workqueue ("events" kernel thread) too +for asynchronous tasks, such like handling inotify/fsnotify, re-creating a +whiteout base and etc. This is unrelated to a privilege. +Most of aufs operation tries acquiring a rw_semaphore for aufs +superblock at the beginning, at the same time waits for the completion +of all queued asynchronous tasks. + + +Whiteout +---------------------------------------------------------------------- +The whiteout in aufs is very similar to Unionfs's. That is represented +by its filename. UnionMount takes an approach of a file mode, but I am +afraid several utilities (find(1) or something) will have to support it. + +Basically the whiteout represents "logical deletion" which stops aufs to +lookup further, but also it represents "dir is opaque" which also stop +further lookup. + +In aufs, rmdir(2) and rename(2) for dir uses whiteout alternatively. +In order to make several functions in a single systemcall to be +revertible, aufs adopts an approach to rename a directory to a temporary +unique whiteouted name. +For example, in rename(2) dir where the target dir already existed, aufs +renames the target dir to a temporary unique whiteouted name before the +actual rename on a branch, and then handles other actions (make it opaque, +update the attributes, etc). If an error happens in these actions, aufs +simply renames the whiteouted name back and returns an error. If all are +succeeded, aufs registers a function to remove the whiteouted unique +temporary name completely and asynchronously to the system global +workqueue. + + +Copy-up +---------------------------------------------------------------------- +It is a well-known feature or concept. +When user modifies a file on a readonly branch, aufs operate "copy-up" +internally and makes change to the new file on the upper writable branch. +When the trigger systemcall does not update the timestamps of the parent +dir, aufs reverts it after copy-up. + + +Move-down (aufs3.9 and later) +---------------------------------------------------------------------- +"Copy-up" is one of the essential feature in aufs. It copies a file from +the lower readonly branch to the upper writable branch when a user +changes something about the file. +"Move-down" is an opposite action of copy-up. Basically this action is +ran manually instead of automatically and internally. +For desgin and implementation, aufs has to consider these issues. +- whiteout for the file may exist on the lower branch. +- ancestor directories may not exist on the lower branch. +- diropq for the ancestor directories may exist on the upper branch. +- free space on the lower branch will reduce. +- another access to the file may happen during moving-down, including + UDBA (see "Revalidate Dentry and UDBA"). +- the file should not be hard-linked nor pseudo-linked. they should be + handled by auplink utility later. + +Sometimes users want to move-down a file from the upper writable branch +to the lower readonly or writable branch. For instance, +- the free space of the upper writable branch is going to run out. +- create a new intermediate branch between the upper and lower branch. +- etc. + +For this purpose, use "aumvdown" command in aufs-util.git. diff --git a/Documentation/filesystems/aufs/design/03atomic_open.txt b/Documentation/filesystems/aufs/design/03atomic_open.txt new file mode 100644 index 0000000000000..fe13f5ac9f78f --- /dev/null +++ b/Documentation/filesystems/aufs/design/03atomic_open.txt @@ -0,0 +1,85 @@ + +# Copyright (C) 2015-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Support for a branch who has its ->atomic_open() +---------------------------------------------------------------------- +The filesystems who implement its ->atomic_open() are not majority. For +example NFSv4 does, and aufs should call NFSv4 ->atomic_open, +particularly for open(O_CREAT|O_EXCL, 0400) case. Other than +->atomic_open(), NFSv4 returns an error for this open(2). While I am not +sure whether all filesystems who have ->atomic_open() behave like this, +but NFSv4 surely returns the error. + +In order to support ->atomic_open() for aufs, there are a few +approaches. + +A. Introduce aufs_atomic_open() + - calls one of VFS:do_last(), lookup_open() or atomic_open() for + branch fs. +B. Introduce aufs_atomic_open() calling create, open and chmod. this is + an aufs user Pip Cet's approach + - calls aufs_create(), VFS finish_open() and notify_change(). + - pass fake-mode to finish_open(), and then correct the mode by + notify_change(). +C. Extend aufs_open() to call branch fs's ->atomic_open() + - no aufs_atomic_open(). + - aufs_lookup() registers the TID to an aufs internal object. + - aufs_create() does nothing when the matching TID is registered, but + registers the mode. + - aufs_open() calls branch fs's ->atomic_open() when the matching + TID is registered. +D. Extend aufs_open() to re-try branch fs's ->open() with superuser's + credential + - no aufs_atomic_open(). + - aufs_create() registers the TID to an internal object. this info + represents "this process created this file just now." + - when aufs gets EACCES from branch fs's ->open(), then confirm the + registered TID and re-try open() with superuser's credential. + +Pros and cons for each approach. + +A. + - straightforward but highly depends upon VFS internal. + - the atomic behavaiour is kept. + - some of parameters such as nameidata are hard to reproduce for + branch fs. + - large overhead. +B. + - easy to implement. + - the atomic behavaiour is lost. +C. + - the atomic behavaiour is kept. + - dirty and tricky. + - VFS checks whether the file is created correctly after calling + ->create(), which means this approach doesn't work. +D. + - easy to implement. + - the atomic behavaiour is lost. + - to open a file with superuser's credential and give it to a user + process is a bad idea, since the file object keeps the credential + in it. It may affect LSM or something. This approach doesn't work + either. + +The approach A is ideal, but it hard to implement. So here is a +variation of A, which is to be implemented. + +A-1. Introduce aufs_atomic_open() + - calls branch fs ->atomic_open() if exists. otherwise calls + vfs_create() and finish_open(). + - the demerit is that the several checks after branch fs + ->atomic_open() are lost. in the ordinary case, the checks are + done by VFS:do_last(), lookup_open() and atomic_open(). some can + be implemented in aufs, but not all I am afraid. diff --git a/Documentation/filesystems/aufs/design/03lookup.txt b/Documentation/filesystems/aufs/design/03lookup.txt new file mode 100644 index 0000000000000..c1fc62dd10010 --- /dev/null +++ b/Documentation/filesystems/aufs/design/03lookup.txt @@ -0,0 +1,113 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Lookup in a Branch +---------------------------------------------------------------------- +Since aufs has a character of sub-VFS (see Introduction), it operates +lookup for branches as VFS does. It may be a heavy work. But almost all +lookup operation in aufs is the simplest case, ie. lookup only an entry +directly connected to its parent. Digging down the directory hierarchy +is unnecessary. VFS has a function lookup_one_len() for that use, and +aufs calls it. + +When a branch is a remote filesystem, aufs basically relies upon its +->d_revalidate(), also aufs forces the hardest revalidate tests for +them. +For d_revalidate, aufs implements three levels of revalidate tests. See +"Revalidate Dentry and UDBA" in detail. + + +Test Only the Highest One for the Directory Permission (dirperm1 option) +---------------------------------------------------------------------- +Let's try case study. +- aufs has two branches, upper readwrite and lower readonly. + /au = /rw + /ro +- "dirA" exists under /ro, but /rw. and its mode is 0700. +- user invoked "chmod a+rx /au/dirA" +- the internal copy-up is activated and "/rw/dirA" is created and its + permission bits are set to world readable. +- then "/au/dirA" becomes world readable? + +In this case, /ro/dirA is still 0700 since it exists in readonly branch, +or it may be a natively readonly filesystem. If aufs respects the lower +branch, it should not respond readdir request from other users. But user +allowed it by chmod. Should really aufs rejects showing the entries +under /ro/dirA? + +To be honest, I don't have a good solution for this case. So aufs +implements 'dirperm1' and 'nodirperm1' mount options, and leave it to +users. +When dirperm1 is specified, aufs checks only the highest one for the +directory permission, and shows the entries. Otherwise, as usual, checks +every dir existing on all branches and rejects the request. + +As a side effect, dirperm1 option improves the performance of aufs +because the number of permission check is reduced when the number of +branch is many. + + +Revalidate Dentry and UDBA (User's Direct Branch Access) +---------------------------------------------------------------------- +Generally VFS helpers re-validate a dentry as a part of lookup. +0. digging down the directory hierarchy. +1. lock the parent dir by its i_mutex. +2. lookup the final (child) entry. +3. revalidate it. +4. call the actual operation (create, unlink, etc.) +5. unlock the parent dir + +If the filesystem implements its ->d_revalidate() (step 3), then it is +called. Actually aufs implements it and checks the dentry on a branch is +still valid. +But it is not enough. Because aufs has to release the lock for the +parent dir on a branch at the end of ->lookup() (step 2) and +->d_revalidate() (step 3) while the i_mutex of the aufs dir is still +held by VFS. +If the file on a branch is changed directly, eg. bypassing aufs, after +aufs released the lock, then the subsequent operation may cause +something unpleasant result. + +This situation is a result of VFS architecture, ->lookup() and +->d_revalidate() is separated. But I never say it is wrong. It is a good +design from VFS's point of view. It is just not suitable for sub-VFS +character in aufs. + +Aufs supports such case by three level of revalidation which is +selectable by user. +1. Simple Revalidate + Addition to the native flow in VFS's, confirm the child-parent + relationship on the branch just after locking the parent dir on the + branch in the "actual operation" (step 4). When this validation + fails, aufs returns EBUSY. ->d_revalidate() (step 3) in aufs still + checks the validation of the dentry on branches. +2. Monitor Changes Internally by Inotify/Fsnotify + Addition to above, in the "actual operation" (step 4) aufs re-lookup + the dentry on the branch, and returns EBUSY if it finds different + dentry. + Additionally, aufs sets the inotify/fsnotify watch for every dir on branches + during it is in cache. When the event is notified, aufs registers a + function to kernel 'events' thread by schedule_work(). And the + function sets some special status to the cached aufs dentry and inode + private data. If they are not cached, then aufs has nothing to + do. When the same file is accessed through aufs (step 0-3) later, + aufs will detect the status and refresh all necessary data. + In this mode, aufs has to ignore the event which is fired by aufs + itself. +3. No Extra Validation + This is the simplest test and doesn't add any additional revalidation + test, and skip the revalidation in step 4. It is useful and improves + aufs performance when system surely hide the aufs branches from user, + by over-mounting something (or another method). diff --git a/Documentation/filesystems/aufs/design/04branch.txt b/Documentation/filesystems/aufs/design/04branch.txt new file mode 100644 index 0000000000000..6e49dbaa1ec00 --- /dev/null +++ b/Documentation/filesystems/aufs/design/04branch.txt @@ -0,0 +1,74 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Branch Manipulation + +Since aufs supports dynamic branch manipulation, ie. add/remove a branch +and changing its permission/attribute, there are a lot of works to do. + + +Add a Branch +---------------------------------------------------------------------- +o Confirm the adding dir exists outside of aufs, including loopback + mount, and its various attributes. +o Initialize the xino file and whiteout bases if necessary. + See struct.txt. + +o Check the owner/group/mode of the directory + When the owner/group/mode of the adding directory differs from the + existing branch, aufs issues a warning because it may impose a + security risk. + For example, when a upper writable branch has a world writable empty + top directory, a malicious user can create any files on the writable + branch directly, like copy-up and modify manually. If something like + /etc/{passwd,shadow} exists on the lower readonly branch but the upper + writable branch, and the writable branch is world-writable, then a + malicious guy may create /etc/passwd on the writable branch directly + and the infected file will be valid in aufs. + I am afraid it can be a security issue, but aufs can do nothing except + producing a warning. + + +Delete a Branch +---------------------------------------------------------------------- +o Confirm the deleting branch is not busy + To be general, there is one merit to adopt "remount" interface to + manipulate branches. It is to discard caches. At deleting a branch, + aufs checks the still cached (and connected) dentries and inodes. If + there are any, then they are all in-use. An inode without its + corresponding dentry can be alive alone (for example, inotify/fsnotify case). + + For the cached one, aufs checks whether the same named entry exists on + other branches. + If the cached one is a directory, because aufs provides a merged view + to users, as long as one dir is left on any branch aufs can show the + dir to users. In this case, the branch can be removed from aufs. + Otherwise aufs rejects deleting the branch. + + If any file on the deleting branch is opened by aufs, then aufs + rejects deleting. + + +Modify the Permission of a Branch +---------------------------------------------------------------------- +o Re-initialize or remove the xino file and whiteout bases if necessary. + See struct.txt. + +o rw --> ro: Confirm the modifying branch is not busy + Aufs rejects the request if any of these conditions are true. + - a file on the branch is mmap-ed. + - a regular file on the branch is opened for write and there is no + same named entry on the upper branch. diff --git a/Documentation/filesystems/aufs/design/05wbr_policy.txt b/Documentation/filesystems/aufs/design/05wbr_policy.txt new file mode 100644 index 0000000000000..ad2c4527f6802 --- /dev/null +++ b/Documentation/filesystems/aufs/design/05wbr_policy.txt @@ -0,0 +1,64 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Policies to Select One among Multiple Writable Branches +---------------------------------------------------------------------- +When the number of writable branch is more than one, aufs has to decide +the target branch for file creation or copy-up. By default, the highest +writable branch which has the parent (or ancestor) dir of the target +file is chosen (top-down-parent policy). +By user's request, aufs implements some other policies to select the +writable branch, for file creation several policies, round-robin, +most-free-space, and other policies. For copy-up, top-down-parent, +bottom-up-parent, bottom-up and others. + +As expected, the round-robin policy selects the branch in circular. When +you have two writable branches and creates 10 new files, 5 files will be +created for each branch. mkdir(2) systemcall is an exception. When you +create 10 new directories, all will be created on the same branch. +And the most-free-space policy selects the one which has most free +space among the writable branches. The amount of free space will be +checked by aufs internally, and users can specify its time interval. + +The policies for copy-up is more simple, +top-down-parent is equivalent to the same named on in create policy, +bottom-up-parent selects the writable branch where the parent dir +exists and the nearest upper one from the copyup-source, +bottom-up selects the nearest upper writable branch from the +copyup-source, regardless the existence of the parent dir. + +There are some rules or exceptions to apply these policies. +- If there is a readonly branch above the policy-selected branch and + the parent dir is marked as opaque (a variation of whiteout), or the + target (creating) file is whiteout-ed on the upper readonly branch, + then the result of the policy is ignored and the target file will be + created on the nearest upper writable branch than the readonly branch. +- If there is a writable branch above the policy-selected branch and + the parent dir is marked as opaque or the target file is whiteouted + on the branch, then the result of the policy is ignored and the target + file will be created on the highest one among the upper writable + branches who has diropq or whiteout. In case of whiteout, aufs removes + it as usual. +- link(2) and rename(2) systemcalls are exceptions in every policy. + They try selecting the branch where the source exists as possible + since copyup a large file will take long time. If it can't be, + ie. the branch where the source exists is readonly, then they will + follow the copyup policy. +- There is an exception for rename(2) when the target exists. + If the rename target exists, aufs compares the index of the branches + where the source and the target exists and selects the higher + one. If the selected branch is readonly, then aufs follows the + copyup policy. diff --git a/Documentation/filesystems/aufs/design/06dirren.dot b/Documentation/filesystems/aufs/design/06dirren.dot new file mode 100644 index 0000000000000..2d62bb6dd55fd --- /dev/null +++ b/Documentation/filesystems/aufs/design/06dirren.dot @@ -0,0 +1,31 @@ + +// to view this graph, run dot(1) command in GRAPHVIZ. + +digraph G { +node [shape=box]; +whinfo [label="detailed info file\n(lower_brid_root-hinum, h_inum, namelen, old name)"]; + +node [shape=oval]; + +aufs_rename -> whinfo [label="store/remove"]; + +node [shape=oval]; +inode_list [label="h_inum list in branch\ncache"]; + +node [shape=box]; +whinode [label="h_inum list file"]; + +node [shape=oval]; +brmgmt [label="br_add/del/mod/umount"]; + +brmgmt -> inode_list [label="create/remove"]; +brmgmt -> whinode [label="load/store"]; + +inode_list -> whinode [style=dashed,dir=both]; + +aufs_rename -> inode_list [label="add/del"]; + +aufs_lookup -> inode_list [label="search"]; + +aufs_lookup -> whinfo [label="load/remove"]; +} diff --git a/Documentation/filesystems/aufs/design/06dirren.txt b/Documentation/filesystems/aufs/design/06dirren.txt new file mode 100644 index 0000000000000..3b80f8659b3ed --- /dev/null +++ b/Documentation/filesystems/aufs/design/06dirren.txt @@ -0,0 +1,102 @@ + +# Copyright (C) 2017-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Special handling for renaming a directory (DIRREN) +---------------------------------------------------------------------- +First, let's assume we have a simple usecase. + +- /u = /rw + /ro +- /rw/dirA exists +- /ro/dirA and /ro/dirA/file exist too +- there is no dirB on both branches +- a user issues rename("dirA", "dirB") + +Now, what should aufs behave against this rename(2)? +There are a few possible cases. + +A. returns EROFS. + since dirA exists on a readonly branch which cannot be renamed. +B. returns EXDEV. + it is possible to copy-up dirA (only the dir itself), but the child + entries ("file" in this case) should not be. it must be a bad + approach to copy-up recursively. +C. returns a success. + even the branch /ro is readonly, aufs tries renaming it. Obviously it + is a violation of aufs' policy. +D. construct an extra information which indicates that /ro/dirA should + be handled as the name of dirB. + overlayfs has a similar feature called REDIRECT. + +Until now, aufs implements the case B only which returns EXDEV, and +expects the userspace application behaves like mv(1) which tries +issueing rename(2) recursively. + +A new aufs feature called DIRREN is introduced which implements the case +D. There are several "extra information" added. + +1. detailed info per renamed directory + path: /rw/dirB/$AUFS_WH_DR_INFO_PFX. +2. the inode-number list of directories on a branch + path: /rw/dirB/$AUFS_WH_DR_BRHINO + +The filename of "detailed info per directory" represents the lower +branch, and its format is +- a type of the branch id + one of these. + + uuid (not implemented yet) + + fsid + + dev +- the inode-number of the branch root dir + +And it contains these info in a single regular file. +- magic number +- branch's inode-number of the logically renamed dir +- the name of the before-renamed dir + +The "detailed info per directory" file is created in aufs rename(2), and +loaded in any lookup. +The info is considered in lookup for the matching case only. Here +"matching" means that the root of branch (in the info filename) is same +to the current looking-up branch. After looking-up the before-renamed +name, the inode-number is compared. And the matched dentry is used. + +The "inode-number list of directories" is a regular file which contains +simply the inode-numbers on the branch. The file is created or updated +in removing the branch, and loaded in adding the branch. Its lifetime is +equal to the branch. +The list is refered in lookup, and when the current target inode is +found in the list, the aufs tries loading the "detailed info per +directory" and get the changed and valid name of the dir. + +Theoretically these "extra informaiton" may be able to be put into XATTR +in the dir inode. But aufs doesn't choose this way because +1. XATTR may not be supported by the branch (or its configuration) +2. XATTR may have its size limit. +3. XATTR may be less easy to convert than a regular file, when the + format of the info is changed in the future. +At the same time, I agree that the regular file approach is much slower +than XATTR approach. So, in the future, aufs may take the XATTR or other +better approach. + +This DIRREN feature is enabled by aufs configuration, and is activated +by a new mount option. + +For the more complicated case, there is a work with UDBA option, which +is to dected the direct access to the branches (by-passing aufs) and to +maintain the cashes in aufs. Since a single cached aufs dentry may +contains two names, before- and after-rename, the name comparision in +UDBA handler may not work correctly. In this case, the behaviour will be +equivalen to udba=reval case. diff --git a/Documentation/filesystems/aufs/design/06fhsm.txt b/Documentation/filesystems/aufs/design/06fhsm.txt new file mode 100644 index 0000000000000..54d739bd758c3 --- /dev/null +++ b/Documentation/filesystems/aufs/design/06fhsm.txt @@ -0,0 +1,120 @@ + +# Copyright (C) 2011-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +File-based Hierarchical Storage Management (FHSM) +---------------------------------------------------------------------- +Hierarchical Storage Management (or HSM) is a well-known feature in the +storage world. Aufs provides this feature as file-based with multiple +writable branches, based upon the principle of "Colder, the Lower". +Here the word "colder" means that the less used files, and "lower" means +that the position in the order of the stacked branches vertically. +These multiple writable branches are prioritized, ie. the topmost one +should be the fastest drive and be used heavily. + +o Characters in aufs FHSM story +- aufs itself and a new branch attribute. +- a new ioctl interface to move-down and to establish a connection with + the daemon ("move-down" is a converse of "copy-up"). +- userspace tool and daemon. + +The userspace daemon establishes a connection with aufs and waits for +the notification. The notified information is very similar to struct +statfs containing the number of consumed blocks and inodes. +When the consumed blocks/inodes of a branch exceeds the user-specified +upper watermark, the daemon activates its move-down process until the +consumed blocks/inodes reaches the user-specified lower watermark. + +The actual move-down is done by aufs based upon the request from +user-space since we need to maintain the inode number and the internal +pointer arrays in aufs. + +Currently aufs FHSM handles the regular files only. Additionally they +must not be hard-linked nor pseudo-linked. + + +o Cowork of aufs and the user-space daemon + During the userspace daemon established the connection, aufs sends a + small notification to it whenever aufs writes something into the + writable branch. But it may cost high since aufs issues statfs(2) + internally. So user can specify a new option to cache the + info. Actually the notification is controlled by these factors. + + the specified cache time. + + classified as "force" by aufs internally. + Until the specified time expires, aufs doesn't send the info + except the forced cases. When aufs decide forcing, the info is always + notified to userspace. + For example, the number of free inodes is generally large enough and + the shortage of it happens rarely. So aufs doesn't force the + notification when creating a new file, directory and others. This is + the typical case which aufs doesn't force. + When aufs writes the actual filedata and the files consumes any of new + blocks, the aufs forces notifying. + + +o Interfaces in aufs +- New branch attribute. + + fhsm + Specifies that the branch is managed by FHSM feature. In other word, + participant in the FHSM. + When nofhsm is set to the branch, it will not be the source/target + branch of the move-down operation. This attribute is set + independently from coo and moo attributes, and if you want full + FHSM, you should specify them as well. +- New mount option. + + fhsm_sec + Specifies a second to suppress many less important info to be + notified. +- New ioctl. + + AUFS_CTL_FHSM_FD + create a new file descriptor which userspace can read the notification + (a subset of struct statfs) from aufs. +- Module parameter 'brs' + It has to be set to 1. Otherwise the new mount option 'fhsm' will not + be set. +- mount helpers /sbin/mount.aufs and /sbin/umount.aufs + When there are two or more branches with fhsm attributes, + /sbin/mount.aufs invokes the user-space daemon and /sbin/umount.aufs + terminates it. As a result of remounting and branch-manipulation, the + number of branches with fhsm attribute can be one. In this case, + /sbin/mount.aufs will terminate the user-space daemon. + + +Finally the operation is done as these steps in kernel-space. +- make sure that, + + no one else is using the file. + + the file is not hard-linked. + + the file is not pseudo-linked. + + the file is a regular file. + + the parent dir is not opaqued. +- find the target writable branch. +- make sure the file is not whiteout-ed by the upper (than the target) + branch. +- make the parent dir on the target branch. +- mutex lock the inode on the branch. +- unlink the whiteout on the target branch (if exists). +- lookup and create the whiteout-ed temporary name on the target branch. +- copy the file as the whiteout-ed temporary name on the target branch. +- rename the whiteout-ed temporary name to the original name. +- unlink the file on the source branch. +- maintain the internal pointer array and the external inode number + table (XINO). +- maintain the timestamps and other attributes of the parent dir and the + file. + +And of course, in every step, an error may happen. So the operation +should restore the original file state after an error happens. diff --git a/Documentation/filesystems/aufs/design/06mmap.txt b/Documentation/filesystems/aufs/design/06mmap.txt new file mode 100644 index 0000000000000..5f65d899a1c9a --- /dev/null +++ b/Documentation/filesystems/aufs/design/06mmap.txt @@ -0,0 +1,72 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +mmap(2) -- File Memory Mapping +---------------------------------------------------------------------- +In aufs, the file-mapped pages are handled by a branch fs directly, no +interaction with aufs. It means aufs_mmap() calls the branch fs's +->mmap(). +This approach is simple and good, but there is one problem. +Under /proc, several entries show the mmapped files by its path (with +device and inode number), and the printed path will be the path on the +branch fs's instead of virtual aufs's. +This is not a problem in most cases, but some utilities lsof(1) (and its +user) may expect the path on aufs. + +To address this issue, aufs adds a new member called vm_prfile in struct +vm_area_struct (and struct vm_region). The original vm_file points to +the file on the branch fs in order to handle everything correctly as +usual. The new vm_prfile points to a virtual file in aufs, and the +show-functions in procfs refers to vm_prfile if it is set. +Also we need to maintain several other places where touching vm_file +such like +- fork()/clone() copies vma and the reference count of vm_file is + incremented. +- merging vma maintains the ref count too. + +This is not a good approach. It just fakes the printed path. But it +leaves all behaviour around f_mapping unchanged. This is surely an +advantage. +Actually aufs had adopted another complicated approach which calls +generic_file_mmap() and handles struct vm_operations_struct. In this +approach, aufs met a hard problem and I could not solve it without +switching the approach. + +There may be one more another approach which is +- bind-mount the branch-root onto the aufs-root internally +- grab the new vfsmount (ie. struct mount) +- lazy-umount the branch-root internally +- in open(2) the aufs-file, open the branch-file with the hidden + vfsmount (instead of the original branch's vfsmount) +- ideally this "bind-mount and lazy-umount" should be done atomically, + but it may be possible from userspace by the mount helper. + +Adding the internal hidden vfsmount and using it in opening a file, the +file path under /proc will be printed correctly. This approach looks +smarter, but is not possible I am afraid. +- aufs-root may be bind-mount later. when it happens, another hidden + vfsmount will be required. +- it is hard to get the chance to bind-mount and lazy-umount + + in kernel-space, FS can have vfsmount in open(2) via + file->f_path, and aufs can know its vfsmount. But several locks are + already acquired, and if aufs tries to bind-mount and lazy-umount + here, then it may cause a deadlock. + + in user-space, bind-mount doesn't invoke the mount helper. +- since /proc shows dev and ino, aufs has to give vma these info. it + means a new member vm_prinode will be necessary. this is essentially + equivalent to vm_prfile described above. + +I have to give up this "looks-smater" approach. diff --git a/Documentation/filesystems/aufs/design/06xattr.txt b/Documentation/filesystems/aufs/design/06xattr.txt new file mode 100644 index 0000000000000..edd7553f32897 --- /dev/null +++ b/Documentation/filesystems/aufs/design/06xattr.txt @@ -0,0 +1,96 @@ + +# Copyright (C) 2014-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Listing XATTR/EA and getting the value +---------------------------------------------------------------------- +For the inode standard attributes (owner, group, timestamps, etc.), aufs +shows the values from the topmost existing file. This behaviour is good +for the non-dir entries since the bahaviour exactly matches the shown +information. But for the directories, aufs considers all the same named +entries on the lower branches. Which means, if one of the lower entry +rejects readdir call, then aufs returns an error even if the topmost +entry allows it. This behaviour is necessary to respect the branch fs's +security, but can make users confused since the user-visible standard +attributes don't match the behaviour. +To address this issue, aufs has a mount option called dirperm1 which +checks the permission for the topmost entry only, and ignores the lower +entry's permission. + +A similar issue can happen around XATTR. +getxattr(2) and listxattr(2) families behave as if dirperm1 option is +always set. Otherwise these very unpleasant situation would happen. +- listxattr(2) may return the duplicated entries. +- users may not be able to remove or reset the XATTR forever, + + +XATTR/EA support in the internal (copy,move)-(up,down) +---------------------------------------------------------------------- +Generally the extended attributes of inode are categorized as these. +- "security" for LSM and capability. +- "system" for posix ACL, 'acl' mount option is required for the branch + fs generally. +- "trusted" for userspace, CAP_SYS_ADMIN is required. +- "user" for userspace, 'user_xattr' mount option is required for the + branch fs generally. + +Moreover there are some other categories. Aufs handles these rather +unpopular categories as the ordinary ones, ie. there is no special +condition nor exception. + +In copy-up, the support for XATTR on the dst branch may differ from the +src branch. In this case, the copy-up operation will get an error and +the original user operation which triggered the copy-up will fail. It +can happen that even all copy-up will fail. +When both of src and dst branches support XATTR and if an error occurs +during copying XATTR, then the copy-up should fail obviously. That is a +good reason and aufs should return an error to userspace. But when only +the src branch support that XATTR, aufs should not return an error. +For example, the src branch supports ACL but the dst branch doesn't +because the dst branch may natively un-support it or temporary +un-support it due to "noacl" mount option. Of course, the dst branch fs +may NOT return an error even if the XATTR is not supported. It is +totally up to the branch fs. + +Anyway when the aufs internal copy-up gets an error from the dst branch +fs, then aufs tries removing the just copied entry and returns the error +to the userspace. The worst case of this situation will be all copy-up +will fail. + +For the copy-up operation, there two basic approaches. +- copy the specified XATTR only (by category above), and return the + error unconditionally if it happens. +- copy all XATTR, and ignore the error on the specified category only. + +In order to support XATTR and to implement the correct behaviour, aufs +chooses the latter approach and introduces some new branch attributes, +"icexsec", "icexsys", "icextr", "icexusr", and "icexoth". +They correspond to the XATTR namespaces (see above). Additionally, to be +convenient, "icex" is also provided which means all "icex*" attributes +are set (here the word "icex" stands for "ignore copy-error on XATTR"). + +The meaning of these attributes is to ignore the error from setting +XATTR on that branch. +Note that aufs tries copying all XATTR unconditionally, and ignores the +error from the dst branch according to the specified attributes. + +Some XATTR may have its default value. The default value may come from +the parent dir or the environment. If the default value is set at the +file creating-time, it will be overwritten by copy-up. +Some contradiction may happen I am afraid. +Do we need another attribute to stop copying XATTR? I am unsure. For +now, aufs implements the branch attributes to ignore the error. diff --git a/Documentation/filesystems/aufs/design/07export.txt b/Documentation/filesystems/aufs/design/07export.txt new file mode 100644 index 0000000000000..0ebf4efb07fbe --- /dev/null +++ b/Documentation/filesystems/aufs/design/07export.txt @@ -0,0 +1,58 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Export Aufs via NFS +---------------------------------------------------------------------- +Here is an approach. +- like xino/xib, add a new file 'xigen' which stores aufs inode + generation. +- iget_locked(): initialize aufs inode generation for a new inode, and + store it in xigen file. +- destroy_inode(): increment aufs inode generation and store it in xigen + file. it is necessary even if it is not unlinked, because any data of + inode may be changed by UDBA. +- encode_fh(): for a root dir, simply return FILEID_ROOT. otherwise + build file handle by + + branch id (4 bytes) + + superblock generation (4 bytes) + + inode number (4 or 8 bytes) + + parent dir inode number (4 or 8 bytes) + + inode generation (4 bytes)) + + return value of exportfs_encode_fh() for the parent on a branch (4 + bytes) + + file handle for a branch (by exportfs_encode_fh()) +- fh_to_dentry(): + + find the index of a branch from its id in handle, and check it is + still exist in aufs. + + 1st level: get the inode number from handle and search it in cache. + + 2nd level: if not found in cache, get the parent inode number from + the handle and search it in cache. and then open the found parent + dir, find the matching inode number by vfs_readdir() and get its + name, and call lookup_one_len() for the target dentry. + + 3rd level: if the parent dir is not cached, call + exportfs_decode_fh() for a branch and get the parent on a branch, + build a pathname of it, convert it a pathname in aufs, call + path_lookup(). now aufs gets a parent dir dentry, then handle it as + the 2nd level. + + to open the dir, aufs needs struct vfsmount. aufs keeps vfsmount + for every branch, but not itself. to get this, (currently) aufs + searches in current->nsproxy->mnt_ns list. it may not be a good + idea, but I didn't get other approach. + + test the generation of the gotten inode. +- every inode operation: they may get EBUSY due to UDBA. in this case, + convert it into ESTALE for NFSD. +- readdir(): call lockdep_on/off() because filldir in NFSD calls + lookup_one_len(), vfs_getattr(), encode_fh() and others. diff --git a/Documentation/filesystems/aufs/design/08shwh.txt b/Documentation/filesystems/aufs/design/08shwh.txt new file mode 100644 index 0000000000000..185ac7d4bc1a6 --- /dev/null +++ b/Documentation/filesystems/aufs/design/08shwh.txt @@ -0,0 +1,52 @@ + +# Copyright (C) 2005-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Show Whiteout Mode (shwh) +---------------------------------------------------------------------- +Generally aufs hides the name of whiteouts. But in some cases, to show +them is very useful for users. For instance, creating a new middle layer +(branch) by merging existing layers. + +(borrowing aufs1 HOW-TO from a user, Michael Towers) +When you have three branches, +- Bottom: 'system', squashfs (underlying base system), read-only +- Middle: 'mods', squashfs, read-only +- Top: 'overlay', ram (tmpfs), read-write + +The top layer is loaded at boot time and saved at shutdown, to preserve +the changes made to the system during the session. +When larger changes have been made, or smaller changes have accumulated, +the size of the saved top layer data grows. At this point, it would be +nice to be able to merge the two overlay branches ('mods' and 'overlay') +and rewrite the 'mods' squashfs, clearing the top layer and thus +restoring save and load speed. + +This merging is simplified by the use of another aufs mount, of just the +two overlay branches using the 'shwh' option. +# mount -t aufs -o ro,shwh,br:/livesys/overlay=ro+wh:/livesys/mods=rr+wh \ + aufs /livesys/merge_union + +A merged view of these two branches is then available at +/livesys/merge_union, and the new feature is that the whiteouts are +visible! +Note that in 'shwh' mode the aufs mount must be 'ro', which will disable +writing to all branches. Also the default mode for all branches is 'ro'. +It is now possible to save the combined contents of the two overlay +branches to a new squashfs, e.g.: +# mksquashfs /livesys/merge_union /path/to/newmods.squash + +This new squashfs archive can be stored on the boot device and the +initramfs will use it to replace the old one at the next boot. diff --git a/Documentation/filesystems/aufs/design/10dynop.txt b/Documentation/filesystems/aufs/design/10dynop.txt new file mode 100644 index 0000000000000..13e8583f2b916 --- /dev/null +++ b/Documentation/filesystems/aufs/design/10dynop.txt @@ -0,0 +1,47 @@ + +# Copyright (C) 2010-2019 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Dynamically customizable FS operations +---------------------------------------------------------------------- +Generally FS operations (struct inode_operations, struct +address_space_operations, struct file_operations, etc.) are defined as +"static const", but it never means that FS have only one set of +operation. Some FS have multiple sets of them. For instance, ext2 has +three sets, one for XIP, for NOBH, and for normal. +Since aufs overrides and redirects these operations, sometimes aufs has +to change its behaviour according to the branch FS type. More importantly +VFS acts differently if a function (member in the struct) is set or +not. It means aufs should have several sets of operations and select one +among them according to the branch FS definition. + +In order to solve this problem and not to affect the behaviour of VFS, +aufs defines these operations dynamically. For instance, aufs defines +dummy direct_IO function for struct address_space_operations, but it may +not be set to the address_space_operations actually. When the branch FS +doesn't have it, aufs doesn't set it to its address_space_operations +while the function definition itself is still alive. So the behaviour +itself will not change, and it will return an error when direct_IO is +not set. + +The lifetime of these dynamically generated operation object is +maintained by aufs branch object. When the branch is removed from aufs, +the reference counter of the object is decremented. When it reaches +zero, the dynamically generated operation object will be freed. + +This approach is designed to support AIO (io_submit), Direct I/O and +XIP (DAX) mainly. +Currently this approach is applied to address_space_operations for +regular files only. diff --git a/Documentation/misc-devices/bone_capemgr.txt b/Documentation/misc-devices/bone_capemgr.txt new file mode 100644 index 0000000000000..2a8c766b4898c --- /dev/null +++ b/Documentation/misc-devices/bone_capemgr.txt @@ -0,0 +1,63 @@ +--------------------------- + Beaglebone Cape-Manager +--------------------------- + +The beaglebone cape manager driver allows the automatic use of external +peripheral capes to be automatically supported by Linux without any manual +setup required by the user. + +Each beaglebone cape should contain an EEPROM that describes +it in a fixed I2C address on the i2c2 bus of the baseboard. +The format of the EEPROM is defined in the beaglebone reference +manual at: +http://beagleboard.org/static/beaglebone/latest/Docs/Hardware/BONE_SRM.pdf + +Reading the part number and revision information the manager +requests a firmware file formatted as a device tree overlay blob. + +Applying the overlay the devices are instantiated and the cape is +ready to be used. + +For instance if the part-number is BB-BONE-SERL-03 and the version is 00A1 +the firmware file requested will be BB-BONE-SERL-03-00A1-00A1.dtbo +It will be located by the in-kernel firmware +loader in the usual place, i.e. /lib/firmware/`uname -r`, /lib/firmware etc. + +The driver supports the following parameters (either as part of the kernel +command line or supplied at module insertion time). + +disable_partno: A comma delimited list of PART-NUMBER[:REV] of + disabled capes. +enable_partno: A comma delimited list of PART-NUMBER[:REV[:PRIO]] of + enabled capes. +boot_scan_period: The boot scan period in ms. When the cape manager is built-in + the kernel image, the firmware loader cannot find the files + before the rootfs is mounted. This parameter controls the + period with which the boot state is checked in that case. + +There's a sysfs control interface which is defined at the ABI documentation +area. + +Theory of operation: +-------------------- + +On driver probe the I2C EEPROM of the baseboard is read and information about +the current baseboard is retrieved. This information includes the mapping from +baseboard board name to DT friendly compatible string. I.e. the "A335BONE" board +name from EEPROM is mapped to the "ti,beaglebone" compatible string which should +be present in the dtbo to be loaded. + +Afterwards the EEPROMs declared in each slot are probed, and the EEPROMs found +are decoded keeping track the cape part-number and version data. + +Using the part-number and version a firmware file is requested (the firmware +file requested is -.dtbo). + +The dtbo is unflattend and the resulting device tree is matched against a +compatible baseboard, and in case of multiple parallel loading capes the +priorities defined are honored. That means that when there are multiple capes +being loaded in parallel the ones with the lowest priority number are loaded +first. + +Applying the device tree overlay makes the cape operational, as if it was part +of the kernel's booting device tree. diff --git a/Documentation/pwm.txt b/Documentation/pwm.txt index 8fbf0aa3ba2d5..72db8dd15f53b 100644 --- a/Documentation/pwm.txt +++ b/Documentation/pwm.txt @@ -85,9 +85,11 @@ will find: The PWM channels are numbered using a per-chip index from 0 to npwm-1. -When a PWM channel is exported a pwmX directory will be created in the +When a PWM channel is exported a pwm-N:X directory will be created in the pwmchipN directory it is associated with, where X is the number of the -channel that was exported. The following properties will then be available: +channel that was exported. It will also be exposed at /sys/class/pwm/ and +can be identified by the pwm_channel device type. +The following properties will then be available: period The total period of the PWM signal (read/write). diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000000..1e73bc717e366 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,11 @@ +pipeline { + agent any + + stages { + stage('Build') { + steps { + sh '/bin/bash ./jenkins_build.sh' + } + } + } +} diff --git a/MAINTAINERS b/MAINTAINERS index 64b40a60f5b71..8166168e2f20e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2474,6 +2474,19 @@ F: include/linux/audit.h F: include/uapi/linux/audit.h F: kernel/audit* +AUFS (advanced multi layered unification filesystem) FILESYSTEM +M: "J. R. Okajima" +L: aufs-users@lists.sourceforge.net (members only) +L: linux-unionfs@vger.kernel.org +W: http://aufs.sourceforge.net +T: git://github.com/sfjro/aufs4-linux.git +S: Supported +F: Documentation/filesystems/aufs/ +F: Documentation/ABI/testing/debugfs-aufs +F: Documentation/ABI/testing/sysfs-aufs +F: fs/aufs/ +F: include/uapi/linux/aufs_type.h + AUXILIARY DISPLAY DRIVERS M: Miguel Ojeda Sandonis W: http://miguelojeda.es/auxdisplay.htm @@ -2591,6 +2604,14 @@ L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/ec_bhf.c +BEAGLEBONE CAPEMANAGER +M: Pantelis Antoniou +S: Maintained +F: drivers/misc/beaglebone-capemgr.c +F: Documentation/misc-devices/bone_capemgr.txt +F: Documentation/devicetree/bindings/misc/bone_capemgr.txt +F: Documentation/ABI/testing/sysfs-devices-platform-bone_capemgr + BEFS FILE SYSTEM M: Luis de Bethencourt M: Salah Triki diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..a35e1f064aa79 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +| kernel | normal | rt | +|:---------:|:--------:|:-------:| +|4.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4-rt/) | +|4.9 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9-rt/) | +|4.14 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14-rt/) | +|4.19 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19-rt/) | +|5.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4-rt/) | diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index 50f8d1be7fcbe..2c30c4491679f 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -29,6 +29,10 @@ export ZRELADDR INITRD_PHYS PARAMS_PHYS targets := Image zImage xipImage bootpImage uImage +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + ifeq ($(CONFIG_XIP_KERNEL),y) $(obj)/xipImage: vmlinux FORCE diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 7949c213a4341..d9457e975144b 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1,12 +1,18 @@ # SPDX-License-Identifier: GPL-2.0 ifeq ($(CONFIG_OF),y) +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + dtb-$(CONFIG_ARCH_ALPINE) += \ alpine-db.dtb dtb-$(CONFIG_MACH_ARTPEC6) += \ artpec6-devboard.dtb + dtb-$(CONFIG_MACH_ASM9260) += \ alphascale-asm9260-devkit.dtb + # Keep at91 dtb files sorted alphabetically for each SoC dtb-$(CONFIG_SOC_AT91RM9200) += \ at91rm9200ek.dtb \ @@ -616,6 +622,26 @@ dtb-$(CONFIG_SOC_AM33XX) += \ am335x-base0033.dtb \ am335x-bone.dtb \ am335x-boneblack.dtb \ + am335x-sancloud-bbe-uboot-univ.dtb \ + am335x-sancloud-bbe-uboot.dtb \ + am335x-bonegreen-gateway.dtb \ + am335x-bonegreen-wireless-uboot-univ.dtb \ + am335x-boneblack-uboot-univ.dtb \ + am335x-bone-uboot-univ.dtb \ + am335x-boneblack-wireless-roboticscape.dtb \ + am335x-boneblack-roboticscape.dtb \ + am335x-pocketbeagle-techlab.dtb \ + am335x-pocketbeagle-gamepup.dtb \ + am335x-pocketbeagle.dtb \ + am335x-boneblack-audio.dtb \ + am335x-boneblack-bbb-exp-r.dtb \ + am335x-boneblack-bbb-exp-c.dtb \ + am335x-boneblack-bbbmini.dtb \ + am335x-boneblack-wl1835mod.dtb \ + am335x-olimex-som.dtb \ + am335x-abbbi.dtb \ + am335x-sancloud-bbe.dtb \ + am335x-boneblack-uboot.dtb \ am335x-boneblack-wireless.dtb \ am335x-boneblue.dtb \ am335x-bonegreen.dtb \ @@ -662,6 +688,11 @@ dtb-$(CONFIG_SOC_OMAP5) += \ omap5-uevm.dtb dtb-$(CONFIG_SOC_DRA7XX) += \ am57xx-beagle-x15.dtb \ + am5729-beagleboneai-roboticscape.dtb \ + am5729-beagleboneai.dtb \ + am57xx-beagle-x15-gssi.dtb \ + am57xx-evm-reva3.dtb \ + am57xx-evm.dtb \ am57xx-beagle-x15-revb1.dtb \ am57xx-beagle-x15-revc.dtb \ am57xx-cl-som-am57x.dtb \ @@ -697,9 +728,9 @@ dtb-merge-$(CONFIG_SOC_DRA7XX) += \ am57xx-evm-reva3-cam-ov10635.dtb \ dra71-evm-nand.dtb \ dra71-evm-lcd-auo-g101evn01.0.dtb -ifeq ($(CONFIG_SOC_DRA7XX),y) -dts-dirs += ti -endif +#ifeq ($(CONFIG_SOC_DRA7XX),y) +#dts-dirs += ti +#endif dtb-$(CONFIG_ARCH_ORION5X) += \ orion5x-kuroboxpro.dtb \ orion5x-lacie-d2-network.dtb \ @@ -1102,26 +1133,26 @@ dtb-$(CONFIG_ARCH_ASPEED) += aspeed-bmc-opp-palmetto.dtb \ aspeed-ast2500-evb.dtb endif -ifeq ($(BUILD_ITBS),y) -dtb-y= -dtb-merge-y= -else -dts-dirs= -endif +#ifeq ($(BUILD_ITBS),y) +#dtb-y= +#dtb-merge-y= +#else +#dts-dirs= +#endif dtstree := $(srctree)/$(src) dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) -always := $(dtb-y) $(dtb-merge-y) +always := $(dtb-y) clean-files := *.dtb -$(addprefix $(obj)/,$(dtb-merge-y)): TI_DTBOS - @$(srctree)/scripts/dtb-merge $(srctree) $(objtree) $@ $(objtree)/scripts/dtc/fdtoverlay $(src)/ti +#$(addprefix $(obj)/,$(dtb-merge-y)): TI_DTBOS +# @$(srctree)/scripts/dtb-merge $(srctree) $(objtree) $@ $(objtree)/scripts/dtc/fdtoverlay $(src)/ti -TI_DTBOS: - $(Q)$(MAKE) $(build)=$(src)/ti +#TI_DTBOS: +# $(Q)$(MAKE) $(build)=$(src)/ti -PHONY += TI_DTBOS +#PHONY += TI_DTBOS -subdir-y += $(dts-dirs) -subdir- += ti +#subdir-y += $(dts-dirs) +#subdir- += ti diff --git a/arch/arm/boot/dts/am335x-abbbi.dts b/arch/arm/boot/dts/am335x-abbbi.dts new file mode 100644 index 0000000000000..14d0174781738 --- /dev/null +++ b/arch/arm/boot/dts/am335x-abbbi.dts @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright 2015 Konsulko Group + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" + +/ { + model = "Arrow BeagleBone Black Industrial"; + compatible = "arrow,am335x-abbbi", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-abbbi.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&am33xx_pinmux { + adi_hdmi_bbbi_pins: adi_hdmi_bbbi_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + mcasp0_pins: mcasp0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + >; + }; +}; + +&lcdc { + status = "okay"; + port { + lcdc_0: endpoint@0 { + remote-endpoint = <&hdmi_0>; + }; + }; +}; + +&i2c0 { + adv7511w { + compatible = "adi,adv7511w"; + reg = <0x39>; + + pinctrl-names = "default"; + pinctrl-0 = <&adi_hdmi_bbbi_pins>; + + port { + hdmi_0: endpoint@0 { + remote-endpoint = <&lcdc_0>; + }; + }; + }; +}; + +&rtc { + system-power-controller; +}; + +&mcasp0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + status = "okay"; + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 0 0 1 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +/ { + clk_mcasp0_fixed: clk_mcasp0_fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24576000>; + }; + + clk_mcasp0: clk_mcasp0 { + #clock-cells = <0>; + compatible = "gpio-gate-clock"; + clocks = <&clk_mcasp0_fixed>; + enable-gpios = <&gpio1 27 0>; /* BeagleBone Black Clk enable on GPIO1_27 */ + }; + + hdmi_audio: hdmi_audio@0 { + compatible = "linux,hdmi-audio"; + status = "okay"; + }; + + sound { + compatible = "ti,beaglebone-black-audio"; + ti,model = "TI BeagleBone Black"; + ti,audio-codec = <&hdmi_audio>; + ti,mcasp-controller = <&mcasp0>; + ti,audio-routing = + "HDMI Out", "TX"; + clocks = <&clk_mcasp0>; + clock-names = "mclk"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi new file mode 100644 index 0000000000000..eb1c7119b149a --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +/ { + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x10000000>; /* 256 MB */ + }; + + chosen { + stdout-path = &uart0; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led2 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led4 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led5 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&am33xx_pinmux { + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ + >; + }; + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + mmc1_pins: pinmux_mmc1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + + status = "okay"; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; +}; + +&usb1_phy { + status = "okay"; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + interrupts-extended = <&intc 18 &tps 0>; + interrupt-names = "mc", "vbus"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + + status = "okay"; + clock-frequency = <100000>; +}; + + +/include/ "tps65217.dtsi" + +&tps { + /* + * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only + * mode") at poweroff. Most BeagleBone versions do not support RTC-only + * mode and risk hardware damage if this mode is entered. + * + * For details, see linux-omap mailing list May 2015 thread + * [PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller + * In particular, messages: + * http://www.spinics.net/lists/linux-omap/msg118585.html + * http://www.spinics.net/lists/linux-omap/msg118615.html + * + * You can override this later with + * &tps { /delete-property/ ti,pmic-shutdown-controller; } + * if you want to use RTC-only mode and made sure you are not affected + * by the hardware problems. (Tip: double-check by performing a current + * measurement after shutdown: it should be less than 1 mA.) + */ + + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "mii"; +}; + +&mac { + slaves = <1>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; +}; + +&mmc1 { + status = "okay"; + bus-width = <0x4>; + pinctrl-names = "default"; + pinctrl-0 = <&mmc1_pins>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + clocks = <&clk_32768_ck>, <&clkdiv32k_ick>; + clock-names = "ext-clk", "int-clk"; + system-power-controller; +}; + +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@0 { +// status = "okay"; +// }; +//}; diff --git a/arch/arm/boot/dts/am335x-bone-common-univ.dtsi b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi new file mode 100644 index 0000000000000..c0f5bf58cf340 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi @@ -0,0 +1,2961 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) gpio0_26 */ + P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) gpio0_27 */ + P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_pwm_pin: pinmux_P8_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio1_29 */ + P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio3_16 */ + P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_pwm_pin: pinmux_P9_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/1"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/2"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/1"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/2"; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/3"; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/4"; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/5"; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/0"; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/1"; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/0"; +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/1"; +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/2"; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) */ + P8_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_14_default_pin>; + pinctrl-1 = <&P8_14_gpio_pin>; + pinctrl-2 = <&P8_14_gpio_pu_pin>; + pinctrl-3 = <&P8_14_gpio_pd_pin>; + pinctrl-4 = <&P8_14_gpio_input_pin>; + pinctrl-5 = <&P8_14_pwm_pin>; + }; + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) */ + P8_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_17_default_pin>; + pinctrl-1 = <&P8_17_gpio_pin>; + pinctrl-2 = <&P8_17_gpio_pu_pin>; + pinctrl-3 = <&P8_17_gpio_pd_pin>; + pinctrl-4 = <&P8_17_gpio_input_pin>; + pinctrl-5 = <&P8_17_pwm_pin>; + }; + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) */ + P8_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_26_default_pin>; + pinctrl-1 = <&P8_26_gpio_pin>; + pinctrl-2 = <&P8_26_gpio_pu_pin>; + pinctrl-3 = <&P8_26_gpio_pd_pin>; + pinctrl-4 = <&P8_26_gpio_input_pin>; + }; + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_30_default_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_gpio_input_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pwm_pin>; + pinctrl-7 = <&P9_30_pruout_pin>; + pinctrl-8 = <&P9_30_pruin_pin>; + }; + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_14 { + gpio-name = "P8_14"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_17 { + gpio-name = "P8_17"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_26 { + gpio-name = "P8_26"; + gpio = <&gpio1 29 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_30 { + gpio-name = "P9_30"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi new file mode 100644 index 0000000000000..c87e3732db468 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi @@ -0,0 +1,967 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + /* P8_02 GND */ + /* P8_03 (ZCZ ball R9 ) emmc */ + /* P8_04 (ZCZ ball T9 ) emmc */ + /* P8_05 (ZCZ ball R8 ) emmc */ + /* P8_06 (ZCZ ball T8 ) emmc */ + + /* P8_07 (ZCZ ball R7 ) */ + P8_07_default_pin: pinmux_P8_07_default_pin { + pinctrl-single,pins = <0x090 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { + pinctrl-single,pins = <0x090 0x2F>; }; /* Mode 7, RxActive */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { + pinctrl-single,pins = <0x090 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { + pinctrl-single,pins = <0x090 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { + pinctrl-single,pins = <0x090 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_08 (ZCZ ball T7 ) */ + P8_08_default_pin: pinmux_P8_08_default_pin { + pinctrl-single,pins = <0x094 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { + pinctrl-single,pins = <0x094 0x2F>; }; /* Mode 7, RxActive */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { + pinctrl-single,pins = <0x094 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { + pinctrl-single,pins = <0x094 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { + pinctrl-single,pins = <0x094 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_09 (ZCZ ball T6 ) */ + P8_09_default_pin: pinmux_P8_09_default_pin { + pinctrl-single,pins = <0x09c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { + pinctrl-single,pins = <0x09c 0x2F>; }; /* Mode 7, RxActive */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { + pinctrl-single,pins = <0x09c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { + pinctrl-single,pins = <0x09c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { + pinctrl-single,pins = <0x09c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_10 (ZCZ ball U6 ) */ + P8_10_default_pin: pinmux_P8_10_default_pin { + pinctrl-single,pins = <0x098 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { + pinctrl-single,pins = <0x098 0x2F>; }; /* Mode 7, RxActive */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { + pinctrl-single,pins = <0x098 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { + pinctrl-single,pins = <0x098 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { + pinctrl-single,pins = <0x098 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_11 (ZCZ ball R12) */ + P8_11_default_pin: pinmux_P8_11_default_pin { + pinctrl-single,pins = <0x034 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { + pinctrl-single,pins = <0x034 0x2F>; }; /* Mode 7, RxActive */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { + pinctrl-single,pins = <0x034 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { + pinctrl-single,pins = <0x034 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { + pinctrl-single,pins = <0x034 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { + pinctrl-single,pins = <0x034 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_12 (ZCZ ball T12) */ + P8_12_default_pin: pinmux_P8_12_default_pin { + pinctrl-single,pins = <0x030 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { + pinctrl-single,pins = <0x030 0x2F>; }; /* Mode 7, RxActive */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { + pinctrl-single,pins = <0x030 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { + pinctrl-single,pins = <0x030 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { + pinctrl-single,pins = <0x030 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { + pinctrl-single,pins = <0x030 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_13 (ZCZ ball T10) */ + P8_13_default_pin: pinmux_P8_13_default_pin { + pinctrl-single,pins = <0x024 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { + pinctrl-single,pins = <0x024 0x2F>; }; /* Mode 7, RxActive */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { + pinctrl-single,pins = <0x024 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { + pinctrl-single,pins = <0x024 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { + pinctrl-single,pins = <0x024 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_14 (ZCZ ball T11) */ + P8_14_default_pin: pinmux_P8_14_default_pin { + pinctrl-single,pins = <0x028 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { + pinctrl-single,pins = <0x028 0x2F>; }; /* Mode 7, RxActive */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { + pinctrl-single,pins = <0x028 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { + pinctrl-single,pins = <0x028 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { + pinctrl-single,pins = <0x028 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_15 (ZCZ ball U13) */ + P8_15_default_pin: pinmux_P8_15_default_pin { + pinctrl-single,pins = <0x03c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { + pinctrl-single,pins = <0x03c 0x2F>; }; /* Mode 7, RxActive */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { + pinctrl-single,pins = <0x03c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { + pinctrl-single,pins = <0x03c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { + pinctrl-single,pins = <0x03c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { + pinctrl-single,pins = <0x03c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_15_pruin_pu_pin: pinmux_P8_15_pruin_pu_pin { + pinctrl-single,pins = <0x03c 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + P8_15_pruecapin_pu_pin: pinmux_P8_15_pruecapin_pu_pin { + pinctrl-single,pins = <0x03c 0x35>; }; /* Mode 5, Pull-Up, RxActive */ + + /* P8_16 (ZCZ ball V13) */ + P8_16_default_pin: pinmux_P8_16_default_pin { + pinctrl-single,pins = <0x038 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { + pinctrl-single,pins = <0x038 0x2F>; }; /* Mode 7, RxActive */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { + pinctrl-single,pins = <0x038 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { + pinctrl-single,pins = <0x038 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { + pinctrl-single,pins = <0x038 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { + pinctrl-single,pins = <0x038 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_17 (ZCZ ball U12) */ + P8_17_default_pin: pinmux_P8_17_default_pin { + pinctrl-single,pins = <0x02c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { + pinctrl-single,pins = <0x02c 0x2F>; }; /* Mode 7, RxActive */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { + pinctrl-single,pins = <0x02c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { + pinctrl-single,pins = <0x02c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_17_pwm_pin: pinmux_P8_17_pwm_pin { + pinctrl-single,pins = <0x02c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_18 (ZCZ ball V12) */ + P8_18_default_pin: pinmux_P8_18_default_pin { + pinctrl-single,pins = <0x08c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { + pinctrl-single,pins = <0x08c 0x2F>; }; /* Mode 7, RxActive */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { + pinctrl-single,pins = <0x08c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { + pinctrl-single,pins = <0x08c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P8_19 (ZCZ ball U10) */ + P8_19_default_pin: pinmux_P8_19_default_pin { + pinctrl-single,pins = <0x020 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { + pinctrl-single,pins = <0x020 0x2F>; }; /* Mode 7, RxActive */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { + pinctrl-single,pins = <0x020 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { + pinctrl-single,pins = <0x020 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { + pinctrl-single,pins = <0x020 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_20 (ZCZ ball V9 ) emmc */ + /* P8_21 (ZCZ ball U9 ) emmc */ + /* P8_22 (ZCZ ball V8 ) emmc */ + /* P8_23 (ZCZ ball U8 ) emmc */ + /* P8_24 (ZCZ ball V7 ) emmc */ + /* P8_25 (ZCZ ball U7 ) emmc */ + + /* P8_26 (ZCZ ball V6 ) */ + P8_26_default_pin: pinmux_P8_26_default_pin { + pinctrl-single,pins = <0x07c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { + pinctrl-single,pins = <0x07c 0x2F>; }; /* Mode 7, RxActive */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { + pinctrl-single,pins = <0x07c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { + pinctrl-single,pins = <0x07c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P8_27 (ZCZ ball U5 ) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { + pinctrl-single,pins = <0x0e0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { + pinctrl-single,pins = <0x0e0 0x2F>; }; /* Mode 7, RxActive */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { + pinctrl-single,pins = <0x0e0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { + pinctrl-single,pins = <0x0e0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { + pinctrl-single,pins = <0x0e0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { + pinctrl-single,pins = <0x0e0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_27_hdmi_pin: pinmux_P8_27_hdmi_pin { + pinctrl-single,pins = <0x0e0 0x00>; }; /* lcd_vsync.lcd_vsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_28 (ZCZ ball V5 ) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { + pinctrl-single,pins = <0x0e8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { + pinctrl-single,pins = <0x0e8 0x2F>; }; /* Mode 7, RxActive */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { + pinctrl-single,pins = <0x0e8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { + pinctrl-single,pins = <0x0e8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { + pinctrl-single,pins = <0x0e8 0x05>; }; /* Mode 5, Pull-Down */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { + pinctrl-single,pins = <0x0e8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_28_hdmi_pin: pinmux_P8_28_hdmi_pin { + pinctrl-single,pins = <0x0e8 0x00>; }; /* lcd_pclk.lcd_pclk, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_29 (ZCZ ball R5 ) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { + pinctrl-single,pins = <0x0e4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { + pinctrl-single,pins = <0x0e4 0x2F>; }; /* Mode 7, RxActive */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { + pinctrl-single,pins = <0x0e4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { + pinctrl-single,pins = <0x0e4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { + pinctrl-single,pins = <0x0e4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { + pinctrl-single,pins = <0x0e4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_29_hdmi_pin: pinmux_P8_29_hdmi_pin { + pinctrl-single,pins = <0x0e4 0x00>; }; /* lcd_hsync.lcd_hsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_30 (ZCZ ball R6 ) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { + pinctrl-single,pins = <0x0ec 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { + pinctrl-single,pins = <0x0ec 0x2F>; }; /* Mode 7, RxActive */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { + pinctrl-single,pins = <0x0ec 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { + pinctrl-single,pins = <0x0ec 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { + pinctrl-single,pins = <0x0ec 0x05>; }; /* Mode 5, Pull-Down*/ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { + pinctrl-single,pins = <0x0ec 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_30_hdmi_pin: pinmux_P8_30_hdmi_pin { + pinctrl-single,pins = <0x0ec 0x00>; }; /* lcd_ac_bias_en.lcd_ac_bias_en, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_31 (ZCZ ball V4 ) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { + pinctrl-single,pins = <0x0d8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { + pinctrl-single,pins = <0x0d8 0x2F>; }; /* Mode 7, RxActive */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { + pinctrl-single,pins = <0x0d8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { + pinctrl-single,pins = <0x0d8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { + pinctrl-single,pins = <0x0d8 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_31_hdmi_pin: pinmux_P8_31_hdmi_pin { + pinctrl-single,pins = <0x0d8 0x08>; }; /* lcd_data14.lcd_data14, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_32 (ZCZ ball T5 ) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { + pinctrl-single,pins = <0x0dc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { + pinctrl-single,pins = <0x0dc 0x2F>; }; /* Mode 7, RxActive */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { + pinctrl-single,pins = <0x0dc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { + pinctrl-single,pins = <0x0dc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_32_uart_pin: pinmux_P8_32_uart_pin { + pinctrl-single,pins = <0x0dc 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_32_hdmi_pin: pinmux_P8_32_hdmi_pin { + pinctrl-single,pins = <0x0dc 0x08>; }; /* lcd_data15.lcd_data15, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_33 (ZCZ ball V3 ) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { + pinctrl-single,pins = <0x0d4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { + pinctrl-single,pins = <0x0d4 0x2F>; }; /* Mode 7, RxActive */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { + pinctrl-single,pins = <0x0d4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { + pinctrl-single,pins = <0x0d4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_33_hdmi_pin: pinmux_P8_33_hdmi_pin { + pinctrl-single,pins = <0x0d4 0x08>; }; /* lcd_data13.lcd_data13, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { + pinctrl-single,pins = <0x0d4 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + + /* P8_34 (ZCZ ball U4 ) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { + pinctrl-single,pins = <0x0cc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { + pinctrl-single,pins = <0x0cc 0x2F>; }; /* Mode 7, RxActive */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { + pinctrl-single,pins = <0x0cc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { + pinctrl-single,pins = <0x0cc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { + pinctrl-single,pins = <0x0cc 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_34_hdmi_pin: pinmux_P8_34_hdmi_pin { + pinctrl-single,pins = <0x0cc 0x08>; }; /* lcd_data11.lcd_data11, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_35 (ZCZ ball V2 ) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { + pinctrl-single,pins = <0x0d0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { + pinctrl-single,pins = <0x0d0 0x2F>; }; /* Mode 7, RxActive */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { + pinctrl-single,pins = <0x0d0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { + pinctrl-single,pins = <0x0d0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_35_hdmi_pin: pinmux_P8_35_hdmi_pin { + pinctrl-single,pins = <0x0d0 0x08>; }; /* lcd_data12.lcd_data12, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { + pinctrl-single,pins = <0x0d0 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + + /* P8_36 (ZCZ ball U3 ) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { + pinctrl-single,pins = <0x0c8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { + pinctrl-single,pins = <0x0c8 0x2F>; }; /* Mode 7, RxActive */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { + pinctrl-single,pins = <0x0c8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { + pinctrl-single,pins = <0x0c8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { + pinctrl-single,pins = <0x0c8 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_36_hdmi_pin: pinmux_P8_36_hdmi_pin { + pinctrl-single,pins = <0x0c8 0x08>; }; /* lcd_data10.lcd_data10, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_37 (ZCZ ball U1 ) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { + pinctrl-single,pins = <0x0c0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { + pinctrl-single,pins = <0x0c0 0x2F>; }; /* Mode 7, RxActive */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { + pinctrl-single,pins = <0x0c0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { + pinctrl-single,pins = <0x0c0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { + pinctrl-single,pins = <0x0c0 0x04>; }; /* Mode 4, Pull-Down*/ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { + pinctrl-single,pins = <0x0c0 0x02>; }; /* Mode 2, Pull-Down*/ + P8_37_hdmi_pin: pinmux_P8_37_hdmi_pin { + pinctrl-single,pins = <0x0c0 0x08>; }; /* lcd_data8.lcd_data8, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + + /* P8_38 (ZCZ ball U2 ) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { + pinctrl-single,pins = <0x0c4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { + pinctrl-single,pins = <0x0c4 0x2F>; }; /* Mode 7, RxActive */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { + pinctrl-single,pins = <0x0c4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { + pinctrl-single,pins = <0x0c4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { + pinctrl-single,pins = <0x0c4 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { + pinctrl-single,pins = <0x0c4 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_38_hdmi_pin: pinmux_P8_38_hdmi_pin { + pinctrl-single,pins = <0x0c4 0x08>; }; /* lcd_data9.lcd_data9, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + + /* P8_39 (ZCZ ball T3 ) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { + pinctrl-single,pins = <0x0b8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { + pinctrl-single,pins = <0x0b8 0x2F>; }; /* Mode 7, RxActive */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { + pinctrl-single,pins = <0x0b8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { + pinctrl-single,pins = <0x0b8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { + pinctrl-single,pins = <0x0b8 0x05>; }; /* Mode 5, Pull-Down*/ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { + pinctrl-single,pins = <0x0b8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_39_hdmi_pin: pinmux_P8_39_hdmi_pin { + pinctrl-single,pins = <0x0b8 0x08>; }; /* lcd_data6.lcd_data6, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_40 (ZCZ ball T4 ) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { + pinctrl-single,pins = <0x0bc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { + pinctrl-single,pins = <0x0bc 0x2F>; }; /* Mode 7, RxActive */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { + pinctrl-single,pins = <0x0bc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { + pinctrl-single,pins = <0x0bc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { + pinctrl-single,pins = <0x0bc 0x05>; }; /* Mode 5, Pull-Down*/ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { + pinctrl-single,pins = <0x0bc 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_40_hdmi_pin: pinmux_P8_40_hdmi_pin { + pinctrl-single,pins = <0x0bc 0x08>; }; /* lcd_data7.lcd_data7, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_41 (ZCZ ball T1 ) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { + pinctrl-single,pins = <0x0b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { + pinctrl-single,pins = <0x0b0 0x2F>; }; /* Mode 7, RxActive */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { + pinctrl-single,pins = <0x0b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { + pinctrl-single,pins = <0x0b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { + pinctrl-single,pins = <0x0b0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { + pinctrl-single,pins = <0x0b0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_41_hdmi_pin: pinmux_P8_41_hdmi_pin { + pinctrl-single,pins = <0x0b0 0x08>; }; /* lcd_data4.lcd_data4, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_42 (ZCZ ball T2 ) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { + pinctrl-single,pins = <0x0b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { + pinctrl-single,pins = <0x0b4 0x2F>; }; /* Mode 7, RxActive */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { + pinctrl-single,pins = <0x0b4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { + pinctrl-single,pins = <0x0b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { + pinctrl-single,pins = <0x0b4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { + pinctrl-single,pins = <0x0b4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_42_hdmi_pin: pinmux_P8_42_hdmi_pin { + pinctrl-single,pins = <0x0b4 0x08>; }; /* lcd_data5.lcd_data5, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_43 (ZCZ ball R3 ) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { + pinctrl-single,pins = <0x0a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { + pinctrl-single,pins = <0x0a8 0x2F>; }; /* Mode 7, RxActive */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { + pinctrl-single,pins = <0x0a8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { + pinctrl-single,pins = <0x0a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { + pinctrl-single,pins = <0x0a8 0x05>; }; /* Mode 5, Pull-Down*/ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { + pinctrl-single,pins = <0x0a8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { + pinctrl-single,pins = <0x0a8 0x03>; }; /* Mode 3, Pull-Down */ + P8_43_hdmi_pin: pinmux_P8_43_hdmi_pin { + pinctrl-single,pins = <0x0a8 0x08>; }; /* lcd_data2.lcd_data2, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_44 (ZCZ ball R4 ) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { + pinctrl-single,pins = <0x0ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { + pinctrl-single,pins = <0x0ac 0x2F>; }; /* Mode 7, RxActive */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { + pinctrl-single,pins = <0x0ac 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { + pinctrl-single,pins = <0x0ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { + pinctrl-single,pins = <0x0ac 0x05>; }; /* Mode 5, Pull-Down*/ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { + pinctrl-single,pins = <0x0ac 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { + pinctrl-single,pins = <0x0ac 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P8_44_hdmi_pin: pinmux_P8_44_hdmi_pin { + pinctrl-single,pins = <0x0ac 0x08>; }; /* lcd_data3.lcd_data3, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_45 (ZCZ ball R1 ) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { + pinctrl-single,pins = <0x0a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { + pinctrl-single,pins = <0x0a0 0x2F>; }; /* Mode 7, RxActive */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { + pinctrl-single,pins = <0x0a0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { + pinctrl-single,pins = <0x0a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { + pinctrl-single,pins = <0x0a0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { + pinctrl-single,pins = <0x0a0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { + pinctrl-single,pins = <0x0a0 0x03>; }; /* Mode 3, Pull-Down*/ + P8_45_hdmi_pin: pinmux_P8_45_hdmi_pin { + pinctrl-single,pins = <0x0a0 0x08>; }; /* lcd_data0.lcd_data0, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_46 (ZCZ ball R2 ) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { + pinctrl-single,pins = <0x0a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { + pinctrl-single,pins = <0x0a4 0x2F>; }; /* Mode 7, RxActive */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { + pinctrl-single,pins = <0x0a4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { + pinctrl-single,pins = <0x0a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { + pinctrl-single,pins = <0x0a4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { + pinctrl-single,pins = <0x0a4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { + pinctrl-single,pins = <0x0a4 0x03>; }; /* Mode 3, Pull-Down*/ + P8_46_hdmi_pin: pinmux_P8_46_hdmi_pin { + pinctrl-single,pins = <0x0a4 0x08>; }; /* lcd_data1.lcd_data1, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + /* P9_02 GND */ + /* P9_03 3.3V */ + /* P9_04 3.3V */ + /* P9_05 VDD_5V */ + /* P9_06 VDD_5V */ + /* P9_07 SYS_5V */ + /* P9_08 SYS_5V */ + /* P9_09 PWR_BUT */ + /* P9_10 (ZCZ ball A10) RESETn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_default_pin: pinmux_P9_11_default_pin { + pinctrl-single,pins = <0x070 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { + pinctrl-single,pins = <0x070 0x2F>; }; /* Mode 7, RxActive */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { + pinctrl-single,pins = <0x070 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { + pinctrl-single,pins = <0x070 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { + pinctrl-single,pins = <0x070 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_12 (ZCZ ball U18) */ + P9_12_default_pin: pinmux_P9_12_default_pin { + pinctrl-single,pins = <0x078 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { + pinctrl-single,pins = <0x078 0x2F>; }; /* Mode 7, RxActive */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { + pinctrl-single,pins = <0x078 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { + pinctrl-single,pins = <0x078 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P9_13 (ZCZ ball U17) */ + P9_13_default_pin: pinmux_P9_13_default_pin { + pinctrl-single,pins = <0x074 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { + pinctrl-single,pins = <0x074 0x2F>; }; /* Mode 7, RxActive */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { + pinctrl-single,pins = <0x074 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { + pinctrl-single,pins = <0x074 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { + pinctrl-single,pins = <0x074 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_14 (ZCZ ball U14) */ + P9_14_default_pin: pinmux_P9_14_default_pin { + pinctrl-single,pins = <0x048 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { + pinctrl-single,pins = <0x048 0x2F>; }; /* Mode 7, RxActive */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { + pinctrl-single,pins = <0x048 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { + pinctrl-single,pins = <0x048 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { + pinctrl-single,pins = <0x048 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_15 (ZCZ ball R13) */ + P9_15_default_pin: pinmux_P9_15_default_pin { + pinctrl-single,pins = <0x040 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { + pinctrl-single,pins = <0x040 0x2F>; }; /* Mode 7, RxActive */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { + pinctrl-single,pins = <0x040 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { + pinctrl-single,pins = <0x040 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { + pinctrl-single,pins = <0x040 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_16 (ZCZ ball T14) */ + P9_16_default_pin: pinmux_P9_16_default_pin { + pinctrl-single,pins = <0x04c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { + pinctrl-single,pins = <0x04c 0x2F>; }; /* Mode 7, RxActive */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { + pinctrl-single,pins = <0x04c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { + pinctrl-single,pins = <0x04c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { + pinctrl-single,pins = <0x04c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_17 (ZCZ ball A16) */ + P9_17_default_pin: pinmux_P9_17_default_pin { + pinctrl-single,pins = <0x15c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { + pinctrl-single,pins = <0x15c 0x2F>; }; /* Mode 7, RxActive */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { + pinctrl-single,pins = <0x15c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { + pinctrl-single,pins = <0x15c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_17_spi_pin: pinmux_P9_17_spi_pin { + pinctrl-single,pins = <0x15c 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { + pinctrl-single,pins = <0x15c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { + pinctrl-single,pins = <0x15c 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_18 (ZCZ ball B16) */ + P9_18_default_pin: pinmux_P9_18_default_pin { + pinctrl-single,pins = <0x158 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { + pinctrl-single,pins = <0x158 0x2F>; }; /* Mode 7, RxActive */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { + pinctrl-single,pins = <0x158 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { + pinctrl-single,pins = <0x158 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { + pinctrl-single,pins = <0x158 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { + pinctrl-single,pins = <0x158 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { + pinctrl-single,pins = <0x158 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_19 (ZCZ ball D17) */ + P9_19_default_pin: pinmux_P9_19_default_pin { + pinctrl-single,pins = <0x17c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { + pinctrl-single,pins = <0x17c 0x2F>; }; /* Mode 7, RxActive */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { + pinctrl-single,pins = <0x17c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { + pinctrl-single,pins = <0x17c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_19_can_pin: pinmux_P9_19_can_pin { + pinctrl-single,pins = <0x17c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { + pinctrl-single,pins = <0x17c 0x73>; }; /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */ + + /* P9_20 (ZCZ ball D18) */ + P9_20_default_pin: pinmux_P9_20_default_pin { + pinctrl-single,pins = <0x178 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { + pinctrl-single,pins = <0x178 0x2F>; }; /* Mode 7, RxActive */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { + pinctrl-single,pins = <0x178 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { + pinctrl-single,pins = <0x178 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_20_can_pin: pinmux_P9_20_can_pin { + pinctrl-single,pins = <0x178 0x12>; }; /* Mode 2, Pull-Up, RxActive */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { + pinctrl-single,pins = <0x178 0x73>; }; /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */ + + /* P9_21 (ZCZ ball B17) */ + P9_21_default_pin: pinmux_P9_21_default_pin { + pinctrl-single,pins = <0x154 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { + pinctrl-single,pins = <0x154 0x2F>; }; /* Mode 7, RxActive */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { + pinctrl-single,pins = <0x154 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { + pinctrl-single,pins = <0x154 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { + pinctrl-single,pins = <0x154 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { + pinctrl-single,pins = <0x154 0x31>; }; /* Mode 1, Pull-Up, RxActive */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { + pinctrl-single,pins = <0x154 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { + pinctrl-single,pins = <0x154 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_22 (ZCZ ball A17) */ + P9_22_default_pin: pinmux_P9_22_default_pin { + pinctrl-single,pins = <0x150 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { + pinctrl-single,pins = <0x150 0x2F>; }; /* Mode 7, RxActive */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { + pinctrl-single,pins = <0x150 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { + pinctrl-single,pins = <0x150 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_22_spi_pin: pinmux_P9_22_spi_pin { + pinctrl-single,pins = <0x150 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { + pinctrl-single,pins = <0x150 0x31>; }; /* Mode 1, Pull-Up, RxActive */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { + pinctrl-single,pins = <0x150 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { + pinctrl-single,pins = <0x150 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_23 (ZCZ ball V14) */ + P9_23_default_pin: pinmux_P9_23_default_pin { + pinctrl-single,pins = <0x044 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { + pinctrl-single,pins = <0x044 0x2F>; }; /* Mode 7, RxActive */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { + pinctrl-single,pins = <0x044 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { + pinctrl-single,pins = <0x044 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { + pinctrl-single,pins = <0x044 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_24 (ZCZ ball D15) */ + P9_24_default_pin: pinmux_P9_24_default_pin { + pinctrl-single,pins = <0x184 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { + pinctrl-single,pins = <0x184 0x2F>; }; /* Mode 7, RxActive */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { + pinctrl-single,pins = <0x184 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { + pinctrl-single,pins = <0x184 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { + pinctrl-single,pins = <0x184 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_24_can_pin: pinmux_P9_24_can_pin { + pinctrl-single,pins = <0x184 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { + pinctrl-single,pins = <0x184 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { + pinctrl-single,pins = <0x184 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_25 (ZCZ ball A14) Audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { + pinctrl-single,pins = <0x1ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { + pinctrl-single,pins = <0x1ac 0x2F>; }; /* Mode 7, RxActive */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { + pinctrl-single,pins = <0x1ac 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { + pinctrl-single,pins = <0x1ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { + pinctrl-single,pins = <0x1ac 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { + pinctrl-single,pins = <0x1ac 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { + pinctrl-single,pins = <0x1ac 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_25_audio_pin: pinmux_P9_25_audio_pin { + pinctrl-single,pins = <0x1ac (PIN_INPUT_PULLUP | MUX_MODE0)>; }; /* mcasp0_ahclkx.mcasp0_ahclkx */ + + /* P9_26 (ZCZ ball D16) */ + P9_26_default_pin: pinmux_P9_26_default_pin { + pinctrl-single,pins = <0x180 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { + pinctrl-single,pins = <0x180 0x2F>; }; /* Mode 7, RxActive */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { + pinctrl-single,pins = <0x180 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { + pinctrl-single,pins = <0x180 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { + pinctrl-single,pins = <0x180 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_26_can_pin: pinmux_P9_26_can_pin { + pinctrl-single,pins = <0x180 0x12>; }; /* Mode 2, Pull-Up, RxActive */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { + pinctrl-single,pins = <0x180 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { + pinctrl-single,pins = <0x180 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_27 (ZCZ ball C13) */ + P9_27_default_pin: pinmux_P9_27_default_pin { + pinctrl-single,pins = <0x1a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { + pinctrl-single,pins = <0x1a4 0x2F>; }; /* Mode 7, RxActive */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { + pinctrl-single,pins = <0x1a4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { + pinctrl-single,pins = <0x1a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { + pinctrl-single,pins = <0x1a4 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { + pinctrl-single,pins = <0x1a4 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { + pinctrl-single,pins = <0x1a4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_28 (ZCZ ball C12) Audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { + pinctrl-single,pins = <0x19c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { + pinctrl-single,pins = <0x19c 0x2F>; }; /* Mode 7, RxActive */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { + pinctrl-single,pins = <0x19c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { + pinctrl-single,pins = <0x19c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { + pinctrl-single,pins = <0x19c 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_28_spi_pin: pinmux_P9_28_spi_pin { + pinctrl-single,pins = <0x19c 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { + pinctrl-single,pins = <0x19c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { + pinctrl-single,pins = <0x19c 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { + pinctrl-single,pins = <0x19c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_28_audio_pin: pinmux_P9_28_audio_pin { + pinctrl-single,pins = <0x19c (PIN_OUTPUT_PULLDOWN | MUX_MODE2)>; }; /* mcasp0_ahclkr.mcasp0_axr2 */ + + /* P9_29 (ZCZ ball B13) Audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { + pinctrl-single,pins = <0x194 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { + pinctrl-single,pins = <0x194 0x2F>; }; /* Mode 7, RxActive */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { + pinctrl-single,pins = <0x194 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { + pinctrl-single,pins = <0x194 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { + pinctrl-single,pins = <0x194 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { + pinctrl-single,pins = <0x194 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { + pinctrl-single,pins = <0x194 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { + pinctrl-single,pins = <0x194 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_29_audio_pin: pinmux_P9_29_audio_pin { + pinctrl-single,pins = <0x194 (PIN_OUTPUT_PULLUP | MUX_MODE0)>; }; /* mcasp0_fsx.mcasp0_fsx */ + + /* P9_30 (ZCZ ball D12) */ + P9_30_default_pin: pinmux_P9_30_default_pin { + pinctrl-single,pins = <0x198 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { + pinctrl-single,pins = <0x198 0x2F>; }; /* Mode 7, RxActive */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { + pinctrl-single,pins = <0x198 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { + pinctrl-single,pins = <0x198 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_30_pwm_pin: pinmux_P9_30_pwm_pin { + pinctrl-single,pins = <0x198 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { + pinctrl-single,pins = <0x198 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { + pinctrl-single,pins = <0x198 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { + pinctrl-single,pins = <0x198 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_31 (ZCZ ball A13) Audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { + pinctrl-single,pins = <0x190 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { + pinctrl-single,pins = <0x190 0x2F>; }; /* Mode 7, RxActive */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { + pinctrl-single,pins = <0x190 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { + pinctrl-single,pins = <0x190 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { + pinctrl-single,pins = <0x190 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_31_spi_pin: pinmux_P9_31_spi_pin { + pinctrl-single,pins = <0x190 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { + pinctrl-single,pins = <0x190 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { + pinctrl-single,pins = <0x190 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_31_audio_pin: pinmux_P9_31_audio_pin { + pinctrl-single,pins = <0x190 (PIN_OUTPUT_PULLDOWN | MUX_MODE0)>; }; /* mcasp0_aclkx.mcasp0_aclkx */ + + /* P9_32 VADC */ + /* P9_33 (ZCZ ball C8 ) AIN4 */ + /* P9_34 AGND */ + /* P9_35 (ZCZ ball A8 ) AIN6 */ + /* P9_36 (ZCZ ball B8 ) AIN5 */ + /* P9_37 (ZCZ ball B7 ) AIN2 */ + /* P9_38 (ZCZ ball A7 ) AIN3 */ + /* P9_39 (ZCZ ball B6 ) AIN0 */ + /* P9_40 (ZCZ ball C7 ) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_default_pin: pinmux_P9_41_default_pin { + pinctrl-single,pins = <0x1b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { + pinctrl-single,pins = <0x1b4 0x2F>; }; /* Mode 7, RxActive */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { + pinctrl-single,pins = <0x1b4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { + pinctrl-single,pins = <0x1b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { + pinctrl-single,pins = <0x1b4 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { + pinctrl-single,pins = <0x1b4 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_default_pin: pinmux_P9_91_default_pin { + pinctrl-single,pins = <0x1a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { + pinctrl-single,pins = <0x1a8 0x2F>; }; /* Mode 7, RxActive */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { + pinctrl-single,pins = <0x1a8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { + pinctrl-single,pins = <0x1a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { + pinctrl-single,pins = <0x1a8 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { + pinctrl-single,pins = <0x1a8 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { + pinctrl-single,pins = <0x1a8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_42 (ZCZ ball C18) */ + P9_42_default_pin: pinmux_P9_42_default_pin { + pinctrl-single,pins = <0x164 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { + pinctrl-single,pins = <0x164 0x2F>; }; /* Mode 7, RxActive */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { + pinctrl-single,pins = <0x164 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { + pinctrl-single,pins = <0x164 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { + pinctrl-single,pins = <0x164 0x20>; }; /* Mode 0, Pull-Down, RxActive */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { + pinctrl-single,pins = <0x164 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_42_spics_pin: pinmux_P9_42_spics_pin { + pinctrl-single,pins = <0x164 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P9_42_spiclk_pin: pinmux_P9_42_spiclk_pin { + pinctrl-single,pins = <0x164 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_default_pin: pinmux_P9_92_default_pin { + pinctrl-single,pins = <0x1a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { + pinctrl-single,pins = <0x1a0 0x2F>; }; /* Mode 7, RxActive */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { + pinctrl-single,pins = <0x1a0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { + pinctrl-single,pins = <0x1a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { + pinctrl-single,pins = <0x1a0 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { + pinctrl-single,pins = <0x1a0 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { + pinctrl-single,pins = <0x1a0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_43 GND */ + /* P9_44 GND */ + /* P9_45 GND */ + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi index e13aed0963289..0d0c0a56a058a 100644 --- a/arch/arm/boot/dts/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/am335x-bone-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ / { @@ -29,14 +26,14 @@ compatible = "gpio-leds"; led2 { - label = "beaglebone:green:heartbeat"; + label = "beaglebone:green:usr0"; gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; default-state = "off"; }; led3 { - label = "beaglebone:green:mmc0"; + label = "beaglebone:green:usr1"; gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; linux,default-trigger = "mmc0"; default-state = "off"; @@ -66,126 +63,160 @@ }; &am33xx_pinmux { - pinctrl-names = "default"; - pinctrl-0 = <&clkout2_pin>; - user_leds_s0: user_leds_s0 { pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ >; }; i2c0_pins: pinmux_i2c0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ - AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_scl.i2c0_scl */ >; }; - i2c2_pins: pinmux_i2c2_pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_ctsn.i2c2_sda */ - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_rtsn.i2c2_scl */ - >; - }; +// i2c2_pins: pinmux_i2c2_pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ +// AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ +// >; +// }; uart0_pins: pinmux_uart0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ - >; - }; - - clkout2_pin: pinmux_clkout2_pin { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b4, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr1.clkout2 */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; cpsw_default: cpsw_default { pinctrl-single,pins = < /* Slave 1 */ - AM33XX_IOPAD(0x910, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */ - AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */ - AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */ - AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */ - AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */ - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */ - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */ - AM33XX_IOPAD(0x930, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */ - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */ - AM33XX_IOPAD(0x938, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */ - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */ - AM33XX_IOPAD(0x940, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0) >; }; cpsw_sleep: cpsw_sleep { pinctrl-single,pins = < /* Slave 1 reset value */ - AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; davinci_mdio_default: davinci_mdio_default { pinctrl-single,pins = < /* MDIO */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) >; }; davinci_mdio_sleep: davinci_mdio_sleep { pinctrl-single,pins = < /* MDIO reset value */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; mmc1_pins: pinmux_mmc1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* spio0_cs1.gpio0_6 */ - AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat0.mmc0_dat0 */ - AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat1.mmc0_dat1 */ - AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat2.mmc0_dat2 */ - AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat3.mmc0_dat3 */ - AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_cmd.mmc0_cmd */ - AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_clk.mmc0_clk */ - AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4) /* mcasp0_aclkr.mmc0_sdwp */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) >; }; emmc_pins: pinmux_emmc_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ - AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ - AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ - AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ - AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ - AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ - AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ - AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ - AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ - AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ >; }; + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ }; &uart0 { @@ -252,7 +283,8 @@ &i2c2 { pinctrl-names = "default"; - pinctrl-0 = <&i2c2_pins>; + //pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; status = "okay"; clock-frequency = <100000>; @@ -425,10 +457,94 @@ ti,scale-data-fw = "am335x-bone-scale-data.bin"; }; -&pruss_soc_bus { - status = "okay"; +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@0 { +// status = "okay"; +// }; +//}; - pruss: pruss@0 { +/* the cape manager */ +/ { + bone_capemgr { + compatible = "ti,bone-capemgr"; status = "okay"; + + nvmem-cells = <&baseboard_data &cape0_data &cape1_data &cape2_data &cape3_data>; + nvmem-cell-names = "baseboard", "slot0", "slot1", "slot2", "slot3"; + #slots = <4>; + + /* map board revisions to compatible definitions */ + baseboardmaps { + baseboard_beaglebone: board@0 { + board-name = "A335BONE"; + compatible-name = "ti,beaglebone"; + }; + + baseboard_beaglebone_black: board@1 { + board-name = "A335BNLT"; + compatible-name = "ti,beaglebone-black"; + }; + }; + }; +}; + +&ocp { + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + }; }; diff --git a/arch/arm/boot/dts/am335x-bone-emmc-in-reset.dtsi b/arch/arm/boot/dts/am335x-bone-emmc-in-reset.dtsi new file mode 100644 index 0000000000000..7d8f673229c0e --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-emmc-in-reset.dtsi @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* standard */ + +&gpio1 { + emmc_rst { + gpio-hog; + gpios = <20 0>; + output-high; + line-name = "EMMC ResetN"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-jtag.dtsi b/arch/arm/boot/dts/am335x-bone-jtag.dtsi new file mode 100644 index 0000000000000..2cdca01417121 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-jtag.dtsi @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2015 Robert Nelson + */ + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&clkout2_pin>; + + clkout2_pin: pinmux_clkout2_pin { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* xdma_event_intr1.clkout2 */ + >; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi new file mode 100644 index 0000000000000..09612160b10d4 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-can0.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_19_pinmux { + * mode = "can"; + * }; + * P9_20_pinmux { + * mode = "can"; + * }; + *}; + * + *&dcan0 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + dcan0_pins: pinmux_dcan0_pins { + pinctrl-single,pins = < + /* P9_20: uart1_ctsn.d_can0_tx */ + BONE_P9_20 (PIN_OUTPUT_PULLUP | MUX_MODE2) + /* P9_19: uart1_rtsn.d_can0_rx */ + BONE_P9_19 (PIN_INPUT_PULLUP | MUX_MODE2) + >; + }; +}; + +&dcan0 { + pinctrl-0 = <&dcan0_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-can1.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-can1.dtsi new file mode 100644 index 0000000000000..9e264132cc46a --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-can1.dtsi @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-can1.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_24_pinmux { + * mode = "can"; + * }; + * P9_26_pinmux { + * mode = "can"; + * }; + *}; + * + *&dcan1 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + dcan1_pins: pinmux_dcan1_pins { + pinctrl-single,pins = < + /* P9_26: uart1_rxd.d_can1_tx */ + BONE_P9_26 (PIN_OUTPUT_PULLUP | MUX_MODE2) + /* P9_24: uart1_txd.d_can1_rx */ + BONE_P9_24 (PIN_INPUT_PULLUP | MUX_MODE2) + >; + }; +}; + +&dcan1 { + pinctrl-0 = <&dcan1_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-emmc.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-emmc.dtsi new file mode 100644 index 0000000000000..22cf4621b3572 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-emmc.dtsi @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* lsblk */ + +#include +#include "am335x-peripheral-emmc.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P8_21_pinmux { + * state = "disabled"; + * }; + * P8_20_pinmux { + * state = "disabled"; + * }; + * P8_25_pinmux { + * state = "disabled"; + * }; + * P8_24_pinmux { + * state = "disabled"; + * }; + * P8_05_pinmux { + * state = "disabled"; + * }; + * P8_06_pinmux { + * state = "disabled"; + * }; + * P8_23_pinmux { + * state = "disabled"; + * }; + * P8_22_pinmux { + * state = "disabled"; + * }; + * P8_03_pinmux { + * state = "disabled"; + * }; + * P8_04_pinmux { + * state = "disabled"; + * }; + *}; + * + *&mmc2 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + /* P8_21: gpmc_csn1.mmc1_clk */ + BONE_P8_21 (PIN_INPUT_PULLUP | MUX_MODE2) + /* P8_20: gpmc_csn2.mmc1_cmd */ + BONE_P8_20 (PIN_INPUT_PULLUP | MUX_MODE2) + /* P8_25: gpmc_ad0.mmc1_dat0 */ + BONE_P8_25 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_24: gpmc_ad1.mmc1_dat1 */ + BONE_P8_24 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_05: gpmc_ad2.mmc1_dat2 */ + BONE_P8_05 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_06: gpmc_ad3.mmc1_dat3 */ + BONE_P8_06 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_23: gpmc_ad4.mmc1_dat4 */ + BONE_P8_23 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_22: gpmc_ad5.mmc1_dat5 */ + BONE_P8_22 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_03: gpmc_ad6.mmc1_dat6 */ + BONE_P8_03 (PIN_INPUT_PULLUP | MUX_MODE1) + /* P8_04: gpmc_ad7.mmc1_dat7 */ + BONE_P8_04 (PIN_INPUT_PULLUP | MUX_MODE1) + >; + }; +}; + +&mmc2 { + pinctrl-0 = <&emmc_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-i2c2.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-i2c2.dtsi new file mode 100644 index 0000000000000..abf3b57522e3f --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-i2c2.dtsi @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-i2c2.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_19_pinmux { + * mode = "i2c"; + * }; + * P9_20_pinmux { + * mode = "i2c"; + * }; + *}; + * + *&dcan0 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + /* P9_20: uart1_ctsn.i2c2_sda */ + BONE_P9_20 (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) + /* P9_19: uart1_rtsn.i2c2_scl */ + BONE_P9_19 (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) + >; + }; +}; + +&i2c2 { + pinctrl-0 = <&i2c2_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-nxp-hdmi.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-nxp-hdmi.dtsi new file mode 100644 index 0000000000000..5205fa0f9a959 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-nxp-hdmi.dtsi @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "am335x-peripheral-nxp-hdmi.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P8_27_pinmux { + * state = "disabled"; + * }; + * P8_28_pinmux { + * state = "disabled"; + * }; + * P8_29_pinmux { + * state = "disabled"; + * }; + * P8_30_pinmux { + * state = "disabled"; + * }; + * P8_31_pinmux { + * state = "disabled"; + * }; + * P8_32_pinmux { + * state = "disabled"; + * }; + * P8_33_pinmux { + * state = "disabled"; + * }; + * P8_34_pinmux { + * state = "disabled"; + * }; + * P8_35_pinmux { + * state = "disabled"; + * }; + * P8_36_pinmux { + * state = "disabled"; + * }; + * P8_37_pinmux { + * state = "disabled"; + * }; + * P8_38_pinmux { + * state = "disabled"; + * }; + * P8_39_pinmux { + * state = "disabled"; + * }; + * P8_40_pinmux { + * state = "disabled"; + * }; + * P8_41_pinmux { + * state = "disabled"; + * }; + * P8_42_pinmux { + * state = "disabled"; + * }; + * P8_43_pinmux { + * state = "disabled"; + * }; + * P8_44_pinmux { + * state = "disabled"; + * }; + * P8_45_pinmux { + * state = "disabled"; + * }; + * P8_46_pinmux { + * state = "disabled"; + * }; + *}; + */ + +/* standard */ + +&am33xx_pinmux { + nxp_hdmi_pins: pinmux_nxp_hdmi_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ + AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0) /* lcd_data0.lcd_data0 */ + AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0) /* lcd_data1.lcd_data1 */ + AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0) /* lcd_data2.lcd_data2 */ + AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0) /* lcd_data3.lcd_data3 */ + AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0) /* lcd_data4.lcd_data4 */ + AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0) /* lcd_data5.lcd_data5 */ + AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0) /* lcd_data6.lcd_data6 */ + AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0) /* lcd_data7.lcd_data7 */ + AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0) /* lcd_data8.lcd_data8 */ + AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0) /* lcd_data9.lcd_data9 */ + AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0) /* lcd_data10.lcd_data10 */ + AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0) /* lcd_data11.lcd_data11 */ + AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0) /* lcd_data12.lcd_data12 */ + AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0) /* lcd_data13.lcd_data13 */ + AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0) /* lcd_data14.lcd_data14 */ + AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0) /* lcd_data15.lcd_data15 */ + AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_vsync.lcd_vsync */ + AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_hsync.lcd_hsync */ + AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_pclk.lcd_pclk */ + AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_ac_bias_en.lcd_ac_bias_en */ + >; + }; + + nxp_hdmi_off_pins: nxp_hdmi_off_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ + >; + }; +}; + +&i2c0 { + tda19988 { + pinctrl-names = "default", "off"; + pinctrl-0 = <&nxp_hdmi_bonelt_pins>; + pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi new file mode 100644 index 0000000000000..65e5fbb0cfcc1 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P8_27_pinmux { + * state = "disabled"; + * }; + * P8_28_pinmux { + * state = "disabled"; + * }; + * P8_29_pinmux { + * state = "disabled"; + * }; + * P8_30_pinmux { + * state = "disabled"; + * }; + * P8_31_pinmux { + * state = "disabled"; + * }; + * P8_32_pinmux { + * state = "disabled"; + * }; + * P8_33_pinmux { + * state = "disabled"; + * }; + * P8_34_pinmux { + * state = "disabled"; + * }; + * P8_35_pinmux { + * state = "disabled"; + * }; + * P8_36_pinmux { + * state = "disabled"; + * }; + * P8_37_pinmux { + * state = "disabled"; + * }; + * P8_38_pinmux { + * state = "disabled"; + * }; + * P8_39_pinmux { + * state = "disabled"; + * }; + * P8_40_pinmux { + * state = "disabled"; + * }; + * P8_41_pinmux { + * state = "disabled"; + * }; + * P8_42_pinmux { + * state = "disabled"; + * }; + * P8_43_pinmux { + * state = "disabled"; + * }; + * P8_44_pinmux { + * state = "disabled"; + * }; + * P8_45_pinmux { + * state = "disabled"; + * }; + * P8_46_pinmux { + * state = "disabled"; + * }; + *}; + */ + +/* standard */ + +&am33xx_pinmux { + lcd_24bit_pins: pinmux_lcd_24bit_pins { + pinctrl-single,pins = < + + /* P8_45: lcd_data0.lcd_data0 */ + BONE_P8_45 (PIN_OUTPUT | MUX_MODE0) + /* P8_46: lcd_data1.lcd_data1 */ + BONE_P8_46 (PIN_OUTPUT | MUX_MODE0) + /* P8_43: lcd_data2.lcd_data2 */ + BONE_P8_43 (PIN_OUTPUT | MUX_MODE0) + /* P8_44: lcd_data3.lcd_data3 */ + BONE_P8_44 (PIN_OUTPUT | MUX_MODE0) + /* P8_41: lcd_data4.lcd_data4 */ + BONE_P8_41 (PIN_OUTPUT | MUX_MODE0) + /* P8_42: lcd_data5.lcd_data5 */ + BONE_P8_42 (PIN_OUTPUT | MUX_MODE0) + /* P8_39: lcd_data6.lcd_data6 */ + BONE_P8_39 (PIN_OUTPUT | MUX_MODE0) + /* P8_40: lcd_data7.lcd_data7 */ + BONE_P8_40 (PIN_OUTPUT | MUX_MODE0) + /* P8_37: lcd_data8.lcd_data8 */ + BONE_P8_37 (PIN_OUTPUT | MUX_MODE0) + /* P8_38: lcd_data9.lcd_data9 */ + BONE_P8_38 (PIN_OUTPUT | MUX_MODE0) + /* P8_36: lcd_data10.lcd_data10 */ + BONE_P8_36 (PIN_OUTPUT | MUX_MODE0) + /* P8_34: lcd_data11.lcd_data11 */ + BONE_P8_34 (PIN_OUTPUT | MUX_MODE0) + /* P8_35: lcd_data12.lcd_data12 */ + BONE_P8_35 (PIN_OUTPUT | MUX_MODE0) + /* P8_33: lcd_data13.lcd_data13 */ + BONE_P8_33 (PIN_OUTPUT | MUX_MODE0) + /* P8_31: lcd_data14.lcd_data14 */ + BONE_P8_31 (PIN_OUTPUT | MUX_MODE0) + /* P8_32: lcd_data15.lcd_data15 */ + BONE_P8_32 (PIN_OUTPUT | MUX_MODE0) + + /* gpmc_ad15.lcd_data16 */ + BONE_P8_15 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad14.lcd_data17 */ + BONE_P8_16 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad13.lcd_data18 */ + BONE_P8_11 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad12.lcd_data19 */ + BONE_P8_12 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad11.lcd_data20 */ + BONE_P8_17 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad10.lcd_data21 */ + BONE_P8_14 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad9.lcd_data22 */ + BONE_P8_13 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad8.lcd_data23 */ + BONE_P8_19 (PIN_OUTPUT | MUX_MODE1) + + /* P8_27: lcd_vsync.lcd_vsync */ + BONE_P8_27 (PIN_OUTPUT | MUX_MODE0) + /* P8_29: lcd_hsync.lcd_hsync */ + BONE_P8_29 (PIN_OUTPUT | MUX_MODE0) + /* P8_28: lcd_pclk.lcd_pclk*/ + BONE_P8_28 (PIN_OUTPUT | MUX_MODE0) + /* P8_30: lcd_ac_bias_en.lcd_ac_bias_en */ + BONE_P8_30 (PIN_OUTPUT | MUX_MODE0) + >; + }; +}; + +/ { + panel { + pinctrl-0 = <&lcd_24bit_pins>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-spi0.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-spi0.dtsi new file mode 100644 index 0000000000000..354e66ad4f044 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-spi0.dtsi @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-spi0.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_17_pinmux { + * status = "disabled"; + * }; + * P9_18_pinmux { + * status = "disabled"; + * }; + * P9_21_pinmux { + * status = "disabled"; + * }; + * P9_22_pinmux { + * status = "disabled"; + * }; + *}; + * + *&spi0 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + spi0_pins: pinmux_spi0_pins { + pinctrl-single,pins = < + 0x150 (PIN_INPUT_PULLUP | MUX_MODE0) /* spi0_sclk.spi0_sclk */ + 0x154 (PIN_INPUT_PULLUP | MUX_MODE0) /* spi0_d0.spi0_d0 */ + 0x158 (PIN_OUTPUT_PULLUP | MUX_MODE0) /* spi0_d1.spi0_d1 */ + 0x15c (PIN_OUTPUT_PULLUP | MUX_MODE0) /* spi0_cs0.spi0_cs0 */ + >; + }; +}; + +&spi0 { + pinctrl-0 = <&spi0_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-spi1.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-spi1.dtsi new file mode 100644 index 0000000000000..bff7f8dae8399 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-spi1.dtsi @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-spi1.dtsi" + +/* standard */ + +&am33xx_pinmux { + spi1_pins: pinmux_spi1_pins { + pinctrl-single,pins = < + 0x190 0x33 /* mcasp0_aclkx.spi1_sclk, INPUT_PULLUP | MODE3 */ + 0x194 0x33 /* mcasp0_fsx.spi1_d0, INPUT_PULLUP | MODE3 */ + 0x198 0x13 /* mcasp0_axr0.spi1_d1, OUTPUT_PULLUP | MODE3 */ + 0x19c 0x13 /* mcasp0_ahclkr.spi1_cs0, OUTPUT_PULLUP | MODE3 */ + // 0x164 0x12 /* eCAP0_in_PWM0_out.spi1_cs1 OUTPUT_PULLUP | MODE2 */ >; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-spi1a.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-spi1a.dtsi new file mode 100644 index 0000000000000..62874c892ff78 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-spi1a.dtsi @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-spi1.dtsi" + +/* standard */ + +&am33xx_pinmux { + spi1a_pins: pinmux_spi1a_pins { + pinctrl-single,pins = < + 0x164 0x34 /* eCAP0_in_PWM0_out.spi1_sclk, INPUT_PULLUP | MODE4 */ + /* NOTE: P9.42 is connected to two pads */ + // 0x1A0 0x27 /* set the other pad to gpio input */ + 0x194 0x33 /* mcasp0_fsx.spi1_d0, INPUT_PULLUP | MODE3 */ + 0x198 0x13 /* mcasp0_axr0.spi1_d1, OUTPUT_PULLUP | MODE3 */ + 0x178 0x14 /* uart1_ctsn.spi1_cs0, OUTPUT_PULLUP | MODE4 */ >; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1a_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi new file mode 100644 index 0000000000000..ae5b813f007ab --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS1 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS1.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_24_pinmux { + * mode = "uart"; + * }; + * P9_26_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart1 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart1_pins: pinmux_uart1_pins { + pinctrl-single,pins = < + /* P9_24: uart1_txd.uart1_txd */ + BONE_P9_24 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) + /* P9_26: uart1_rxd.uart1_rxd */ + BONE_P9_26 (PIN_INPUT_PULLUP | MUX_MODE0) + >; + }; +}; + +&uart1 { + pinctrl-0 = <&uart1_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi new file mode 100644 index 0000000000000..5fa593ac31a55 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS2 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS2.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_21_pinmux { + * mode = "uart"; + * }; + * P9_22_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart2 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart2_pins: pinmux_uart2_pins { + pinctrl-single,pins = < + /* P9_21: spi0_d0.uart2_txd */ + BONE_P9_21 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) + /* P9_22: spi0_sclk.uart2_rxd */ + BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE1) + >; + }; +}; + +&uart2 { + pinctrl-0 = <&uart2_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi new file mode 100644 index 0000000000000..1d22a95a23a23 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS4 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS4.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_11_pinmux { + * mode = "uart"; + * }; + * P9_13_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart4 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + /* P9_11: gpmc_wait0.uart4_rxd_mux2 */ + BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6) + /* P9_13: gpmc_wpn.uart4_txd_mux2 */ + BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) + >; + }; +}; + +&uart4 { + pinctrl-0 = <&uart4_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS5.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS5.dtsi new file mode 100644 index 0000000000000..01d0aec1de447 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS5.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS5 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS5.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P8_37_pinmux { + * mode = "uart"; + * }; + * P8_38_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart5 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart5_pins: pinmux_uart5_pins { + pinctrl-single,pins = < + /* P8_38: lcd_data9.uart5_rxd */ + BONE_P8_38 (PIN_INPUT_PULLUP | MUX_MODE4) + /* P8_37: lcd_data8.uart5_txd */ + BONE_P8_37 (PIN_OUTPUT_PULLDOWN | MUX_MODE4) + >; + }; +}; + +&uart5 { + pinctrl-0 = <&uart5_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-uboot-univ.dts b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts new file mode 100644 index 0000000000000..b31b0988e8553 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone"; + compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&ldo3_reg>; +}; diff --git a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts index ce1b68a763637..e09a54f18582e 100644 --- a/arch/arm/boot/dts/am335x-bone.dts +++ b/arch/arm/boot/dts/am335x-bone.dts @@ -1,18 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone"; compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &ldo3_reg { diff --git a/arch/arm/boot/dts/am335x-boneblack-audio.dts b/arch/arm/boot/dts/am335x-boneblack-audio.dts new file mode 100644 index 0000000000000..663d549a99456 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-audio.dts @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + clk_mcasp0_fixed: clk_mcasp0_fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24576000>; + }; + + clk_mcasp0: clk_mcasp0 { + #clock-cells = <0>; + compatible = "gpio-gate-clock"; + clocks = <&clk_mcasp0_fixed>; + enable-gpios = <&gpio1 27 GPIO_ACTIVE_HIGH>; /* BeagleBone Black Clk enable on GPIO1_27 */ + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts new file mode 100644 index 0000000000000..60195222cd03e --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +#include "am335x-cape-bbb-exp-c.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts new file mode 100644 index 0000000000000..a004914041974 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +#include "am335x-cape-bbb-exp-r.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts new file mode 100644 index 0000000000000..d9b443581637b --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * Modified by Mirko Denecke + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include +#include + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&am33xx_pinmux { + dcan1_pins: pinmux_dcan1_pins { + pinctrl-single,pins = < + /* P9_26: uart1_rxd.d_can1_tx */ + BONE_P9_26 (PIN_OUTPUT_PULLUP | MUX_MODE2) + /* P9_24: uart1_txd.d_can1_rx */ + BONE_P9_24 (PIN_INPUT_PULLUP | MUX_MODE2) + >; + }; + + pru_pins: pinmux_pru_pins { + pinctrl-single,pins = < + 0x03c 0x35 /* ecap0_in_pwm0_out.pr1_ecap0_ecap_capin, MODE5 | INPUT_PULLUP | PRU, PPM-sum, SBUS, DSM */ + + 0x0e8 0x25 /* lcd_pclk.pr1_pru1_pru_r30_10, MODE5 | OUTPUT | PRU, CH_1 */ + 0x0e0 0x25 /* lcd_vsync.pr1_pru1_pru_r30_8, MODE5 | OUTPUT | PRU, CH_2 */ + 0x0ec 0x25 /* lcd_ac_bias_en.pr1_pru1_pru_r30_11, MODE5 | OUTPUT | PRU, CH_3 */ + 0x0e4 0x25 /* lcd_hsync.pr1_pru1_pru_r30_9, MODE5 | OUTPUT | PRU, CH_4 */ + 0x0bc 0x25 /* lcd_data7.pr1_pru1_pru_r30_7, MODE5 | OUTPUT | PRU, CH_5 */ + 0x0b8 0x25 /* lcd_data6.pr1_pru1_pru_r30_6, MODE5 | OUTPUT | PRU, CH_6 */ + 0x0b4 0x25 /* lcd_data5.pr1_pru1_pru_r30_5, MODE5 | OUTPUT | PRU, CH_7 */ + 0x0b0 0x25 /* lcd_data4.pr1_pru1_pru_r30_4, MODE5 | OUTPUT | PRU, CH_8 */ + 0x0ac 0x25 /* lcd_data3.pr1_pru1_pru_r30_3, MODE5 | OUTPUT | PRU, CH_9 */ + 0x0a8 0x25 /* lcd_data2.pr1_pru1_pru_r30_2, MODE5 | OUTPUT | PRU, CH_10 */ + 0x0a4 0x25 /* lcd_data1.pr1_pru1_pru_r30_1, MODE5 | OUTPUT | PRU, CH_11 */ + 0x0a0 0x25 /* lcd_data0.pr1_pru1_pru_r30_0, MODE5 | OUTPUT | PRU, CH_12 */ + + BONE_P8_12 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 TRIG */ + BONE_P8_16 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 ECHO */ + + BONE_P9_25 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* MPU9250 INT */ + >; + }; + + spi0_pins: pinmux_spi0_pins { + pinctrl-single,pins = < + /* P9_22: spi0_sclk.spi0_sclk */ + BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE0) + /* P9_21: spi0_d0.spi0_d0 */ + BONE_P9_21 (PIN_INPUT_PULLUP | MUX_MODE0) + /* P9_18: spi0_d1.spi0_d1 */ + BONE_P9_18 (PIN_OUTPUT_PULLUP | MUX_MODE0) + /* P9_17: spi0_cs0.spi0_cs0 */ + BONE_P9_17 (PIN_OUTPUT_PULLUP | MUX_MODE0) + >; + }; + + spi1_pins: pinmux_spi1_pins { + pinctrl-single,pins = < + /* P9_31: mcasp0_aclkx.spi1_sclk */ + BONE_P9_31 (PIN_INPUT_PULLUP | MUX_MODE3) + + /* P9_29: mcasp0_fsx.spi1_d0 */ + BONE_P9_29 (PIN_INPUT_PULLUP | MUX_MODE3) + + /* P9_30: mcasp0_axr0.spi1_d1 */ + BONE_P9_30 (PIN_OUTPUT_PULLUP | MUX_MODE3) + + /* P9_28: mcasp0_ahclkr.spi1_cs0 */ + BONE_P9_28 (PIN_OUTPUT_PULLUP | MUX_MODE3) + + /* P9_19: uart1_rtsn.spi1_cs1 */ +/* BONE_P9_19 (PIN_OUTPUT_PULLUP | MUX_MODE4)*/ + + /* P9_42: ecap0_in_pwm0_out.spi1_cs1 */ + BONE_P9_42A (PIN_OUTPUT_PULLUP | MUX_MODE2) + >; + }; + + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + /* P9_11: gpmc_wait0.uart4_rxd_mux2 */ + BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6) + /* P9_13: gpmc_wpn.uart4_txd_mux2 */ + BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) + >; + }; + + uart5_pins: pinmux_uart5_pins { + pinctrl-single,pins = < + /* P8_38: lcd_data9.uart5_rxd */ + BONE_P8_38 (PIN_INPUT_PULLUP | MUX_MODE4) + /* P8_37: lcd_data8.uart5_txd */ + BONE_P8_37 (PIN_OUTPUT_PULLDOWN | MUX_MODE4) + >; + }; +}; + +&dcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&dcan1_pins>; + status = "okay"; +}; + +&i2c2 { + clock-frequency = <400000>; +}; + +&spi0 { + pinctrl-names = "default"; + pinctrl-0 = <&spi0_pins>; + status = "okay"; + + spi0_0 { + #address-cells = <1>; + #size-cells = <0>; + spi-max-frequency = <24000000>; + reg = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + }; +}; + +&spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_pins>; + status = "okay"; + + spi1_0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + spi-max-frequency = <24000000>; + compatible = "spidev"; + symlink = "spi/1.0"; + }; + + spi1_1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + spi-max-frequency = <24000000>; + compatible = "spidev"; + symlink = "spi/1.1"; + }; +}; + +&tscadc { + adc { + ti,adc-channels = <0 1>; + }; +}; + +&pruss { + pinctrl-names = "default"; + pinctrl-0 = <&pru_pins>; + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins>; + status = "okay"; +}; + +&uart5 { + pinctrl-names = "default"; + pinctrl-0 = <&uart5_pins>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-common.dtsi b/arch/arm/boot/dts/am335x-boneblack-common.dtsi index 325daae40278a..bdfca259e04f1 100644 --- a/arch/arm/boot/dts/am335x-boneblack-common.dtsi +++ b/arch/arm/boot/dts/am335x-boneblack-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include @@ -29,43 +26,36 @@ &am33xx_pinmux { nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ - AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0) /* lcd_data0.lcd_data0 */ - AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0) /* lcd_data1.lcd_data1 */ - AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0) /* lcd_data2.lcd_data2 */ - AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0) /* lcd_data3.lcd_data3 */ - AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0) /* lcd_data4.lcd_data4 */ - AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0) /* lcd_data5.lcd_data5 */ - AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0) /* lcd_data6.lcd_data6 */ - AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0) /* lcd_data7.lcd_data7 */ - AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0) /* lcd_data8.lcd_data8 */ - AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0) /* lcd_data9.lcd_data9 */ - AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0) /* lcd_data10.lcd_data10 */ - AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0) /* lcd_data11.lcd_data11 */ - AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0) /* lcd_data12.lcd_data12 */ - AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0) /* lcd_data13.lcd_data13 */ - AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0) /* lcd_data14.lcd_data14 */ - AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0) /* lcd_data15.lcd_data15 */ - AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_vsync.lcd_vsync */ - AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_hsync.lcd_hsync */ - AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_pclk.lcd_pclk */ - AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_ac_bias_en.lcd_ac_bias_en */ - >; - }; - - nxp_hdmi_bonelt_off_pins: nxp_hdmi_bonelt_off_pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; mcasp0_pins: mcasp0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ - AM33XX_IOPAD(0x99c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ - AM33XX_IOPAD(0x994, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mcasp0_fsx.mcasp0_fsx */ - AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mcasp0_aclkx.mcasp0_aclkx */ - AM33XX_IOPAD(0x86c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ >; }; }; @@ -88,13 +78,12 @@ }; &i2c0 { - tda19988: tda19988 { + tda19988: tda19988@70 { compatible = "nxp,tda998x"; reg = <0x70>; - pinctrl-names = "default", "off"; + pinctrl-names = "default"; pinctrl-0 = <&nxp_hdmi_bonelt_pins>; - pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ /* video-ports = <0x234501>; */ @@ -131,6 +120,11 @@ }; / { + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x20000000>; /* 512 MB */ + }; + clk_mcasp0_fixed: clk_mcasp0_fixed { #clock-cells = <0>; compatible = "fixed-clock"; diff --git a/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts new file mode 100644 index 0000000000000..23b80e202a461 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts @@ -0,0 +1,57 @@ +/* + * @file am335x-boneblack-roboticscape.dtb + * + * based on am335x-boneblack.dtb + * + * This device tree serves to replace the need for an overlay when using + * the RoboticsCape. It is similar to the boneblue tree but preserves + * pin config for the black. + * + * This file was updated in April 2018 to support LED driver like in BB Blue + * This goes in sync with V0.4.0 of the Robotics Cape library + * + * @author James Strawson + * @date April 19, 2018 + */ + + +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" +#include "am335x-bone-common-universal-pins.dtsi" +#include "am335x-roboticscape.dtsi" + +/ { + model = "TI AM335x BeagleBone Black RoboticsCape"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts new file mode 100644 index 0000000000000..2dd94e621ac65 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot.dts b/arch/arm/boot/dts/am335x-boneblack-uboot.dts new file mode 100644 index 0000000000000..a1d7e92f5365b --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot.dts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts new file mode 100644 index 0000000000000..857c61f943fc5 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts @@ -0,0 +1,66 @@ +/* + * @file am335x-boneblack-wireless-roboticscape.dtb + * + * based on am335x-boneblack-wireless.dtb + * + * This device tree serves to replace the need for an overlay when using + * the RoboticsCape. It is similar to the boneblue tree but preserves + * pin config for the black. + * + * This file was updated in April 2018 to support LED driver like in BB Blue + * This goes in sync with V0.4.0 of the Robotics Cape library + * + * @author James Strawson + * @date April 19, 2018 + */ + + +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" +#include "am335x-bone-common-universal-pins.dtsi" +#include "am335x-boneblack-wl1835.dtsi" +#include "am335x-roboticscape.dtsi" + +/ { + model = "TI AM335x BeagleBone Black Wireless RoboticsCape"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&mac { + status = "disabled"; +}; + +&mmc3 { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wireless.dts b/arch/arm/boot/dts/am335x-boneblack-wireless.dts index 83f49f616b19c..01a940a71abf4 100644 --- a/arch/arm/boot/dts/am335x-boneblack-wireless.dts +++ b/arch/arm/boot/dts/am335x-boneblack-wireless.dts @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; @@ -11,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black Wireless"; compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-boneblack-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; @@ -29,38 +32,42 @@ }; }; +&sgx { + status = "okay"; +}; + &am33xx_pinmux { bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ - AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ - AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ - AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ - AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-boneblack-wl1835.dtsi b/arch/arm/boot/dts/am335x-boneblack-wl1835.dtsi new file mode 100644 index 0000000000000..ec6c0e49af9f3 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wl1835.dtsi @@ -0,0 +1,143 @@ + +#include + +/ { + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us= <70000>; + + /* WL_EN */ + gpio = <&gpio3 9 0>; + enable-active-high; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&wl18xx_pins>; + compatible = "gpio-leds"; + + wl18xx_bt_en { + label = "wl18xx_bt_en"; + gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + + btwilink { + compatible = "btwilink"; + }; +}; + +&am33xx_pinmux { + wl18xx_pins: pinmux_wl18xx_pins { + pinctrl-single,pins = < + 0x128 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* (K17) gmii1_txd0.gpio0[28] - BT_EN */ + >; + }; + + wlbtbuf_pin: pinmux_wlbtbuf_pin { + pinctrl-single,pins = < + 0x130 ( PIN_OUTPUT_PULLUP | MUX_MODE7 ) /* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */ + >; + }; + + mmc3_pins: pinmux_mmc3_pins { + pinctrl-single,pins = < + 0x13c ( PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + 0x114 ( PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + 0x118 ( PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + 0x11c ( PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + 0x120 ( PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + 0x108 ( PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + mmc3_pins_sleep: pinmux_mmc3_pins_sleep { + pinctrl-single,pins = < + 0x13c ( PIN_INPUT_PULLDOWN | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + 0x114 ( PIN_INPUT_PULLDOWN | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + 0x118 ( PIN_INPUT_PULLDOWN | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + 0x11c ( PIN_INPUT_PULLDOWN | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + 0x120 ( PIN_INPUT_PULLDOWN | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + 0x108 ( PIN_INPUT_PULLDOWN | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins: pinmux_wlan_pins { + pinctrl-single,pins = < + 0x12c (PIN_OUTPUT_PULLDOWN | MUX_MODE7 ) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ + 0x144 (PIN_INPUT_PULLDOWN | MUX_MODE7 ) /* (H18) rmii1_refclk.gpio0[29] - WL_IRQ */ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins_sleep: pinmux_wlan_pins_sleep { + pinctrl-single,pins = < + 0x12c (PIN_OUTPUT_PULLDOWN | MUX_MODE7 ) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ + 0x144 (PIN_INPUT_PULLDOWN | MUX_MODE7 ) /* (H18) rmii1_refclk.gpio0[29] - WL_IRQ */ + >; + }; + + uart3_pins_default: pinmux_uart3_pins_default { + pinctrl-single,pins = < + 0x134 ( PIN_INPUT_PULLUP | MUX_MODE1 ) /* (L17) gmii1_rxd3.uart3_rxd */ + 0x138 ( PIN_OUTPUT_PULLDOWN | MUX_MODE1 ) /* (L16) gmii1_rxd2.uart3_txd */ + 0x148 ( PIN_INPUT | MUX_MODE3 ) /* (M17) mdio_data.uart3_ctsn */ + 0x14c ( PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* (M18) mdio_clk.uart3_rtsn */ + >; + }; + + uart3_pins_sleep: pinmux_uart3_pins_sleep { + pinctrl-single,pins = < + 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* (L17) gmii1_rxd3.uart3_rxd */ + 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* (L16) gmii1_rxd2.uart3_txd */ + 0x148 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* (M17) mdio_data.uart3_ctsn */ + 0x14c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* (M18) mdio_clk.uart3_rtsn */ + >; + }; +}; + +&mmc3 { + dmas = <&edma_xbar 12 0 1 + &edma_xbar 13 0 2>; + dma-names = "tx", "rx"; + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&mmc3_pins &wlan_pins &wlbtbuf_pin>; + pinctrl-1 = <&mmc3_pins_sleep &wlan_pins_sleep &wlbtbuf_pin>; + ti,non-removable; + ti,needs-special-hs-handling; + cap-power-off-card; + keep-power-in-suspend; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@0 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart3 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&uart3_pins_default>; + pinctrl-1 = <&uart3_pins_sleep>; + status = "okay"; +}; + +&gpio3 { + ls_buf_en { + gpio-hog; + gpios = <10 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi new file mode 100644 index 0000000000000..94caa22903017 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include + +/ { + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + /* WL_EN */ + gpio = <&gpio0 26 0>; + enable-active-high; + }; + + kim { + compatible = "kim"; + nshutdown_gpio = <44>; /* Bank1, pin12 */ + dev_name = "/dev/ttyO4"; + flow_cntrl = <1>; + baud_rate = <3000000>; + }; + + btwilink { + compatible = "btwilink"; + }; +}; + +&am33xx_pinmux { + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + 0x30 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad12.gpio1_12 */ + >; + }; + + mmc2_pins: pinmux_mmc2_pins { + pinctrl-single,pins = < + 0x80 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + 0x84 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + 0x00 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + 0x04 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + 0x08 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + 0x0c (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + >; + }; + + mmc2_pins_sleep: pinmux_mmc2_pins_sleep { + pinctrl-single,pins = < + 0x80 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn1.mmc1_clk */ + 0x84 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn2.mmc1_cmd */ + 0x00 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad0.mmc1_dat0 */ + 0x04 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad1.mmc1_dat1 */ + 0x08 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad2.mmc1_dat2 */ + 0x0c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad3.mmc1_dat3 */ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins: pinmux_wlan_pins { + pinctrl-single,pins = < + 0x28 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN*/ + 0x2C (PIN_INPUT_PULLUP | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ*/ + 0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0) /* gpmc_csn0.gpio1_29 BF_EN*/ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins_sleep: pinmux_wlan_pins_sleep { + pinctrl-single,pins = < + 0x28 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN*/ + 0x2C (PIN_INPUT_PULLUP | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ*/ + 0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0) /* gpmc_csn0.gpio1_29 BF_EN*/ + >; + }; + + uart4_pins_default: pinmux_uart4_pins_default { + pinctrl-single,pins = < + 0xD0 (PIN_INPUT | MUX_MODE6) /* lcd_data12.uart4_cts */ + 0xD4 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* lcd_data13.uart4_rts */ + 0x70 (PIN_INPUT_PULLUP | MUX_MODE6) /* gpmc_wait0.uart4_rxd */ + 0x74 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_wpn.uart4_txd */ + >; + }; + + uart4_pins_sleep: pinmux_uart4_pins_sleep { + pinctrl-single,pins = < + 0xD0 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_data12.uart4_cts */ + 0xD4 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_data13.uart4_rts */ + 0x70 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_wait0.uart4_rxd */ + 0x74 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_wpn.uart4_txd */ + >; + }; +}; + +&mmc2 { + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&mmc2_pins &wlan_pins>; + pinctrl-1 = <&mmc2_pins_sleep &wlan_pins_sleep>; + ti,non-removable; + ti,needs-special-hs-handling; + cap-power-off-card; + keep-power-in-suspend; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@0 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <27 IRQ_TYPE_LEVEL_HIGH>; + }; +}; + +&uart4 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&uart4_pins_default>; + pinctrl-1 = <&uart4_pins_sleep>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts new file mode 100644 index 0000000000000..78abd6c34821d --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +/* EMMC in reset */ +&gpio1 { + emmc_rst { + gpio-hog; + gpios = <20 0>; + output-high; + line-name = "EMMC ResetN"; + }; +}; + +#include "am335x-boneblack-wl1835mod-cape.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts index 5497718f5d44f..7bcf13bad47dd 100644 --- a/arch/arm/boot/dts/am335x-boneblack.dts +++ b/arch/arm/boot/dts/am335x-boneblack.dts @@ -1,23 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" - -&sgx { - status = "okay"; -}; +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black"; compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; }; &cpu0_opp_table { diff --git a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts index cdc1b2be792f3..64cc61c35fb1c 100644 --- a/arch/arm/boot/dts/am335x-boneblue.dts +++ b/arch/arm/boot/dts/am335x-boneblue.dts @@ -1,19 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" +#include "am335x-bone-common-universal-pins.dtsi" #include / { model = "TI AM335x BeagleBone Blue"; compatible = "ti,am335x-bone-blue", "ti,am33xx"; + chosen { + stdout-path = &uart0; + base_dtb = "am335x-boneblue.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + cpus { cpu@0 { cpu0-supply = <&dcdc2_reg>; @@ -25,10 +29,6 @@ reg = <0x80000000 0x20000000>; /* 512 MB */ }; - chosen { - stdout-path = &uart0; - }; - leds { pinctrl-names = "default"; pinctrl-0 = <&user_leds_s0>; @@ -125,22 +125,117 @@ gpio = <&gpio3 9 0>; enable-active-high; }; + + bt_en { + pinctrl-names = "default"; + pinctrl-0 = <&bt_pins>; + compatible = "gpio-leds"; + + wl18xx_bt_en { + label = "wl18xx_bt_en"; + gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; +}; + +&aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + d_can0 = &dcan0; + d_can1 = &dcan1; + usb0 = &usb0; + usb1 = &usb1; + phy0 = &usb0_phy; + phy1 = &usb1_phy; + ethernet0 = &cpsw_emac0; + ethernet1 = &cpsw_emac1; + spi0 = &spi0; + spi1 = &spi1; +}; + +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * Octavo Systems: + * The EFUSE_SMA register is not programmed for any of the AM335x wafers + * we get and we are not programming them during our production test. + * Therefore, from a DEVICE_ID revision point of view, the silicon looks + * like it is Revision 2.1. However, from an EFUSE_SMA point of view for + * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the + * EFUSE_SMA register reads as all zeros). + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; }; &am33xx_pinmux { + /*************************************************************************** + * Static Pinmux + ***************************************************************************/ + mux_helper_pins: pins { + pinctrl-single,pins = < + + /* GPIO Inputs */ + 0x09c 0x37 /*P8.9 Pause BUTTON, input pullup*/ + 0x098 0x37 /*P8.10 MODE BUTTON input pullup*/ + 0x1AC 0x37 /*P9.25 MPU-9150 INTERRUPT IN*/ + + /* Motor Control GPIO Out*/ + 0x088 ( PIN_OUTPUT | MUX_MODE7 ) /* (T13) gpmc_csn3.gpio2[0] - MDIR_1A different from cape! */ + 0x074 ( PIN_OUTPUT | MUX_MODE7 ) /* (U17) gpmc_wpn.gpio0[31] - P9.13, MDIR_1B */ + 0x040 ( PIN_OUTPUT | MUX_MODE7 ) /* (R13) gpmc_a0.gpio1[16] - P9.15, MDIR_2A */ + 0x0D8 ( PIN_OUTPUT | MUX_MODE7 ) /* (V4) lcd_data14.gpio0[10] - P8.31, MDIR_2B different from cape! */ + 0x0AC ( PIN_OUTPUT | MUX_MODE7 ) /* (R4) lcd_data3.gpio2[9] - P8.44, MDIR_3A */ + 0x0A8 ( PIN_OUTPUT | MUX_MODE7 ) /* (R3) lcd_data2.gpio2[8] - P8.43, MDIR_3B */ + 0x0A0 ( PIN_OUTPUT | MUX_MODE7 ) /* (R1) lcd_data0.gpio2[6] - P8.45, MDIR_4A */ + 0x0A4 ( PIN_OUTPUT | MUX_MODE7 ) /* (R2) lcd_data1.gpio2[7] - P8.46, MDIR_4B */ + 0x1B4 ( PIN_OUTPUT | MUX_MODE7 ) /* (D14) xdma_event_intr1.gpio0[20] - P9.41, MOT_STBY */ + + /* PRU encoder input */ + 0x038 0x36 /* P8_16,PRU0_r31_16,MODE6 */ + + /* PRU Servo output */ + 0x0e0 0x05 /*pru1_pru_r30_8, MODE5*/ + 0x0e8 0x05 /*pru1_pru_r30_10, MODE5 */ + 0x0e4 0x05 /*pr1_pru1_pru_r30_9, MODE5 */ + 0x0ec 0x05 /*pru1_pru_r30_11, MODE5 */ + 0x0b8 0x05 /*pru1_pru_r30_6, MODE5 */ + 0x0bc 0x05 /*pru1_pru_r30_7, MODE5 */ + 0x0b0 0x05 /*pru1_pru_r30_4, MODE5 */ + 0x0b4 0x05 /*pru1_pru_r30_5, MODE5 */ + 0x0C8 0x0F /*P8.36, SERVO_PWR GPIO OUT*/ + + /* WILINK 8 */ + 0x08c 0x0F /*P8.18 V12 A2DP FSYNC */ + 0x078 0x0F /*P9.12 A2DP_CLOCK*/ + >; + }; + user_leds_s0: user_leds_s0 { pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ - AM33XX_IOPAD(0x9b0, PIN_OUTPUT | MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */ - AM33XX_IOPAD(0x890, PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ - AM33XX_IOPAD(0x894, PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ - AM33XX_IOPAD(0x82c, PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ - AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */ - AM33XX_IOPAD(0x87c, PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ - AM33XX_IOPAD(0x828, PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT, MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT, MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT, MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT, MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ + AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT, MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */ + AM33XX_PADCONF(AM335X_PIN_GPMC_ADVN_ALE, PIN_OUTPUT, MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_OEN_REN, PIN_OUTPUT, MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_OUTPUT, MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT, MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT, MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ >; }; @@ -152,74 +247,207 @@ >; }; + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + 0x15C 0x32 /* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */ + 0x158 0x32 /* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */ + >; + }; + i2c2_pins: pinmux_i2c2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ >; }; + /* UT0 */ uart0_pins: pinmux_uart0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* (E15) uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* (E16) uart0_txd.uart0_txd */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + /* UT5 */ + uart5_pins: pinmux_uart5_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_INPUT_PULLUP, MUX_MODE4) /* (U2) lcd_data9.uart5_rxd */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT_PULLDOWN, MUX_MODE4) /* (U1) lcd_data8.uart5_txd */ >; }; mmc1_pins: pinmux_mmc1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ >; }; mmc2_pins: pinmux_mmc2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* (U9) gpmc_csn1.mmc1_clk */ - AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* (V9) gpmc_csn2.mmc1_cmd */ - AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* (U7) gpmc_ad0.mmc1_dat0 */ - AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* (V7) gpmc_ad1.mmc1_dat1 */ - AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* (R8) gpmc_ad2.mmc1_dat2 */ - AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* (T8) gpmc_ad3.mmc1_dat3 */ - AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* (U8) gpmc_ad4.mmc1_dat4 */ - AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* (V8) gpmc_ad5.mmc1_dat5 */ - AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* (R9) gpmc_ad6.mmc1_dat6 */ - AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* (T9) gpmc_ad7.mmc1_dat7 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* (U9) gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* (V9) gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* (U7) gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* (V7) gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* (R8) gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* (T8) gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* (U8) gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* (V8) gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* (R9) gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* (T9) gpmc_ad7.mmc1_dat7 */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6) /* (L15) gmii1_rxd1.mmc2_clk */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6) /* (J16) gmii1_txen.mmc2_cmd */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5) /* (J17) gmii1_rxdv.mmc2_dat0 */ - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5) /* (J18) gmii1_txd3.mmc2_dat1 */ - AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5) /* (K15) gmii1_txd2.mmc2_dat2 */ - AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5) /* (H16) gmii1_col.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5) /* (H16) gmii1_col.mmc2_dat3 */ >; }; bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7) /* (K17) gmii1_txd0.gpio0[28] - BT_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* (K17) gmii1_txd0.gpio0[28] - BT_EN */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* (M17) mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* (M18) mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* (M17) mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* (M18) mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ - AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (H18) rmii1_refclk.gpio0[29] - WL_IRQ */ - AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7) /* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) /* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */ + >; + }; + + /* DCAN */ + dcan1_pins: pinmux_dcan1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT, MUX_MODE2) /* (E17) uart0_rtsn.dcan1_rx */ + AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_OUTPUT, MUX_MODE2) /* (E18) uart0_ctsn.dcan1_tx */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_OUTPUT, MUX_MODE7) /* (M16) gmii1_rxd0.gpio2[21] */ + >; + }; + + /* E1 */ + eqep0_pins: pinmux_eqep0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MCASP0_AXR0, PIN_INPUT, MUX_MODE1) /* (B12) mcasp0_aclkr.eQEP0A_in */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSR, PIN_INPUT, MUX_MODE1) /* (C13) mcasp0_fsr.eQEP0B_in */ + >; + }; + + /* E2 */ + eqep1_pins: pinmux_eqep1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_INPUT, MUX_MODE2) /* (V2) lcd_data12.eQEP1A_in */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_INPUT, MUX_MODE2) /* (V3) lcd_data13.eQEP1B_in */ + >; + }; + + /* E3 */ + eqep2_pins: pinmux_eqep2_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT, MUX_MODE4) /* (T12) gpmc_ad12.eQEP2A_in */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT, MUX_MODE4) /* (R12) gpmc_ad13.eQEP2B_in */ + >; + }; + + ehrpwm1_pins: pinmux_ehrpwm1_pins { + pinctrl-single,pins = < + 0x020 0x24 /* P8_19_pwm_pin */ + 0x024 0x24 /* P8_13_pwm_pin */ + >; + }; + + ehrpwm2_pins: pinmux_ehrpwm2_pins { + pinctrl-single,pins = < + 0x048 0x26 /* P9_14_pwm_pin */ + 0x04c 0x26 /* P9_16_pwm_pin */ >; }; + + /* H18 SPI1_SS1 default SPI*/ + H18_default_pin: pinmux_H18_default_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + H18_gpio_pin: pinmux_H18_gpio_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + H18_gpio_pu_pin: pinmux_H18_gpio_pu_pin { + pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H18_gpio_pd_pin: pinmux_H18_gpio_pd_pin { + pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + H18_spi_pin: pinmux_H18_spi_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + + + /* C18 SPI1_SS2 default spi*/ + C18_default_pin: pinmux_C18_default_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + C18_gpio_pin: pinmux_C18_gpio_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + C18_gpio_pu_pin: pinmux_C18_gpio_pu_pin { + pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + C18_gpio_pd_pin: pinmux_C18_gpio_pd_pin { + pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + C18_spi_pin: pinmux_C18_spi_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + + + + /* D13 BLUE_GP0_PIN_5 gpio 3_20 */ + D13_default_pin: pinmux_D13_default_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + D13_gpio_pin: pinmux_D13_gpio_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + D13_gpio_pu_pin: pinmux_D13_gpio_pu_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + D13_gpio_pd_pin: pinmux_D13_gpio_pd_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + /* H17 BLUE_GP1_PIN_4 gpio 3_1 */ + H17_default_pin: pinmux_H17_default_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H17_gpio_pin: pinmux_H17_gpio_pin { + pinctrl-single,pins = < 0x10C ( PIN_OUTPUT | MUX_MODE7 ) >; }; + H17_gpio_pu_pin: pinmux_H17_gpio_pu_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H17_gpio_pd_pin: pinmux_H17_gpio_pd_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + /* J15 BLUE_GP1_PIN_3 gpio 3_2 */ + J15_default_pin: pinmux_J15_default_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + J15_gpio_pin: pinmux_J15_gpio_pin { + pinctrl-single,pins = < 0x110 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + J15_gpio_pu_pin: pinmux_J15_gpio_pu_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + J15_gpio_pd_pin: pinmux_J15_gpio_pd_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + + /* U16 BLUE_GP0_PIN_3 gpio 1_25 */ + U16_default_pin: pinmux_U16_default_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + U16_gpio_pin: pinmux_U16_gpio_pin { + pinctrl-single,pins = < 0x064 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + U16_gpio_pu_pin: pinmux_U16_gpio_pu_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + U16_gpio_pd_pin: pinmux_U16_gpio_pd_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + }; &uart0 { @@ -229,6 +457,27 @@ status = "okay"; }; +&uart1 { + status = "okay"; +}; + +&uart2 { + + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + status = "okay"; +}; + +&uart5 { + pinctrl-names = "default"; + pinctrl-0 = <&uart5_pins>; + + status = "okay"; +}; + &usb { status = "okay"; }; @@ -273,7 +522,7 @@ }; baseboard_eeprom: baseboard_eeprom@50 { - compatible = "at,24c256"; + compatible = "atmel,24c256"; reg = <0x50>; #address-cells = <1>; @@ -284,32 +533,19 @@ }; }; +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + status = "okay"; + clock-frequency = <400000>; +}; + &i2c2 { pinctrl-names = "default"; pinctrl-0 = <&i2c2_pins>; status = "okay"; clock-frequency = <400000>; - - mpu9250@68 { - compatible = "invensense,mpu9250"; - reg = <0x68>; - interrupt-parent = <&gpio3>; - interrupts = <21 GPIO_ACTIVE_LOW>; - i2c-gate { - #address-cells = <1>; - #size-cells = <0>; - ax8975@c { - compatible = "ak,ak8975"; - reg = <0x0c>; - }; - }; - }; - - pressure@76 { - compatible = "bosch,bmp280"; - reg = <0x76>; - }; }; /include/ "tps65217.dtsi" @@ -318,14 +554,13 @@ interrupts = <7>; /* NMI */ interrupt-parent = <&intc>; + ti,pmic-shutdown-controller; + charger { - interrupts = <0>, <1>; - interrupt-names = "USB", "AC"; status = "okay"; }; pwrbutton { - interrupts = <2>; status = "okay"; }; @@ -414,26 +649,27 @@ compatible = "ti,wl1835"; reg = <2>; interrupt-parent = <&gpio0>; - interrupts = <29 IRQ_TYPE_EDGE_RISING>; + interrupts = <21 IRQ_TYPE_EDGE_RISING>; }; }; -&tscadc { - status = "okay"; - adc { - ti,adc-channels = <0 1 2 3 4 5 6 7>; - }; -}; +//&tscadc { +// status = "okay"; +// adc { +// ti,adc-channels = <0 1 2 3 4 5 6 7>; +// }; +//}; &uart3 { pinctrl-names = "default"; - pinctrl-0 = <&uart3_pins &bt_pins>; + //pinctrl-0 = <&uart3_pins &bt_pins>; + pinctrl-0 = <&uart3_pins>; status = "okay"; - bluetooth { - compatible = "ti,wl1835-st"; - enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; - }; +// bluetooth { +// compatible = "ti,wl1835-st"; +// enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; +// }; }; &aes { @@ -446,8 +682,24 @@ &rtc { system-power-controller; - clocks = <&clk_32768_ck>, <&clkdiv32k_ick>; - clock-names = "ext-clk", "int-clk"; +}; + +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@0 { +// status = "okay"; +// }; +//}; + +&dcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&dcan1_pins>; + status = "okay"; }; &gpio3 { @@ -458,3 +710,360 @@ line-name = "LS_BUF_EN"; }; }; + +&ocp { + /* activate the static pinmux helper list of pin modes above */ + test_helper: helper { + compatible = "bone-pinmux-helper"; + pinctrl-names = "default"; + pinctrl-0 = <&mux_helper_pins>; + + status = "okay"; + }; + + + /* UART 2 RX GPS */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_22_uart_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_spi_pin>; + pinctrl-5 = <&P9_22_uart_pin>; + pinctrl-6 = <&P9_22_i2c_pin>; + pinctrl-7 = <&P9_22_pwm_pin>; + }; + + /* UART 2 TX GPS*/ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_21_uart_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_spi_pin>; + pinctrl-5 = <&P9_21_uart_pin>; + pinctrl-6 = <&P9_21_i2c_pin>; + pinctrl-7 = <&P9_21_pwm_pin>; + }; + + /* UART 1 RX */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_26_uart_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_uart_pin>; + pinctrl-5 = <&P9_26_can_pin>; + pinctrl-6 = <&P9_26_i2c_pin>; + pinctrl-7 = <&P9_26_pruin_pin>; + }; + + /* UART 1 TX */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_24_uart_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_uart_pin>; + pinctrl-5 = <&P9_24_can_pin>; + pinctrl-6 = <&P9_24_i2c_pin>; + pinctrl-7 = <&P9_24_pruin_pin>; + }; + + /* SPI MOSI */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_30_spi_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_pwm_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pruout_pin>; + pinctrl-7 = <&P9_30_pruin_pin>; + }; + + /* SPI MISO */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_29_spi_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_pwm_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pruout_pin>; + pinctrl-7 = <&P9_29_pruin_pin>; + }; + + /* SPI SCK */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin"; + pinctrl-0 = <&P9_31_spi_sclk_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_pwm_pin>; + pinctrl-5 = <&P9_31_spi_pin>; + pinctrl-6 = <&P9_31_spi_sclk_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* Encoder 4 (U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "pruin_pu", "gpio", "gpio_pu", "gpio_pd", "pruin", "qep", "pruecapin_pu"; + pinctrl-0 = <&P8_15_pruin_pu_pin>; + pinctrl-1 = <&P8_15_pruin_pu_pin>; + pinctrl-2 = <&P8_15_gpio_pin>; + pinctrl-3 = <&P8_15_gpio_pu_pin>; + pinctrl-4 = <&P8_15_gpio_pd_pin>; + pinctrl-5 = <&P8_15_pruin_pin>; + pinctrl-6 = <&P8_15_qep_pin>; + pinctrl-7 = <&P8_15_pruecapin_pu_pin>; + }; + + /* UART4 RX DSM */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "uart", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&P9_11_uart_pin>; + pinctrl-1 = <&P9_11_uart_pin>; + pinctrl-2 = <&P9_11_gpio_pin>; + pinctrl-3 = <&P9_11_gpio_pu_pin>; + pinctrl-4 = <&P9_11_gpio_pd_pin>; + }; + + /* U16 BLUE_GP0_PIN_3 gpio 1_25*/ + U16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&U16_default_pin>; + pinctrl-1 = <&U16_gpio_pin>; + pinctrl-2 = <&U16_gpio_pu_pin>; + pinctrl-3 = <&U16_gpio_pd_pin>; + }; + + + /* BLUE_GP0_PIN_3 gpio1_17*/ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_pwm_pin>; + }; + + /* BLUE_GP0_PIN_5 gpio3_20 */ + D13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&D13_default_pin>; + pinctrl-1 = <&D13_gpio_pin>; + pinctrl-2 = <&D13_gpio_pu_pin>; + pinctrl-3 = <&D13_gpio_pd_pin>; + }; + + /* BLUE_GP0_PIN_6 gpio3_17 */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_pwm_pin>; + pinctrl-5 = <&P9_28_spi_pin>; + pinctrl-6 = <&P9_28_spi_cs_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* BLUE_GP1_PIN_3 gpio3_2 */ + J15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&J15_default_pin>; + pinctrl-1 = <&J15_gpio_pin>; + pinctrl-2 = <&J15_gpio_pu_pin>; + pinctrl-3 = <&J15_gpio_pd_pin>; + }; + + /* BLUE_GP1_PIN_4 gpio3_1 */ + H17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&H17_default_pin>; + pinctrl-1 = <&H17_gpio_pin>; + pinctrl-2 = <&H17_gpio_pu_pin>; + pinctrl-3 = <&H17_gpio_pd_pin>; + }; + + + //////////////////////////////////////////////////////////////////////// + // Pinmux options only for BB BLue + //////////////////////////////////////////////////////////////////////// + + + + /* H18 SPI1_SS1 default SPI*/ + H18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi"; + pinctrl-0 = <&H18_spi_pin>; + pinctrl-1 = <&H18_gpio_pin>; + pinctrl-2 = <&H18_gpio_pu_pin>; + pinctrl-3 = <&H18_gpio_pd_pin>; + pinctrl-4 = <&H18_spi_pin>; + }; + + + /* C18 SPI1_SS2 default spi*/ + C18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi"; + pinctrl-0 = <&C18_spi_pin>; + pinctrl-1 = <&C18_gpio_pin>; + pinctrl-2 = <&C18_gpio_pu_pin>; + pinctrl-3 = <&C18_gpio_pd_pin>; + pinctrl-4 = <&C18_spi_pin>; + }; + +}; + +/******************************************************************************* +* PWMSS +*******************************************************************************/ +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; +}; + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm1_pins>; + status = "okay"; +}; + +&ehrpwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm2_pins>; + status = "okay"; +}; + +&ecap0 { status="okay"; }; +&ecap1 { status="okay"; }; +&ecap2 { status="okay"; }; + +/******************************************************************************* +* EQEP +*******************************************************************************/ +&eqep0 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep0_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep1 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep1_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep2 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep2_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +/******************************************************************************* + SPI +*******************************************************************************/ +&spi1 { + + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi index 853e6d3a028dc..7a8826633cef7 100644 --- a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi +++ b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ &ldo3_reg { @@ -27,8 +24,8 @@ &am33xx_pinmux { uart2_pins: uart2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x950, PIN_INPUT | MUX_MODE1) /* spi0_sclk.uart2_rxd */ - AM33XX_IOPAD(0x954, PIN_OUTPUT | MUX_MODE1) /* spi0_d0.uart2_txd */ + AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT, MUX_MODE1) /* spi0_sclk.uart2_rxd */ + AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_OUTPUT, MUX_MODE1) /* spi0_d0.uart2_txd */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-bonegreen-gateway.dts b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts new file mode 100644 index 0000000000000..5b3a5b8095586 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-common.dtsi" +#include +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "SeeedStudio BeagleBone Green Gateway"; + compatible = "ti,am335x-bone-green-gateway", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + aliases { + rtc0 = &extrtc; + rtc1 = &rtc; + }; + + chosen { + base_dtb = "am335x-bonegreen-gateway.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us= <70000>; + + /* WL_EN */ + gpio = <&gpio3 9 0>; + enable-active-high; + }; + + leds { + led6 { + label = "beaglebone:green:usr4"; + gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "netdev"; + default-state = "off"; + }; + }; +}; + +&cpu0_opp_table { + /* + * Octavo Systems: + * The EFUSE_SMA register is not programmed for any of the AM335x wafers + * we get and we are not programming them during our production test. + * Therefore, from a DEVICE_ID revision point of view, the silicon looks + * like it is Revision 2.1. However, from an EFUSE_SMA point of view for + * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the + * EFUSE_SMA register reads as all zeros). + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&sgx { + status = "okay"; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&usbhost_pins>; + + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + >; + }; + + mmc3_pins: pinmux_mmc3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + uart2_grove_pins: pinmux_uart2_grove_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x90c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + AM33XX_IOPAD(0x910, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + >; + }; + + uart3_pins: pinmux_uart3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ + >; + }; + + wl18xx_pins: pinmux_wl18xx_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + >; + }; + + usbhost_pins: pinmux_usbhost_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd1.gpio0[21] */ + >; + }; +}; + +&mac { + status = "disabled"; +}; + +&mmc3 { + dmas = <&edma_xbar 12 0 1 + &edma_xbar 13 0 2>; + dma-names = "tx", "rx"; + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + non-removable; + cap-power-off-card; + ti,needs-special-hs-handling; + keep-power-in-suspend; + pinctrl-names = "default"; + pinctrl-0 = <&mmc3_pins &wl18xx_pins>; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@2 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_grove_pins>; + status = "okay"; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_pins &bt_pins>; + status = "okay"; + + bluetooth { + compatible = "ti,wl1835-st"; + enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c0 { + extrtc: rtc@68 { + compatible = "dallas,ds1340"; + reg = <0x68>; + }; +}; + +// (K16) gmii1_txd1.gpio0[21] +&gpio0 { + usb_reset { + gpio-hog; + gpios = <21 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb_reset"; + }; +}; + +&gpio3 { + ls_buf_en { + gpio-hog; + gpios = <10 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +&usb1 { + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb424,9512"; + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + ethernet: ethernet@1 { + compatible = "usb424,ec00"; + reg = <1>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi new file mode 100644 index 0000000000000..019f90e02e3f1 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi @@ -0,0 +1,2835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/1"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/2"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/1"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/2"; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/3"; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/4"; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/5"; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/0"; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/1"; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/0"; +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/1"; +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/2"; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts new file mode 100644 index 0000000000000..3adb9721f9f7b --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-wireless-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Green Wireless"; + compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen-wireless-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&gpio1 { + ls_buf_en { + gpio-hog; + gpios = <29 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +/* BT_AUD_OUT from wl1835 has to be pulled low when WL_EN is activated.*/ +/* in case it isn't, wilink8 ends up in one of the test modes that */ +/* intruces various issues (elp wkaeup timeouts etc.) */ +/* On the BBGW this pin is routed through the level shifter (U21) that */ +/* introduces a pullup on the line and wilink8 ends up in a bad state. */ +/* use a gpio hog to force this pin low. An alternative may be adding */ +/* an external pulldown on U21 pin 4. */ + +&gpio3 { + bt_aud_in { + gpio-hog; + gpios = <16 GPIO_ACTIVE_HIGH>; + output-low; + line-name = "MCASP0_AHCLKR"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts index 57731f0daf103..8372b8e496622 100644 --- a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; @@ -11,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green Wireless"; compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-bonegreen-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; @@ -29,38 +32,42 @@ }; }; +&sgx { + status = "okay"; +}; + &am33xx_pinmux { bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x878, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad12.gpio1_28 BT_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_BEN1, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_ad12.gpio1_28 BT_EN */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x830, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad12.mmc2_dat0 */ - AM33XX_IOPAD(0x834, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad13.mmc2_dat1 */ - AM33XX_IOPAD(0x838, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad14.mmc2_dat2 */ - AM33XX_IOPAD(0x83c, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad15.mmc2_dat3 */ - AM33XX_IOPAD(0x888, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_csn3.mmc2_cmd */ - AM33XX_IOPAD(0x88c, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_clk.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad12.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad13.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD14, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad14.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD15, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad15.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN3, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_csn3.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CLK, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_clk.mmc2_clk */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x828, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN */ - AM33XX_IOPAD(0x82C, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ */ - AM33XX_IOPAD(0x87C, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_csn0.gpio1_29 LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_INPUT_PULLDOWN, MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_csn0.gpio1_29 LS_BUF_EN */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-bonegreen.dts b/arch/arm/boot/dts/am335x-bonegreen.dts index a8b4d969ce2a0..dc6f69a85bf08 100644 --- a/arch/arm/boot/dts/am335x-bonegreen.dts +++ b/arch/arm/boot/dts/am335x-bonegreen.dts @@ -1,17 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green"; compatible = "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; }; diff --git a/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi new file mode 100644 index 0000000000000..d2704c6921083 --- /dev/null +++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#include "am335x-peripheral-can0.dtsi" +#include "am335x-bone-pinmux-can0.dtsi" + +#include "am335x-peripheral-ttyS1.dtsi" +#include "am335x-bone-pinmux-ttyS1.dtsi" + +#include "am335x-peripheral-ttyS2.dtsi" +#include "am335x-bone-pinmux-ttyS2.dtsi" + +#include "am335x-peripheral-ttyS4.dtsi" +#include "am335x-bone-pinmux-ttyS4.dtsi" + +&am33xx_pinmux { + user_leds_s1: user_leds_s1 { + pinctrl-single,pins = < + 0x98 0x7 /* gpmc_wen.gpio2_4, OUTPUT | MODE7 */ + 0x9c 0x7 /* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */ + >; + }; + + bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins { + pinctrl-single,pins = < + BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */ + >; + }; + + keymap3_pins: pinmux_keymap3_pins { + pinctrl-single,pins = < + 0x040 0x2f /* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */ + 0x04c 0x2f /* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */ + 0x078 0x2f /* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */ + 0x164 0x2f /* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */ + 0x1a4 0x2f /* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */ + >; + }; + + edt_ft5306_ts_pins: pinmux_edt_ft5306_ts_pins { + pinctrl-single,pins = < + /* CAP_TSC gpmc_a1.gpio1_17, INPUT | MODE7 */ + BONE_P9_23 (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + /* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + /* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + >; + }; + + mcasp0_pins: pinmux_mcasp0_pins { + pinctrl-single,pins = < + 0x190 0x20 /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */ + 0x194 0x20 /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */ + 0x198 0x20 /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */ + 0x19c 0x22 /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */ + >; + }; +}; + +&epwmss1 { + status = "okay"; +}; + + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&bb_lcd_pwm_backlight_pins>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + clock-frequency = <400000>; + + edt-ft5306@38 { + status = "okay"; + compatible = "edt,edt-ft5306", "edt,edt-ft5x06"; + pinctrl-names = "default"; + pinctrl-0 = <&edt_ft5306_ts_pins>; + + reg = <0x38>; + interrupt-parent = <&gpio1>; + interrupts = <17 0>; + + touchscreen-size-x = <600>; + touchscreen-size-y = <1024>; + touchscreen-inverted-x; + touchscreen-inverted-y; + touchscreen-swapped-x-y; + }; + + tlv320aic3x: tlv320aic3x@1b { + compatible = "ti,tlv320aic3x"; + reg = <0x1b>; + status = "okay"; + }; +}; + +&mcasp0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + + status = "okay"; + + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + num-serializer = <16>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 1 0 2 0 + 0 0 0 0 + 0 0 0 0 + 0 0 0 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +/ { + backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&ehrpwm1 0 50000 0>; + brightness-levels = <0 51 53 56 62 75 101 152 255>; + default-brightness-level = <8>; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&keymap3_pins>; + + #address-cells = <1>; + #size-cells = <0>; + + button@1 { + debounce_interval = <50>; + linux,code = <105>; + label = "left"; + gpios = <&gpio0 7 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@2 { + debounce_interval = <50>; + linux,code = <106>; + label = "right"; + gpios = <&gpio1 28 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@3 { + debounce_interval = <50>; + linux,code = <103>; + label = "up"; + gpios = <&gpio1 16 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@4 { + debounce_interval = <50>; + linux,code = <108>; + label = "down"; + gpios = <&gpio1 19 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@5 { + debounce_interval = <50>; + linux,code = <28>; + label = "enter"; + gpios = <&gpio3 19 0x1>; + gpio-key,wakeup; + }; + }; + + gpio-leds-cape-lcd { + compatible = "gpio-leds"; + pinctrl-names = "default"; + + pinctrl-0 = <&user_leds_s1>; + + lcd-led0 { + label = "lcd:green:usr0"; + gpios = <&gpio2 4 0>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + lcd-led1 { + label = "lcd:green:usr1"; + gpios = <&gpio2 5 0>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + }; + + sound { + compatible = "ti,da830-evm-audio"; + ti,model = "DA830 EVM"; + ti,audio-codec = <&tlv320aic3x>; + ti,mcasp-controller = <&mcasp0>; + ti,codec-clock-rate = <12000000>; + ti,audio-routing = + "Headphone Jack", "HPLOUT", + "Headphone Jack", "HPROUT", + "MIC3L", "Mic Jack", + "MIC3R", "Mic Jack"; + }; +}; + +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" +#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi" diff --git a/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi new file mode 100644 index 0000000000000..539409c9fcd81 --- /dev/null +++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#include "am335x-peripheral-can0.dtsi" +#include "am335x-bone-pinmux-can0.dtsi" + +#include "am335x-peripheral-ttyS1.dtsi" +#include "am335x-bone-pinmux-ttyS1.dtsi" + +#include "am335x-peripheral-ttyS2.dtsi" +#include "am335x-bone-pinmux-ttyS2.dtsi" + +#include "am335x-peripheral-ttyS4.dtsi" +#include "am335x-bone-pinmux-ttyS4.dtsi" + +&am33xx_pinmux { + user_leds_s1: user_leds_s1 { + pinctrl-single,pins = < + 0x98 0x7 /* gpmc_wen.gpio2_4, OUTPUT | MODE7 */ + 0x9c 0x7 /* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */ + >; + }; + + bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins { + pinctrl-single,pins = < + BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */ + >; + }; + + keymap3_pins: pinmux_keymap3_pins { + pinctrl-single,pins = < + 0x040 0x2f /* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */ + 0x04c 0x2f /* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */ + 0x078 0x2f /* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */ + 0x164 0x2f /* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */ + 0x1a4 0x2f /* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */ + >; + }; + + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + /* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + /* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + >; + }; + + mcasp0_pins: pinmux_mcasp0_pins { + pinctrl-single,pins = < + 0x190 0x20 /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */ + 0x194 0x20 /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */ + 0x198 0x20 /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */ + 0x19c 0x22 /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */ + >; + }; +}; + +&epwmss1 { + status = "okay"; +}; + + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&bb_lcd_pwm_backlight_pins>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + clock-frequency = <400000>; + + tlv320aic3x: tlv320aic3x@1b { + compatible = "ti,tlv320aic3x"; + reg = <0x1b>; + status = "okay"; + }; +}; + +&mcasp0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + + status = "okay"; + + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + num-serializer = <16>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 1 0 2 0 + 0 0 0 0 + 0 0 0 0 + 0 0 0 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +&tscadc { + status = "okay"; + tsc { + ti,wires = <4>; + ti,x-plate-resistance = <200>; + ti,coordinate-readouts = <5>; + ti,wire-config = <0x00 0x11 0x22 0x33>; + }; + + adc { + ti,adc-channels = <4 5 6 7>; + }; +}; + +/ { + backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&ehrpwm1 0 50000 0>; + brightness-levels = <0 51 53 56 62 75 101 152 255>; + default-brightness-level = <8>; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&keymap3_pins>; + + #address-cells = <1>; + #size-cells = <0>; + + button@1 { + debounce_interval = <50>; + linux,code = <105>; + label = "left"; + gpios = <&gpio0 7 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@2 { + debounce_interval = <50>; + linux,code = <106>; + label = "right"; + gpios = <&gpio1 28 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@3 { + debounce_interval = <50>; + linux,code = <103>; + label = "up"; + gpios = <&gpio1 16 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@4 { + debounce_interval = <50>; + linux,code = <108>; + label = "down"; + gpios = <&gpio1 19 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@5 { + debounce_interval = <50>; + linux,code = <28>; + label = "enter"; + gpios = <&gpio3 19 0x1>; + gpio-key,wakeup; + }; + }; + + gpio-leds-cape-lcd { + compatible = "gpio-leds"; + pinctrl-names = "default"; + + pinctrl-0 = <&user_leds_s1>; + + lcd-led0 { + label = "lcd:green:usr0"; + gpios = <&gpio2 4 0>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + lcd-led1 { + label = "lcd:green:usr1"; + gpios = <&gpio2 5 0>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + }; + + sound { + compatible = "ti,da830-evm-audio"; + ti,model = "DA830 EVM"; + ti,audio-codec = <&tlv320aic3x>; + ti,mcasp-controller = <&mcasp0>; + ti,codec-clock-rate = <12000000>; + ti,audio-routing = + "Headphone Jack", "HPLOUT", + "Headphone Jack", "HPROUT", + "MIC3L", "Mic Jack", + "MIC3R", "Mic Jack"; + }; +}; + +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" +#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi" diff --git a/arch/arm/boot/dts/am335x-cape-rtc-ds1307.dtsi b/arch/arm/boot/dts/am335x-cape-rtc-ds1307.dtsi new file mode 100644 index 0000000000000..bce6ac50cd89a --- /dev/null +++ b/arch/arm/boot/dts/am335x-cape-rtc-ds1307.dtsi @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +&am33xx_pinmux { + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + BONE_P9_20 0x73 /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) uart1_ctsn.i2c2_sda */ + BONE_P9_19 0x73 /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) uart1_rtsn.i2c2_scl */ + >; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + + status = "okay"; + clock-frequency = <100000>; + + rtc@68 { + compatible = "maxim,ds1307"; + reg = <0x68>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-olimex-som.dts b/arch/arm/boot/dts/am335x-olimex-som.dts new file mode 100644 index 0000000000000..2b00ad28340b1 --- /dev/null +++ b/arch/arm/boot/dts/am335x-olimex-som.dts @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-som-common.dtsi" + +/ { + model = "Olimex AM335x SOM"; + compatible = "olimex,am335x-olimex-som", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&am33xx_pinmux { + lcd_pins_default: lcd_pins_default { + pinctrl-single,pins = < + 0x20 0x01 /* gpmc_ad8.lcd_data16, OUTPUT | MODE1 */ + 0x24 0x01 /* gpmc_ad9.lcd_data17, OUTPUT | MODE1 */ + 0x28 0x01 /* gpmc_ad10.lcd_data18, OUTPUT | MODE1 */ + 0x2c 0x01 /* gpmc_ad11.lcd_data19, OUTPUT | MODE1 */ + 0x30 0x01 /* gpmc_ad12.lcd_data20, OUTPUT | MODE1 */ + 0x34 0x01 /* gpmc_ad13.lcd_data21, OUTPUT | MODE1 */ + 0x38 0x01 /* gpmc_ad14.lcd_data22, OUTPUT | MODE1 */ + 0x3c 0x01 /* gpmc_ad15.lcd_data23, OUTPUT | MODE1 */ + 0xa0 0x00 /* lcd_data0.lcd_data0, OUTPUT | MODE0 */ + 0xa4 0x00 /* lcd_data1.lcd_data1, OUTPUT | MODE0 */ + 0xa8 0x00 /* lcd_data2.lcd_data2, OUTPUT | MODE0 */ + 0xac 0x00 /* lcd_data3.lcd_data3, OUTPUT | MODE0 */ + 0xb0 0x00 /* lcd_data4.lcd_data4, OUTPUT | MODE0 */ + 0xb4 0x00 /* lcd_data5.lcd_data5, OUTPUT | MODE0 */ + 0xb8 0x00 /* lcd_data6.lcd_data6, OUTPUT | MODE0 */ + 0xbc 0x00 /* lcd_data7.lcd_data7, OUTPUT | MODE0 */ + 0xc0 0x00 /* lcd_data8.lcd_data8, OUTPUT | MODE0 */ + 0xc4 0x00 /* lcd_data9.lcd_data9, OUTPUT | MODE0 */ + 0xc8 0x00 /* lcd_data10.lcd_data10, OUTPUT | MODE0 */ + 0xcc 0x00 /* lcd_data11.lcd_data11, OUTPUT | MODE0 */ + 0xd0 0x00 /* lcd_data12.lcd_data12, OUTPUT | MODE0 */ + 0xd4 0x00 /* lcd_data13.lcd_data13, OUTPUT | MODE0 */ + 0xd8 0x00 /* lcd_data14.lcd_data14, OUTPUT | MODE0 */ + 0xdc 0x00 /* lcd_data15.lcd_data15, OUTPUT | MODE0 */ + 0xe0 0x00 /* lcd_vsync.lcd_vsync, OUTPUT | MODE0 */ + 0xe4 0x00 /* lcd_hsync.lcd_hsync, OUTPUT | MODE0 */ + 0xe8 0x00 /* lcd_pclk.lcd_pclk, OUTPUT | MODE0 */ + 0xec 0x00 /* lcd_ac_bias_en.lcd_ac_bias_en, OUTPUT | MODE0 */ + >; + }; + + lcd_pins_sleep: lcd_pins_sleep { + pinctrl-single,pins = < + 0x20 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad8.lcd_data16 */ + 0x24 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad9.lcd_data17 */ + 0x28 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad10.lcd_data18 */ + 0x2c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad11.lcd_data19 */ + 0x30 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad12.lcd_data20 */ + 0x34 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad13.lcd_data21 */ + 0x38 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad14.lcd_data22 */ + 0x3c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad15.lcd_data23 */ + 0xa0 (PULL_DISABLE | MUX_MODE7) /* lcd_data0.lcd_data0 */ + 0xa4 (PULL_DISABLE | MUX_MODE7) /* lcd_data1.lcd_data1 */ + 0xa8 (PULL_DISABLE | MUX_MODE7) /* lcd_data2.lcd_data2 */ + 0xac (PULL_DISABLE | MUX_MODE7) /* lcd_data3.lcd_data3 */ + 0xb0 (PULL_DISABLE | MUX_MODE7) /* lcd_data4.lcd_data4 */ + 0xb4 (PULL_DISABLE | MUX_MODE7) /* lcd_data5.lcd_data5 */ + 0xb8 (PULL_DISABLE | MUX_MODE7) /* lcd_data6.lcd_data6 */ + 0xbc (PULL_DISABLE | MUX_MODE7) /* lcd_data7.lcd_data7 */ + 0xc0 (PULL_DISABLE | MUX_MODE7) /* lcd_data8.lcd_data8 */ + 0xc4 (PULL_DISABLE | MUX_MODE7) /* lcd_data9.lcd_data9 */ + 0xc8 (PULL_DISABLE | MUX_MODE7) /* lcd_data10.lcd_data10 */ + 0xcc (PULL_DISABLE | MUX_MODE7) /* lcd_data11.lcd_data11 */ + 0xd0 (PULL_DISABLE | MUX_MODE7) /* lcd_data12.lcd_data12 */ + 0xd4 (PULL_DISABLE | MUX_MODE7) /* lcd_data13.lcd_data13 */ + 0xd8 (PULL_DISABLE | MUX_MODE7) /* lcd_data14.lcd_data14 */ + 0xdc (PULL_DISABLE | MUX_MODE7) /* lcd_data15.lcd_data15 */ + /* lcd_vsync.lcd_vsync,OUTPUT | MODE0 */ + 0xe0 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0xe4 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_hsync.lcd_hsync */ + 0xe8 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_pclk.lcd_pclk */ + /* lcd_ac_bias_en.lcd_ac_bias_en */ + 0xec (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + +}; + +&lcdc { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&lcd_pins_default>; + pinctrl-1 = <&lcd_pins_sleep>; + status = "okay"; + /* display-timings { + 480x272 { + hactive = <480>; + vactive = <272>; + hback-porch = <43>; + hfront-porch = <8>; + hsync-len = <4>; + vback-porch = <12>; + vfront-porch = <4>; + vsync-len = <10>; + clock-frequency = <9000000>; + hsync-active = <0>; + vsync-active = <0>; + }; + };*/ + + display-timings { + native-mode = <&vga1024x768>; + lcd4: 480x272 { + clock-frequency = <9000000>; + hactive = <480>; + vactive = <272>; + hfront-porch = <3>; + hback-porch = <40>; + vback-porch = <8>; + vfront-porch = <7>; + hsync-len = <2>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + }; + lcd7: 800x480 { + clock-frequency = <33300000>; + hactive = <800>; + vactive = <480>; + hfront-porch = <210>; + hback-porch = <40>; + vback-porch = <23>; + vfront-porch = <20>; + hsync-len = <6>; + vsync-len = <2>; + hsync-active = <0>; + vsync-active = <0>; + }; + lcd10: 1024x600 { + clock-frequency = <51200000>; + hactive = <1024>; + vactive = <600>; + hfront-porch = <160>; + hback-porch = <140>; + vback-porch = <20>; + vfront-porch = <12>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + }; + + vga800x600: 800x600 { + clock-frequency = <40000000>; + hactive = <800>; + vactive = <600>; + hfront-porch = <40>; + hback-porch = <88>; + vfront-porch = <1>; + vback-porch = <23>; + hsync-len = <128>; + vsync-len = <4>; + hsync-active = <0>; + vsync-active = <0>; + }; + vga1024x768: 1024x768 { + clock-frequency = <65000000>; + hactive = <1024>; + hfront-porch = <24>; + hback-porch = <160>; + hsync-len = <136>; + vactive = <768>; + vfront-porch = <3>; + vback-porch = <29>; + vsync-len = <6>; + hsync-active = <0>; + vsync-active = <0>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-can0.dtsi b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi new file mode 100644 index 0000000000000..4335e39bf1999 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&dcan0 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-can1.dtsi b/arch/arm/boot/dts/am335x-peripheral-can1.dtsi new file mode 100644 index 0000000000000..02b5bd17066eb --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-can1.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&dcan1 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-emmc.dtsi b/arch/arm/boot/dts/am335x-peripheral-emmc.dtsi new file mode 100644 index 0000000000000..603f34ec6dc47 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-emmc.dtsi @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + + bus-width = <8>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-i2c2.dtsi b/arch/arm/boot/dts/am335x-peripheral-i2c2.dtsi new file mode 100644 index 0000000000000..ed9a0b52f9135 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-i2c2.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&i2c2 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-nxp-hdmi.dtsi b/arch/arm/boot/dts/am335x-peripheral-nxp-hdmi.dtsi new file mode 100644 index 0000000000000..1dfd26a691a6d --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-nxp-hdmi.dtsi @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&lcdc { + status = "okay"; + port { + lcdc_0: endpoint@0 { + remote-endpoint = <&hdmi_0>; + }; + }; +}; + +&i2c0 { + tda19988 { + compatible = "nxp,tda998x"; + reg = <0x70>; + + port { + hdmi_0: endpoint@0 { + remote-endpoint = <&lcdc_0>; + }; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi new file mode 100644 index 0000000000000..f7c8f8af499fa --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&lcdc { + status = "okay"; + blue-and-red-wiring = "crossed"; +}; + +/ { + panel { + status = "okay"; + compatible = "ti,tilcdc,panel"; + pinctrl-names = "default"; + + panel-info { + ac-bias = <255>; + ac-bias-intrpt = <0>; + dma-burst-sz = <16>; + bpp = <32>; + fdd = <0x80>; + sync-edge = <0>; + sync-ctrl = <0>; + raster-order = <1>; + fifo-th = <0>; + }; + display-timings { + native-mode = <&timing0>; + timing0: 1024x600 { + clock-frequency = <36000000>; + hactive = <1024>; + vactive = <600>; + hfront-porch = <1>; + hback-porch = <45>; + hsync-len = <30>; + vback-porch = <22>; + vfront-porch = <12>; + vsync-len = <2>; + hsync-active = <1>; + vsync-active = <1>; + de-active = <1>; + pixelclk-active = <0>; + }; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-spi0.dtsi b/arch/arm/boot/dts/am335x-peripheral-spi0.dtsi new file mode 100644 index 0000000000000..969e35259979c --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-spi0.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&spi0 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-spi1.dtsi b/arch/arm/boot/dts/am335x-peripheral-spi1.dtsi new file mode 100644 index 0000000000000..ac5fe97b65d99 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-spi1.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&spi1 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-spi1a.dtsi b/arch/arm/boot/dts/am335x-peripheral-spi1a.dtsi new file mode 100644 index 0000000000000..ac5fe97b65d99 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-spi1a.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&spi1 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi new file mode 100644 index 0000000000000..f59fa4c5025fa --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart1 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi new file mode 100644 index 0000000000000..a25d6cfcb1d6a --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart2 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi new file mode 100644 index 0000000000000..adc89f046fbb5 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart4 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS5.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS5.dtsi new file mode 100644 index 0000000000000..8b42fb02b9b4a --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS5.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart5 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-pocketbeagle-common.dtsi b/arch/arm/boot/dts/am335x-pocketbeagle-common.dtsi new file mode 100644 index 0000000000000..fcac7f2ae66d5 --- /dev/null +++ b/arch/arm/boot/dts/am335x-pocketbeagle-common.dtsi @@ -0,0 +1,1181 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "am33xx.dtsi" + +/ { + model = "TI AM335x PocketBeagle"; + compatible = "ti,am335x-pocketbeagle", "ti,am335x-bone", "ti,am33xx"; + + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x20000000>; /* 512 MB */ + }; + + chosen { + stdout-path = &uart0; + base_dtb = "am335x-pocketbeagle.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + leds { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&user_leds_default>; + pinctrl-1 = <&user_leds_sleep>; + + compatible = "gpio-leds"; + + led@2 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led@3 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led@4 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led@5 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&am33xx_pinmux { + user_leds_default: user_leds_default { + pinctrl-single,pins = < + AM33XX_IOPAD(0x0854, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_IOPAD(0x0858, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_IOPAD(0x085c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_IOPAD(0x0860, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + user_leds_sleep: user_leds_sleep { + pinctrl-single,pins = < + AM33XX_IOPAD(0x0854, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_IOPAD(0x0858, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_IOPAD(0x085c, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_IOPAD(0x0860, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x0988, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_IOPAD(0x098c, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + mmc0_pins: pinmux_mmc0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x0960, PIN_INPUT | MUX_MODE7) /* GPIO0_6 */ + >; + }; + + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio2_23 */ + P1_02_default_pin: pinmux_P1_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pin: pinmux_P1_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pu_pin: pinmux_P1_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pd_pin: pinmux_P1_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_input_pin: pinmux_P1_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_pruout_pin: pinmux_P1_02_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P1_02_pruin_pin: pinmux_P1_02_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) gpio2_25 */ + P1_04_default_pin: pinmux_P1_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pin: pinmux_P1_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pu_pin: pinmux_P1_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pd_pin: pinmux_P1_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_input_pin: pinmux_P1_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_pruout_pin: pinmux_P1_04_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P1_04_pruin_pin: pinmux_P1_04_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi0_cs0 */ + P1_06_default_pin: pinmux_P1_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_gpio_pin: pinmux_P1_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pu_pin: pinmux_P1_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pd_pin: pinmux_P1_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_input_pin: pinmux_P1_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_spi_cs_pin: pinmux_P1_06_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_i2c_pin: pinmux_P1_06_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P1_06_pwm_pin: pinmux_P1_06_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P1_06_pru_uart_pin: pinmux_P1_06_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi0_sclk */ + P1_08_default_pin: pinmux_P1_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_gpio_pin: pinmux_P1_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pu_pin: pinmux_P1_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pd_pin: pinmux_P1_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_input_pin: pinmux_P1_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_spi_sclk_pin: pinmux_P1_08_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_uart_pin: pinmux_P1_08_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P1_08_i2c_pin: pinmux_P1_08_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P1_08_pwm_pin: pinmux_P1_08_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P1_08_pru_uart_pin: pinmux_P1_08_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi0_d0 */ + P1_10_default_pin: pinmux_P1_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_gpio_pin: pinmux_P1_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pu_pin: pinmux_P1_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pd_pin: pinmux_P1_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_input_pin: pinmux_P1_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_spi_pin: pinmux_P1_10_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_uart_pin: pinmux_P1_10_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P1_10_i2c_pin: pinmux_P1_10_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P1_10_pwm_pin: pinmux_P1_10_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P1_10_pru_uart_pin: pinmux_P1_10_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi0_d1 */ + P1_12_default_pin: pinmux_P1_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_gpio_pin: pinmux_P1_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pu_pin: pinmux_P1_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pd_pin: pinmux_P1_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_input_pin: pinmux_P1_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_spi_pin: pinmux_P1_12_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_i2c_pin: pinmux_P1_12_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P1_12_pwm_pin: pinmux_P1_12_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P1_12_pru_uart_pin: pinmux_P1_12_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) gpio0_20 */ + P1_20_default_pin: pinmux_P1_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pin: pinmux_P1_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pu_pin: pinmux_P1_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pd_pin: pinmux_P1_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_input_pin: pinmux_P1_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_pruin_pin: pinmux_P1_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c2_sda */ + P1_26_default_pin: pinmux_P1_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_gpio_pin: pinmux_P1_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pu_pin: pinmux_P1_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pd_pin: pinmux_P1_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_input_pin: pinmux_P1_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_can_pin: pinmux_P1_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P1_26_i2c_pin: pinmux_P1_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_spi_cs_pin: pinmux_P1_26_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P1_26_pru_uart_pin: pinmux_P1_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c2_scl */ + P1_28_default_pin: pinmux_P1_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_gpio_pin: pinmux_P1_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pu_pin: pinmux_P1_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pd_pin: pinmux_P1_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_input_pin: pinmux_P1_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_can_pin: pinmux_P1_28_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P1_28_i2c_pin: pinmux_P1_28_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_spi_cs_pin: pinmux_P1_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P1_28_pru_uart_pin: pinmux_P1_28_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P1_29 (ZCZ ball A14) pru0_in7 */ + P1_29_default_pin: pinmux_P1_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + P1_29_gpio_pin: pinmux_P1_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pu_pin: pinmux_P1_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pd_pin: pinmux_P1_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_input_pin: pinmux_P1_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_qep_pin: pinmux_P1_29_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P1_29_pruout_pin: pinmux_P1_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P1_29_pruin_pin: pinmux_P1_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P1_30 (ZCZ ball E16) uart0_txd */ + P1_30_default_pin: pinmux_P1_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_gpio_pin: pinmux_P1_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pu_pin: pinmux_P1_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pd_pin: pinmux_P1_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_input_pin: pinmux_P1_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_uart_pin: pinmux_P1_30_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_spi_cs_pin: pinmux_P1_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_txd.spi1_cs1 */ + P1_30_can_pin: pinmux_P1_30_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_txd.dcan0_rx */ + P1_30_i2c_pin: pinmux_P1_30_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_txd.i2c2_scl */ + P1_30_pruout_pin: pinmux_P1_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_txd.pru1_out15 */ + P1_30_pruin_pin: pinmux_P1_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE6) >; }; /* uart0_txd.pru1_in15 */ + + /* P1_31 (ZCZ ball B12) pru0_in4 */ + P1_31_default_pin: pinmux_P1_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + P1_31_gpio_pin: pinmux_P1_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pu_pin: pinmux_P1_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pd_pin: pinmux_P1_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_input_pin: pinmux_P1_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_qep_pin: pinmux_P1_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P1_31_pruout_pin: pinmux_P1_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P1_31_pruin_pin: pinmux_P1_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P1_32 (ZCZ ball E15) uart0_rxd */ + P1_32_default_pin: pinmux_P1_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_gpio_pin: pinmux_P1_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pu_pin: pinmux_P1_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pd_pin: pinmux_P1_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_input_pin: pinmux_P1_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_uart_pin: pinmux_P1_32_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_spi_cs_pin: pinmux_P1_32_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rxd.spi1_cs0 */ + P1_32_can_pin: pinmux_P1_32_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rxd.dcan0_tx */ + P1_32_i2c_pin: pinmux_P1_32_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rxd.i2c2_sda */ + P1_32_pruout_pin: pinmux_P1_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_rxd.pru1_out14 */ + P1_32_pruin_pin: pinmux_P1_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE6) >; }; /* uart0_rxd.pru1_in14 */ + + /* P1_33 (ZCZ ball B13) pru0_in1 */ + P1_33_default_pin: pinmux_P1_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + P1_33_gpio_pin: pinmux_P1_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pu_pin: pinmux_P1_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pd_pin: pinmux_P1_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_input_pin: pinmux_P1_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_pwm_pin: pinmux_P1_33_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P1_33_spi_pin: pinmux_P1_33_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P1_33_pruout_pin: pinmux_P1_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P1_33_pruin_pin: pinmux_P1_33_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P1_34 (ZCZ ball T11) gpio0_26 */ + P1_34_default_pin: pinmux_P1_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pin: pinmux_P1_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pu_pin: pinmux_P1_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pd_pin: pinmux_P1_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_input_pin: pinmux_P1_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_pwm_pin: pinmux_P1_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P1_35 (ZCZ ball V5) pru1_in10 */ + P1_35_default_pin: pinmux_P1_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + P1_35_gpio_pin: pinmux_P1_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pu_pin: pinmux_P1_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pd_pin: pinmux_P1_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_input_pin: pinmux_P1_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_pruout_pin: pinmux_P1_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P1_35_pruin_pin: pinmux_P1_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P1_36 (ZCZ ball A13) ehrpwm0a */ + P1_36_default_pin: pinmux_P1_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_gpio_pin: pinmux_P1_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pu_pin: pinmux_P1_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pd_pin: pinmux_P1_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_input_pin: pinmux_P1_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_pwm_pin: pinmux_P1_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_spi_sclk_pin: pinmux_P1_36_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P1_36_pruout_pin: pinmux_P1_36_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P1_36_pruin_pin: pinmux_P1_36_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) ehrpwm1a */ + P2_01_default_pin: pinmux_P2_01_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + P2_01_gpio_pin: pinmux_P2_01_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pu_pin: pinmux_P2_01_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pd_pin: pinmux_P2_01_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_input_pin: pinmux_P2_01_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_pwm_pin: pinmux_P2_01_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P2_02 (ZCZ ball V17) gpio1_27 */ + P2_02_default_pin: pinmux_P2_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pin: pinmux_P2_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pu_pin: pinmux_P2_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pd_pin: pinmux_P2_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_input_pin: pinmux_P2_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + + /* P2_03 (ZCZ ball T10) gpio0_23 */ + P2_03_default_pin: pinmux_P2_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pin: pinmux_P2_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pu_pin: pinmux_P2_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pd_pin: pinmux_P2_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_input_pin: pinmux_P2_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_pwm_pin: pinmux_P2_03_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P2_04 (ZCZ ball T16) gpio1_26 */ + P2_04_default_pin: pinmux_P2_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pin: pinmux_P2_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pu_pin: pinmux_P2_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pd_pin: pinmux_P2_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_input_pin: pinmux_P2_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + + /* P2_05 (ZCZ ball T17) uart4_rxd */ + P2_05_default_pin: pinmux_P2_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + P2_05_gpio_pin: pinmux_P2_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pu_pin: pinmux_P2_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pd_pin: pinmux_P2_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_input_pin: pinmux_P2_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_uart_pin: pinmux_P2_05_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P2_06 (ZCZ ball U16) gpio1_25 */ + P2_06_default_pin: pinmux_P2_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pin: pinmux_P2_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pu_pin: pinmux_P2_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pd_pin: pinmux_P2_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_input_pin: pinmux_P2_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + + /* P2_07 (ZCZ ball U17) uart4_txd */ + P2_07_default_pin: pinmux_P2_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + P2_07_gpio_pin: pinmux_P2_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pu_pin: pinmux_P2_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pd_pin: pinmux_P2_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_input_pin: pinmux_P2_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_uart_pin: pinmux_P2_07_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P2_08 (ZCZ ball U18) gpio1_28 */ + P2_08_default_pin: pinmux_P2_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pin: pinmux_P2_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pu_pin: pinmux_P2_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pd_pin: pinmux_P2_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_input_pin: pinmux_P2_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P2_09 (ZCZ ball D15) i2c1_scl */ + P2_09_default_pin: pinmux_P2_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_gpio_pin: pinmux_P2_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pu_pin: pinmux_P2_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pd_pin: pinmux_P2_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_input_pin: pinmux_P2_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_uart_pin: pinmux_P2_09_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P2_09_can_pin: pinmux_P2_09_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P2_09_i2c_pin: pinmux_P2_09_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_pru_uart_pin: pinmux_P2_09_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P2_09_pruin_pin: pinmux_P2_09_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P2_10 (ZCZ ball R14) gpio1_20 */ + P2_10_default_pin: pinmux_P2_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pin: pinmux_P2_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pu_pin: pinmux_P2_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pd_pin: pinmux_P2_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_input_pin: pinmux_P2_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_qep_pin: pinmux_P2_10_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a4.eqep1a_in */ + + /* P2_11 (ZCZ ball D16) i2c1_sda */ + P2_11_default_pin: pinmux_P2_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_gpio_pin: pinmux_P2_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pu_pin: pinmux_P2_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pd_pin: pinmux_P2_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_input_pin: pinmux_P2_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_uart_pin: pinmux_P2_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P2_11_can_pin: pinmux_P2_11_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P2_11_i2c_pin: pinmux_P2_11_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_pru_uart_pin: pinmux_P2_11_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P2_11_pruin_pin: pinmux_P2_11_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) gpio2_1 */ + P2_17_default_pin: pinmux_P2_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pin: pinmux_P2_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pu_pin: pinmux_P2_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pd_pin: pinmux_P2_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_input_pin: pinmux_P2_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P2_18 (ZCZ ball U13) gpio1_15 */ + P2_18_default_pin: pinmux_P2_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pin: pinmux_P2_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pu_pin: pinmux_P2_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pd_pin: pinmux_P2_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_input_pin: pinmux_P2_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_qep_pin: pinmux_P2_18_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P2_18_pru_ecap_pin: pinmux_P2_18_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P2_18_pruin_pin: pinmux_P2_18_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P2_19 (ZCZ ball U12) gpio0_27 */ + P2_19_default_pin: pinmux_P2_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pin: pinmux_P2_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pu_pin: pinmux_P2_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pd_pin: pinmux_P2_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_input_pin: pinmux_P2_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_pwm_pin: pinmux_P2_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P2_20 (ZCZ ball T13) gpio2_0 */ + P2_20_default_pin: pinmux_P2_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pin: pinmux_P2_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pu_pin: pinmux_P2_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pd_pin: pinmux_P2_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_input_pin: pinmux_P2_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) gpio1_14 */ + P2_22_default_pin: pinmux_P2_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pin: pinmux_P2_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pu_pin: pinmux_P2_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pd_pin: pinmux_P2_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_input_pin: pinmux_P2_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_qep_pin: pinmux_P2_22_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P2_22_pruin_pin: pinmux_P2_22_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) gpio1_12 */ + P2_24_default_pin: pinmux_P2_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pin: pinmux_P2_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pu_pin: pinmux_P2_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pd_pin: pinmux_P2_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_input_pin: pinmux_P2_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_qep_pin: pinmux_P2_24_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P2_24_pruout_pin: pinmux_P2_24_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P2_25 (ZCZ ball E17) spi1_d1 */ + P2_25_default_pin: pinmux_P2_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_gpio_pin: pinmux_P2_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pu_pin: pinmux_P2_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pd_pin: pinmux_P2_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_input_pin: pinmux_P2_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_uart_pin: pinmux_P2_25_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rtsn.uart4_txd */ + P2_25_can_pin: pinmux_P2_25_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rtsn.dcan1_rx */ + P2_25_i2c_pin: pinmux_P2_25_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rtsn.i2c1_scl */ + P2_25_spi_pin: pinmux_P2_25_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_spi_cs_pin: pinmux_P2_25_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart0_rtsn.spi1_cs0 */ + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi1_d0 */ + P2_27_default_pin: pinmux_P2_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + P2_27_gpio_pin: pinmux_P2_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pu_pin: pinmux_P2_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pd_pin: pinmux_P2_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_input_pin: pinmux_P2_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_INPUT | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_uart_pin: pinmux_P2_27_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_ctsn.uart4_rxd */ + P2_27_can_pin: pinmux_P2_27_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_ctsn.dcan1_tx */ + P2_27_i2c_pin: pinmux_P2_27_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_ctsn.i2c1_sda */ + P2_27_spi_pin: pinmux_P2_27_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + + /* P2_28 (ZCZ ball D13) pru0_in6 */ + P2_28_default_pin: pinmux_P2_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + P2_28_gpio_pin: pinmux_P2_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pu_pin: pinmux_P2_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pd_pin: pinmux_P2_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_input_pin: pinmux_P2_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_qep_pin: pinmux_P2_28_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P2_28_pruout_pin: pinmux_P2_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P2_28_pruin_pin: pinmux_P2_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P2_29 (ZCZ ball C18) spi1_sclk */ + P2_29_default_pin: pinmux_P2_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + P2_29_gpio_pin: pinmux_P2_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pu_pin: pinmux_P2_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pd_pin: pinmux_P2_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_input_pin: pinmux_P2_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_pwm_pin: pinmux_P2_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P2_29_uart_pin: pinmux_P2_29_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P2_29_spi_cs_pin: pinmux_P2_29_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P2_29_pru_ecap_pin: pinmux_P2_29_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P2_29_spi_sclk_pin: pinmux_P2_29_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P2_30 (ZCZ ball C12) pru0_in3 */ + P2_30_default_pin: pinmux_P2_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + P2_30_gpio_pin: pinmux_P2_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pu_pin: pinmux_P2_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pd_pin: pinmux_P2_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_input_pin: pinmux_P2_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_pwm_pin: pinmux_P2_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P2_30_spi_cs_pin: pinmux_P2_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P2_30_pruout_pin: pinmux_P2_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P2_30_pruin_pin: pinmux_P2_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P2_31 (ZCZ ball A15) spi1_cs1 */ + P2_31_default_pin: pinmux_P2_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_gpio_pin: pinmux_P2_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pu_pin: pinmux_P2_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pd_pin: pinmux_P2_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_input_pin: pinmux_P2_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_spi_cs_pin: pinmux_P2_31_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_pruin_pin: pinmux_P2_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr0.pru1_in16 */ + + /* P2_32 (ZCZ ball D12) pru0_in2 */ + P2_32_default_pin: pinmux_P2_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + P2_32_gpio_pin: pinmux_P2_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pu_pin: pinmux_P2_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pd_pin: pinmux_P2_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_input_pin: pinmux_P2_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_pwm_pin: pinmux_P2_32_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P2_32_spi_pin: pinmux_P2_32_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P2_32_pruout_pin: pinmux_P2_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P2_32_pruin_pin: pinmux_P2_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P2_33 (ZCZ ball R12) gpio1_13 */ + P2_33_default_pin: pinmux_P2_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pin: pinmux_P2_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pu_pin: pinmux_P2_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pd_pin: pinmux_P2_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_input_pin: pinmux_P2_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_qep_pin: pinmux_P2_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P2_33_pruout_pin: pinmux_P2_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P2_34 (ZCZ ball C13) pru0_in5 */ + P2_34_default_pin: pinmux_P2_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + P2_34_gpio_pin: pinmux_P2_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pu_pin: pinmux_P2_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pd_pin: pinmux_P2_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_input_pin: pinmux_P2_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_qep_pin: pinmux_P2_34_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P2_34_pruout_pin: pinmux_P2_34_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P2_34_pruin_pin: pinmux_P2_34_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P2_35 (ZCZ ball U5) gpio2_22 */ + P2_35_default_pin: pinmux_P2_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pin: pinmux_P2_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pu_pin: pinmux_P2_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pd_pin: pinmux_P2_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_input_pin: pinmux_P2_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_pruout_pin: pinmux_P2_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P2_35_pruin_pin: pinmux_P2_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P2_36 (ZCZ ball C9) AIN7 */ +}; + +&uart0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; +}; + +&usb0 { + status = "okay"; + dr_mode = "otg"; +}; + +&usb1_phy { + status = "okay"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; +}; + +&i2c1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + clock-frequency = <400000>; +}; + +&i2c2 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + clock-frequency = <400000>; +}; + +/include/ "tps65217.dtsi" + +&tps { + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&tscadc { + status = "okay"; + adc { + ti,adc-channels = <0 1 2 3 4 5 6 7>; + ti,chan-step-avg = <0x16 0x16 0x16 0x16 0x16 0x16 0x16 0x16>; + ti,chan-step-opendelay = <0x98 0x98 0x98 0x98 0x98 0x98 0x98 0x98>; + ti,chan-step-sampledelay = <0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0>; + }; +}; + +&mmc1 { + status = "okay"; + vmmc-supply = <&vmmcsd_fixed>; + bus-width = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&mmc0_pins>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + system-power-controller; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@0 { +// status = "okay"; +// }; +//}; + +&epwmss0 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&epwmss1 { + status = "okay"; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; diff --git a/arch/arm/boot/dts/am335x-pocketbeagle-gamepup.dts b/arch/arm/boot/dts/am335x-pocketbeagle-gamepup.dts new file mode 100644 index 0000000000000..c90a0245ab828 --- /dev/null +++ b/arch/arm/boot/dts/am335x-pocketbeagle-gamepup.dts @@ -0,0 +1,860 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-pocketbeagle-common.dtsi" + +&sgx { + status = "okay"; +}; + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + adafruit18: adafruit18@0{ + compatible = "sitronix,st7735r"; + pinctrl-names = "default"; + pinctrl-0 = < + &P1_06_spi_cs_pin /* CS */ + &P1_08_spi_sclk_pin /* SCLK */ + &P1_10_spi_pin /* D0 (MOSI?) */ + &P1_12_spi_pin /* D1 (MISO?) */ + &P2_19_gpio_pin /* RST */ + &P2_17_gpio_pin /* DC */ + &P2_01_gpio_pin /* LED */ + >; + reg = <0>; + buswidth = <8>; + reset-gpios = <&gpio0 27 0>; + dc-gpios = <&gpio2 1 0>; + led-gpios = <&gpio1 18 1>; + debug = <1>; + spi-max-frequency = <32000000>; + rotate = <0>; + rgb; + fps = <30>; + }; +}; + +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; + +&ocp { + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio_input */ + P1_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_02_default_pin>; + pinctrl-1 = <&P1_02_gpio_pin>; + pinctrl-2 = <&P1_02_gpio_pu_pin>; + pinctrl-3 = <&P1_02_gpio_pd_pin>; + pinctrl-4 = <&P1_02_gpio_input_pin>; + pinctrl-5 = <&P1_02_pruout_pin>; + pinctrl-6 = <&P1_02_pruin_pin>; + }; + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) */ + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi_cs */ + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi_sclk */ + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi */ + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi */ + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) */ + P1_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin"; + pinctrl-0 = <&P1_20_default_pin>; + pinctrl-1 = <&P1_20_gpio_pin>; + pinctrl-2 = <&P1_20_gpio_pu_pin>; + pinctrl-3 = <&P1_20_gpio_pd_pin>; + pinctrl-4 = <&P1_20_gpio_input_pin>; + pinctrl-5 = <&P1_20_pruin_pin>; + }; + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c */ + P1_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_26_default_pin>; + pinctrl-1 = <&P1_26_gpio_pin>; + pinctrl-2 = <&P1_26_gpio_pu_pin>; + pinctrl-3 = <&P1_26_gpio_pd_pin>; + pinctrl-4 = <&P1_26_gpio_input_pin>; + pinctrl-5 = <&P1_26_spi_cs_pin>; + pinctrl-6 = <&P1_26_can_pin>; + pinctrl-7 = <&P1_26_i2c_pin>; + pinctrl-8 = <&P1_26_pru_uart_pin>; + }; + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c */ + P1_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_28_default_pin>; + pinctrl-1 = <&P1_28_gpio_pin>; + pinctrl-2 = <&P1_28_gpio_pu_pin>; + pinctrl-3 = <&P1_28_gpio_pd_pin>; + pinctrl-4 = <&P1_28_gpio_input_pin>; + pinctrl-5 = <&P1_28_spi_cs_pin>; + pinctrl-6 = <&P1_28_can_pin>; + pinctrl-7 = <&P1_28_i2c_pin>; + pinctrl-8 = <&P1_28_pru_uart_pin>; + }; + + /* P1_29 (ZCZ ball A14) pruin */ + P1_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_29_default_pin>; + pinctrl-1 = <&P1_29_gpio_pin>; + pinctrl-2 = <&P1_29_gpio_pu_pin>; + pinctrl-3 = <&P1_29_gpio_pd_pin>; + pinctrl-4 = <&P1_29_gpio_input_pin>; + pinctrl-5 = <&P1_29_qep_pin>; + pinctrl-6 = <&P1_29_pruout_pin>; + pinctrl-7 = <&P1_29_pruin_pin>; + }; + + /* P1_30 (ZCZ ball E16) uart */ + P1_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_30_default_pin>; + pinctrl-1 = <&P1_30_gpio_pin>; + pinctrl-2 = <&P1_30_gpio_pu_pin>; + pinctrl-3 = <&P1_30_gpio_pd_pin>; + pinctrl-4 = <&P1_30_gpio_input_pin>; + pinctrl-5 = <&P1_30_spi_cs_pin>; + pinctrl-6 = <&P1_30_uart_pin>; + pinctrl-7 = <&P1_30_can_pin>; + pinctrl-8 = <&P1_30_i2c_pin>; + pinctrl-9 = <&P1_30_pruout_pin>; + pinctrl-10 = <&P1_30_pruin_pin>; + }; + + /* P1_31 (ZCZ ball B12) pruin */ + + /* P1_32 (ZCZ ball E15) uart */ + P1_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_32_default_pin>; + pinctrl-1 = <&P1_32_gpio_pin>; + pinctrl-2 = <&P1_32_gpio_pu_pin>; + pinctrl-3 = <&P1_32_gpio_pd_pin>; + pinctrl-4 = <&P1_32_gpio_input_pin>; + pinctrl-5 = <&P1_32_spi_cs_pin>; + pinctrl-6 = <&P1_32_uart_pin>; + pinctrl-7 = <&P1_32_can_pin>; + pinctrl-8 = <&P1_32_i2c_pin>; + pinctrl-9 = <&P1_32_pruout_pin>; + pinctrl-10 = <&P1_32_pruin_pin>; + }; + + /* P1_33 (ZCZ ball B13) pruin */ + P1_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_33_default_pin>; + pinctrl-1 = <&P1_33_gpio_pin>; + pinctrl-2 = <&P1_33_gpio_pu_pin>; + pinctrl-3 = <&P1_33_gpio_pd_pin>; + pinctrl-4 = <&P1_33_gpio_input_pin>; + pinctrl-5 = <&P1_33_spi_pin>; + pinctrl-6 = <&P1_33_pwm_pin>; + pinctrl-7 = <&P1_33_pruout_pin>; + pinctrl-8 = <&P1_33_pruin_pin>; + }; + + /* P1_34 (ZCZ ball T11) */ + P1_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P1_34_default_pin>; + pinctrl-1 = <&P1_34_gpio_pin>; + pinctrl-2 = <&P1_34_gpio_pu_pin>; + pinctrl-3 = <&P1_34_gpio_pd_pin>; + pinctrl-4 = <&P1_34_gpio_input_pin>; + pinctrl-5 = <&P1_34_pwm_pin>; + }; + + /* P1_35 (ZCZ ball V5) pruin */ + + /* P1_36 (ZCZ ball A13) pwm */ + P1_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_36_default_pin>; + pinctrl-1 = <&P1_36_gpio_pin>; + pinctrl-2 = <&P1_36_gpio_pu_pin>; + pinctrl-3 = <&P1_36_gpio_pd_pin>; + pinctrl-4 = <&P1_36_gpio_input_pin>; + pinctrl-5 = <&P1_36_spi_sclk_pin>; + pinctrl-6 = <&P1_36_pwm_pin>; + pinctrl-7 = <&P1_36_pruout_pin>; + pinctrl-8 = <&P1_36_pruin_pin>; + }; + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) pwm */ + + /* P2_02 (ZCZ ball V17) */ + + /* P2_03 (ZCZ ball T10) */ + + /* P2_04 (ZCZ ball T16) */ + + /* P2_05 (ZCZ ball T17) uart */ + P2_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_05_default_pin>; + pinctrl-1 = <&P2_05_gpio_pin>; + pinctrl-2 = <&P2_05_gpio_pu_pin>; + pinctrl-3 = <&P2_05_gpio_pd_pin>; + pinctrl-4 = <&P2_05_gpio_input_pin>; + pinctrl-5 = <&P2_05_uart_pin>; + }; + + /* P2_06 (ZCZ ball U16) */ + + /* P2_07 (ZCZ ball U17) uart */ + P2_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_07_default_pin>; + pinctrl-1 = <&P2_07_gpio_pin>; + pinctrl-2 = <&P2_07_gpio_pu_pin>; + pinctrl-3 = <&P2_07_gpio_pd_pin>; + pinctrl-4 = <&P2_07_gpio_input_pin>; + pinctrl-5 = <&P2_07_uart_pin>; + }; + + /* P2_08 (ZCZ ball U18) */ + + /* P2_09 (ZCZ ball D15) i2c */ + P2_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_09_default_pin>; + pinctrl-1 = <&P2_09_gpio_pin>; + pinctrl-2 = <&P2_09_gpio_pu_pin>; + pinctrl-3 = <&P2_09_gpio_pd_pin>; + pinctrl-4 = <&P2_09_gpio_input_pin>; + pinctrl-5 = <&P2_09_uart_pin>; + pinctrl-6 = <&P2_09_can_pin>; + pinctrl-7 = <&P2_09_i2c_pin>; + pinctrl-8 = <&P2_09_pru_uart_pin>; + pinctrl-9 = <&P2_09_pruin_pin>; + }; + + /* P2_10 (ZCZ ball R14) */ + P2_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P2_10_default_pin>; + pinctrl-1 = <&P2_10_gpio_pin>; + pinctrl-2 = <&P2_10_gpio_pu_pin>; + pinctrl-3 = <&P2_10_gpio_pd_pin>; + pinctrl-4 = <&P2_10_gpio_input_pin>; + pinctrl-5 = <&P2_10_qep_pin>; + }; + + /* P2_11 (ZCZ ball D16) i2c */ + P2_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_11_default_pin>; + pinctrl-1 = <&P2_11_gpio_pin>; + pinctrl-2 = <&P2_11_gpio_pu_pin>; + pinctrl-3 = <&P2_11_gpio_pd_pin>; + pinctrl-4 = <&P2_11_gpio_input_pin>; + pinctrl-5 = <&P2_11_uart_pin>; + pinctrl-6 = <&P2_11_can_pin>; + pinctrl-7 = <&P2_11_i2c_pin>; + pinctrl-8 = <&P2_11_pru_uart_pin>; + pinctrl-9 = <&P2_11_pruin_pin>; + }; + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) */ + + /* P2_18 (ZCZ ball U13) */ + + /* P2_19 (ZCZ ball U12) */ + + /* P2_20 (ZCZ ball T13) */ + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) */ + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) */ + + /* P2_25 (ZCZ ball E17) spi */ + P2_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c"; + pinctrl-0 = <&P2_25_default_pin>; + pinctrl-1 = <&P2_25_gpio_pin>; + pinctrl-2 = <&P2_25_gpio_pu_pin>; + pinctrl-3 = <&P2_25_gpio_pd_pin>; + pinctrl-4 = <&P2_25_gpio_input_pin>; + pinctrl-5 = <&P2_25_spi_pin>; + pinctrl-6 = <&P2_25_spi_cs_pin>; + pinctrl-7 = <&P2_25_uart_pin>; + pinctrl-8 = <&P2_25_can_pin>; + pinctrl-9 = <&P2_25_i2c_pin>; + }; + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi */ + P2_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c"; + pinctrl-0 = <&P2_27_default_pin>; + pinctrl-1 = <&P2_27_gpio_pin>; + pinctrl-2 = <&P2_27_gpio_pu_pin>; + pinctrl-3 = <&P2_27_gpio_pd_pin>; + pinctrl-4 = <&P2_27_gpio_input_pin>; + pinctrl-5 = <&P2_27_spi_pin>; + pinctrl-6 = <&P2_27_uart_pin>; + pinctrl-7 = <&P2_27_can_pin>; + pinctrl-8 = <&P2_27_i2c_pin>; + }; + + /* P2_28 (ZCZ ball D13) pruin */ + P2_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_28_default_pin>; + pinctrl-1 = <&P2_28_gpio_pin>; + pinctrl-2 = <&P2_28_gpio_pu_pin>; + pinctrl-3 = <&P2_28_gpio_pd_pin>; + pinctrl-4 = <&P2_28_gpio_input_pin>; + pinctrl-5 = <&P2_28_qep_pin>; + pinctrl-6 = <&P2_28_pruout_pin>; + pinctrl-7 = <&P2_28_pruin_pin>; + }; + + /* P2_29 (ZCZ ball C18) spi_sclk */ + P2_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P2_29_default_pin>; + pinctrl-1 = <&P2_29_gpio_pin>; + pinctrl-2 = <&P2_29_gpio_pu_pin>; + pinctrl-3 = <&P2_29_gpio_pd_pin>; + pinctrl-4 = <&P2_29_gpio_input_pin>; + pinctrl-5 = <&P2_29_spi_cs_pin>; + pinctrl-6 = <&P2_29_spi_sclk_pin>; + pinctrl-7 = <&P2_29_uart_pin>; + pinctrl-8 = <&P2_29_pwm_pin>; + pinctrl-9 = <&P2_29_pru_ecap_pin>; + }; + + /* P2_30 (ZCZ ball C12) pruin */ + P2_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_30_default_pin>; + pinctrl-1 = <&P2_30_gpio_pin>; + pinctrl-2 = <&P2_30_gpio_pu_pin>; + pinctrl-3 = <&P2_30_gpio_pd_pin>; + pinctrl-4 = <&P2_30_gpio_input_pin>; + pinctrl-5 = <&P2_30_spi_cs_pin>; + pinctrl-6 = <&P2_30_pwm_pin>; + pinctrl-7 = <&P2_30_pruout_pin>; + pinctrl-8 = <&P2_30_pruin_pin>; + }; + + /* P2_31 (ZCZ ball A15) spi_cs */ + P2_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin"; + pinctrl-0 = <&P2_31_default_pin>; + pinctrl-1 = <&P2_31_gpio_pin>; + pinctrl-2 = <&P2_31_gpio_pu_pin>; + pinctrl-3 = <&P2_31_gpio_pd_pin>; + pinctrl-4 = <&P2_31_gpio_input_pin>; + pinctrl-5 = <&P2_31_spi_cs_pin>; + pinctrl-6 = <&P2_31_pruin_pin>; + }; + + /* P2_32 (ZCZ ball D12) pruin */ + P2_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_32_default_pin>; + pinctrl-1 = <&P2_32_gpio_pin>; + pinctrl-2 = <&P2_32_gpio_pu_pin>; + pinctrl-3 = <&P2_32_gpio_pd_pin>; + pinctrl-4 = <&P2_32_gpio_input_pin>; + pinctrl-5 = <&P2_32_spi_pin>; + pinctrl-6 = <&P2_32_pwm_pin>; + pinctrl-7 = <&P2_32_pruout_pin>; + pinctrl-8 = <&P2_32_pruin_pin>; + }; + + /* P2_33 (ZCZ ball R12) */ + P2_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_33_default_pin>; + pinctrl-1 = <&P2_33_gpio_pin>; + pinctrl-2 = <&P2_33_gpio_pu_pin>; + pinctrl-3 = <&P2_33_gpio_pd_pin>; + pinctrl-4 = <&P2_33_gpio_input_pin>; + pinctrl-5 = <&P2_33_qep_pin>; + pinctrl-6 = <&P2_33_pruout_pin>; + }; + + /* P2_34 (ZCZ ball C13) pruin */ + P2_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_34_default_pin>; + pinctrl-1 = <&P2_34_gpio_pin>; + pinctrl-2 = <&P2_34_gpio_pu_pin>; + pinctrl-3 = <&P2_34_gpio_pd_pin>; + pinctrl-4 = <&P2_34_gpio_input_pin>; + pinctrl-5 = <&P2_34_qep_pin>; + pinctrl-6 = <&P2_34_pruout_pin>; + pinctrl-7 = <&P2_34_pruin_pin>; + }; + + /* P2_35 (ZCZ ball U5) gpio_input */ + P2_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P2_35_default_pin>; + pinctrl-1 = <&P2_35_gpio_pin>; + pinctrl-2 = <&P2_35_gpio_pu_pin>; + pinctrl-3 = <&P2_35_gpio_pd_pin>; + pinctrl-4 = <&P2_35_gpio_input_pin>; + pinctrl-5 = <&P2_35_pruout_pin>; + pinctrl-6 = <&P2_35_pruin_pin>; + }; + + /* P2_36 (ZCZ ball C9) AIN7 */ + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P1_02 { + gpio-name = "P1_02"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P1_20 { + gpio-name = "P1_20"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P1_26 { + gpio-name = "P1_26"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P1_28 { + gpio-name = "P1_28"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P1_29 { + gpio-name = "P1_29"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P1_30 { + gpio-name = "P1_30"; + gpio = <&gpio1 11 0>; + input; + dir-changeable; + }; + + P1_32 { + gpio-name = "P1_32"; + gpio = <&gpio1 10 0>; + input; + dir-changeable; + }; + + P1_33 { + gpio-name = "P1_33"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P1_34 { + gpio-name = "P1_34"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P1_36 { + gpio-name = "P1_36"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P2_05 { + gpio-name = "P2_05"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P2_07 { + gpio-name = "P2_07"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P2_09 { + gpio-name = "P2_09"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P2_10 { + gpio-name = "P2_10"; + gpio = <&gpio1 20 0>; + input; + dir-changeable; + }; + + P2_11 { + gpio-name = "P2_11"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P2_17 { + gpio-name = "P2_17"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P2_19 { + gpio-name = "P2_19"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P2_25 { + gpio-name = "P2_25"; + gpio = <&gpio1 9 0>; + input; + dir-changeable; + }; + + P2_27 { + gpio-name = "P2_27"; + gpio = <&gpio1 8 0>; + input; + dir-changeable; + }; + + P2_28 { + gpio-name = "P2_28"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P2_29 { + gpio-name = "P2_29"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P2_30 { + gpio-name = "P2_30"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P2_31 { + gpio-name = "P2_31"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + + P2_32 { + gpio-name = "P2_32"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P2_33 { + gpio-name = "P2_33"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P2_34 { + gpio-name = "P2_34"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P2_35 { + gpio-name = "P2_35"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + }; +}; + + +/ { + /* + * Load the drivers for the buttons and LEDs. + */ + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = < + &P2_02_gpio_pu_pin /* left-pad down */ + &P2_04_gpio_pu_pin /* left-pad right */ + &P2_06_gpio_pu_pin /* left-pad up */ + &P2_08_gpio_pu_pin /* left-pad left */ + &P2_18_gpio_pu_pin /* right-pad down */ + &P2_20_gpio_pu_pin /* right-pad right */ + &P2_22_gpio_pu_pin /* right-pad up */ + &P2_24_gpio_pu_pin /* right-pad left */ + &P1_31_gpio_pu_pin /* select */ + &P1_35_gpio_pu_pin /* start */ + >; + + lp-left { + label = "left"; + linux,code = <105>; + gpios = <&gpio1 28 GPIO_ACTIVE_LOW>; + }; + + lp-right { + label = "right"; + linux,code = <106>; + gpios = <&gpio1 26 GPIO_ACTIVE_LOW>; + }; + + lp-up { + label = "up"; + linux,code = <103>; + gpios = <&gpio1 25 GPIO_ACTIVE_LOW>; + }; + + lp-down { + label = "down"; + linux,code = <108>; + gpios = <&gpio1 27 GPIO_ACTIVE_LOW>; + }; + + rp-left { + label = "p"; + linux,code = <25>; + gpios = <&gpio1 12 GPIO_ACTIVE_LOW>; + }; + + rp-right { + label = "tab"; + linux,code = <15>; + gpios = <&gpio2 0 GPIO_ACTIVE_LOW>; + }; + + rp-up { + label = "esc"; + linux,code = <1>; + gpios = <&gpio1 14 GPIO_ACTIVE_LOW>; + }; + + rp-down { + label = "enter"; + linux,code = <28>; + gpios = <&gpio1 15 GPIO_ACTIVE_LOW>; + }; + + select { + label = "five"; + linux,code = <6>; + gpios = <&gpio3 18 GPIO_ACTIVE_LOW>; + }; + + start { + label = "one"; + linux,code = <2>; + gpios = <&gpio2 24 GPIO_ACTIVE_LOW>; + }; + }; + + capeleds { + compatible = "gpio-leds"; + + pinctrl-names = "default"; + pinctrl-0 = < + &P1_04_gpio_pin /* left */ + &P2_03_gpio_pin /* right */ + >; + + led@1 { + label = "gamepup::left"; + gpios = <&gpio2 25 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led@2 { + label = "gamepup::right"; + gpios = <&gpio0 23 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-pocketbeagle-techlab.dts b/arch/arm/boot/dts/am335x-pocketbeagle-techlab.dts new file mode 100644 index 0000000000000..6519615443c09 --- /dev/null +++ b/arch/arm/boot/dts/am335x-pocketbeagle-techlab.dts @@ -0,0 +1,1228 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-pocketbeagle-common.dtsi" + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.1"; + reg = <1>; + spi-max-frequency = <24000000>; + status = "disabled"; + }; +}; + +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + sevenseg: mcp23s18@1 { + #address-cells = <1>; + #size-cells = <1>; + #gpio-cells = <2>; + compatible = "microchip,mcp23s18"; + gpio-controller; + microchip,spi-present-mask = <0x01>; + reg = <1>; + spi-max-frequency = <1000000>; + }; +}; + +&i2c2 { + #address-cells = <1>; + #size-cells = <0>; + mma8453@1c { + status = "okay"; + compatible = "fsl,mma8453"; + reg = <0x1c>; + }; +}; + +/ { + techlab_sevenseg { + compatible = "gpio-leds"; + + techlab_sevenseg@0 { + label = "techlab::seg0"; + gpios = <&sevenseg 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@1 { + label = "techlab::seg1"; + gpios = <&sevenseg 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@2 { + label = "techlab::seg2"; + gpios = <&sevenseg 2 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@3 { + label = "techlab::seg3"; + gpios = <&sevenseg 3 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@4 { + label = "techlab::seg4"; + gpios = <&sevenseg 4 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@5 { + label = "techlab::seg5"; + gpios = <&sevenseg 5 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@6 { + label = "techlab::seg6"; + gpios = <&sevenseg 6 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@7 { + label = "techlab::seg7"; + gpios = <&sevenseg 7 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@8 { + label = "techlab::seg8"; + gpios = <&sevenseg 8 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@9 { + label = "techlab::seg9"; + gpios = <&sevenseg 9 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@10 { + label = "techlab::seg10"; + gpios = <&sevenseg 10 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@11 { + label = "techlab::seg11"; + gpios = <&sevenseg 11 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@12 { + label = "techlab::seg12"; + gpios = <&sevenseg 12 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@13 { + label = "techlab::seg13"; + gpios = <&sevenseg 13 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@14 { + label = "techlab::seg14"; + gpios = <&sevenseg 14 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + techlab_sevenseg@15 { + label = "techlab::seg15"; + gpios = <&sevenseg 15 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + }; + + techlab_rgb { + compatible = "pwm-leds"; + + techlab_pwm_red { + // P1_33 + // ehrpwm0B + //gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>; + //default-state = "off"; + label = "techlab::red"; + pwms = <&ehrpwm0 1 500000 0>; + max-brightness = <255>; + }; + + + techlab_pwm_green { + // P2_1 + // ehrpwm1A + //gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; + //default-state = "off"; + label = "techlab::green"; + pwms = <&ehrpwm1 0 500000 0>; + max-brightness = <255>; + }; + + techlab_pwm_blue { + // P1_36 + // ehrpwm0A + //gpios = <&gpio3 14 GPIO_ACTIVE_HIGH>; + //default-state = "off"; + label = "techlab::blue"; + pwms = <&ehrpwm0 0 500000 0>; + max-brightness = <255>; + }; + }; +}; + +&ocp { + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio_input */ + P1_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_02_default_pin>; + pinctrl-1 = <&P1_02_gpio_pin>; + pinctrl-2 = <&P1_02_gpio_pu_pin>; + pinctrl-3 = <&P1_02_gpio_pd_pin>; + pinctrl-4 = <&P1_02_gpio_input_pin>; + pinctrl-5 = <&P1_02_pruout_pin>; + pinctrl-6 = <&P1_02_pruin_pin>; + }; + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) */ + P1_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_04_default_pin>; + pinctrl-1 = <&P1_04_gpio_pin>; + pinctrl-2 = <&P1_04_gpio_pu_pin>; + pinctrl-3 = <&P1_04_gpio_pd_pin>; + pinctrl-4 = <&P1_04_gpio_input_pin>; + pinctrl-5 = <&P1_04_pruout_pin>; + pinctrl-6 = <&P1_04_pruin_pin>; + }; + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi_cs */ + P1_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_06_default_pin>; + pinctrl-1 = <&P1_06_gpio_pin>; + pinctrl-2 = <&P1_06_gpio_pu_pin>; + pinctrl-3 = <&P1_06_gpio_pd_pin>; + pinctrl-4 = <&P1_06_gpio_input_pin>; + pinctrl-5 = <&P1_06_spi_cs_pin>; + pinctrl-6 = <&P1_06_i2c_pin>; + pinctrl-7 = <&P1_06_pwm_pin>; + pinctrl-8 = <&P1_06_pru_uart_pin>; + }; + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi_sclk */ + P1_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_08_default_pin>; + pinctrl-1 = <&P1_08_gpio_pin>; + pinctrl-2 = <&P1_08_gpio_pu_pin>; + pinctrl-3 = <&P1_08_gpio_pd_pin>; + pinctrl-4 = <&P1_08_gpio_input_pin>; + pinctrl-5 = <&P1_08_spi_sclk_pin>; + pinctrl-6 = <&P1_08_uart_pin>; + pinctrl-7 = <&P1_08_i2c_pin>; + pinctrl-8 = <&P1_08_pwm_pin>; + pinctrl-9 = <&P1_08_pru_uart_pin>; + }; + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi */ + P1_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_10_default_pin>; + pinctrl-1 = <&P1_10_gpio_pin>; + pinctrl-2 = <&P1_10_gpio_pu_pin>; + pinctrl-3 = <&P1_10_gpio_pd_pin>; + pinctrl-4 = <&P1_10_gpio_input_pin>; + pinctrl-5 = <&P1_10_spi_pin>; + pinctrl-6 = <&P1_10_uart_pin>; + pinctrl-7 = <&P1_10_i2c_pin>; + pinctrl-8 = <&P1_10_pwm_pin>; + pinctrl-9 = <&P1_10_pru_uart_pin>; + }; + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi */ + P1_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_12_default_pin>; + pinctrl-1 = <&P1_12_gpio_pin>; + pinctrl-2 = <&P1_12_gpio_pu_pin>; + pinctrl-3 = <&P1_12_gpio_pd_pin>; + pinctrl-4 = <&P1_12_gpio_input_pin>; + pinctrl-5 = <&P1_12_spi_pin>; + pinctrl-6 = <&P1_12_i2c_pin>; + pinctrl-7 = <&P1_12_pwm_pin>; + pinctrl-8 = <&P1_12_pru_uart_pin>; + }; + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) */ + P1_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin"; + pinctrl-0 = <&P1_20_default_pin>; + pinctrl-1 = <&P1_20_gpio_pin>; + pinctrl-2 = <&P1_20_gpio_pu_pin>; + pinctrl-3 = <&P1_20_gpio_pd_pin>; + pinctrl-4 = <&P1_20_gpio_input_pin>; + pinctrl-5 = <&P1_20_pruin_pin>; + }; + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c */ + P1_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_26_default_pin>; + pinctrl-1 = <&P1_26_gpio_pin>; + pinctrl-2 = <&P1_26_gpio_pu_pin>; + pinctrl-3 = <&P1_26_gpio_pd_pin>; + pinctrl-4 = <&P1_26_gpio_input_pin>; + pinctrl-5 = <&P1_26_spi_cs_pin>; + pinctrl-6 = <&P1_26_can_pin>; + pinctrl-7 = <&P1_26_i2c_pin>; + pinctrl-8 = <&P1_26_pru_uart_pin>; + }; + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c */ + P1_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_28_default_pin>; + pinctrl-1 = <&P1_28_gpio_pin>; + pinctrl-2 = <&P1_28_gpio_pu_pin>; + pinctrl-3 = <&P1_28_gpio_pd_pin>; + pinctrl-4 = <&P1_28_gpio_input_pin>; + pinctrl-5 = <&P1_28_spi_cs_pin>; + pinctrl-6 = <&P1_28_can_pin>; + pinctrl-7 = <&P1_28_i2c_pin>; + pinctrl-8 = <&P1_28_pru_uart_pin>; + }; + + /* P1_29 (ZCZ ball A14) pruin */ + P1_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_29_default_pin>; + pinctrl-1 = <&P1_29_gpio_pin>; + pinctrl-2 = <&P1_29_gpio_pu_pin>; + pinctrl-3 = <&P1_29_gpio_pd_pin>; + pinctrl-4 = <&P1_29_gpio_input_pin>; + pinctrl-5 = <&P1_29_qep_pin>; + pinctrl-6 = <&P1_29_pruout_pin>; + pinctrl-7 = <&P1_29_pruin_pin>; + }; + + /* P1_30 (ZCZ ball E16) uart */ + P1_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_30_default_pin>; + pinctrl-1 = <&P1_30_gpio_pin>; + pinctrl-2 = <&P1_30_gpio_pu_pin>; + pinctrl-3 = <&P1_30_gpio_pd_pin>; + pinctrl-4 = <&P1_30_gpio_input_pin>; + pinctrl-5 = <&P1_30_spi_cs_pin>; + pinctrl-6 = <&P1_30_uart_pin>; + pinctrl-7 = <&P1_30_can_pin>; + pinctrl-8 = <&P1_30_i2c_pin>; + pinctrl-9 = <&P1_30_pruout_pin>; + pinctrl-10 = <&P1_30_pruin_pin>; + }; + + /* P1_31 (ZCZ ball B12) pruin */ + P1_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_31_default_pin>; + pinctrl-1 = <&P1_31_gpio_pin>; + pinctrl-2 = <&P1_31_gpio_pu_pin>; + pinctrl-3 = <&P1_31_gpio_pd_pin>; + pinctrl-4 = <&P1_31_gpio_input_pin>; + pinctrl-5 = <&P1_31_qep_pin>; + pinctrl-6 = <&P1_31_pruout_pin>; + pinctrl-7 = <&P1_31_pruin_pin>; + }; + + /* P1_32 (ZCZ ball E15) uart */ + P1_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_32_default_pin>; + pinctrl-1 = <&P1_32_gpio_pin>; + pinctrl-2 = <&P1_32_gpio_pu_pin>; + pinctrl-3 = <&P1_32_gpio_pd_pin>; + pinctrl-4 = <&P1_32_gpio_input_pin>; + pinctrl-5 = <&P1_32_spi_cs_pin>; + pinctrl-6 = <&P1_32_uart_pin>; + pinctrl-7 = <&P1_32_can_pin>; + pinctrl-8 = <&P1_32_i2c_pin>; + pinctrl-9 = <&P1_32_pruout_pin>; + pinctrl-10 = <&P1_32_pruin_pin>; + }; + + /* P1_33 (ZCZ ball B13) pruin */ + P1_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_33_default_pin>; + pinctrl-1 = <&P1_33_gpio_pin>; + pinctrl-2 = <&P1_33_gpio_pu_pin>; + pinctrl-3 = <&P1_33_gpio_pd_pin>; + pinctrl-4 = <&P1_33_gpio_input_pin>; + pinctrl-5 = <&P1_33_spi_pin>; + pinctrl-6 = <&P1_33_pwm_pin>; + pinctrl-7 = <&P1_33_pruout_pin>; + pinctrl-8 = <&P1_33_pruin_pin>; + }; + + /* P1_34 (ZCZ ball T11) */ + P1_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P1_34_default_pin>; + pinctrl-1 = <&P1_34_gpio_pin>; + pinctrl-2 = <&P1_34_gpio_pu_pin>; + pinctrl-3 = <&P1_34_gpio_pd_pin>; + pinctrl-4 = <&P1_34_gpio_input_pin>; + pinctrl-5 = <&P1_34_pwm_pin>; + }; + + /* P1_35 (ZCZ ball V5) pruin */ + P1_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_35_default_pin>; + pinctrl-1 = <&P1_35_gpio_pin>; + pinctrl-2 = <&P1_35_gpio_pu_pin>; + pinctrl-3 = <&P1_35_gpio_pd_pin>; + pinctrl-4 = <&P1_35_gpio_input_pin>; + pinctrl-5 = <&P1_35_pruout_pin>; + pinctrl-6 = <&P1_35_pruin_pin>; + }; + + /* P1_36 (ZCZ ball A13) pwm */ + P1_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_36_default_pin>; + pinctrl-1 = <&P1_36_gpio_pin>; + pinctrl-2 = <&P1_36_gpio_pu_pin>; + pinctrl-3 = <&P1_36_gpio_pd_pin>; + pinctrl-4 = <&P1_36_gpio_input_pin>; + pinctrl-5 = <&P1_36_spi_sclk_pin>; + pinctrl-6 = <&P1_36_pwm_pin>; + pinctrl-7 = <&P1_36_pruout_pin>; + pinctrl-8 = <&P1_36_pruin_pin>; + }; + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) pwm */ + P2_01_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_01_default_pin>; + pinctrl-1 = <&P2_01_gpio_pin>; + pinctrl-2 = <&P2_01_gpio_pu_pin>; + pinctrl-3 = <&P2_01_gpio_pd_pin>; + pinctrl-4 = <&P2_01_gpio_input_pin>; + pinctrl-5 = <&P2_01_pwm_pin>; + }; + + /* P2_02 (ZCZ ball V17) */ + P2_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_02_default_pin>; + pinctrl-1 = <&P2_02_gpio_pin>; + pinctrl-2 = <&P2_02_gpio_pu_pin>; + pinctrl-3 = <&P2_02_gpio_pd_pin>; + pinctrl-4 = <&P2_02_gpio_input_pin>; + }; + + /* P2_03 (ZCZ ball T10) */ + P2_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_03_default_pin>; + pinctrl-1 = <&P2_03_gpio_pin>; + pinctrl-2 = <&P2_03_gpio_pu_pin>; + pinctrl-3 = <&P2_03_gpio_pd_pin>; + pinctrl-4 = <&P2_03_gpio_input_pin>; + pinctrl-5 = <&P2_03_pwm_pin>; + }; + + /* P2_04 (ZCZ ball T16) */ + P2_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_04_default_pin>; + pinctrl-1 = <&P2_04_gpio_pin>; + pinctrl-2 = <&P2_04_gpio_pu_pin>; + pinctrl-3 = <&P2_04_gpio_pd_pin>; + pinctrl-4 = <&P2_04_gpio_input_pin>; + }; + + /* P2_05 (ZCZ ball T17) uart */ + P2_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_05_default_pin>; + pinctrl-1 = <&P2_05_gpio_pin>; + pinctrl-2 = <&P2_05_gpio_pu_pin>; + pinctrl-3 = <&P2_05_gpio_pd_pin>; + pinctrl-4 = <&P2_05_gpio_input_pin>; + pinctrl-5 = <&P2_05_uart_pin>; + }; + + /* P2_06 (ZCZ ball U16) */ + P2_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_06_default_pin>; + pinctrl-1 = <&P2_06_gpio_pin>; + pinctrl-2 = <&P2_06_gpio_pu_pin>; + pinctrl-3 = <&P2_06_gpio_pd_pin>; + pinctrl-4 = <&P2_06_gpio_input_pin>; + }; + + /* P2_07 (ZCZ ball U17) uart */ + P2_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_07_default_pin>; + pinctrl-1 = <&P2_07_gpio_pin>; + pinctrl-2 = <&P2_07_gpio_pu_pin>; + pinctrl-3 = <&P2_07_gpio_pd_pin>; + pinctrl-4 = <&P2_07_gpio_input_pin>; + pinctrl-5 = <&P2_07_uart_pin>; + }; + + /* P2_08 (ZCZ ball U18) */ + P2_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_08_default_pin>; + pinctrl-1 = <&P2_08_gpio_pin>; + pinctrl-2 = <&P2_08_gpio_pu_pin>; + pinctrl-3 = <&P2_08_gpio_pd_pin>; + pinctrl-4 = <&P2_08_gpio_input_pin>; + }; + + /* P2_09 (ZCZ ball D15) i2c */ + P2_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_09_default_pin>; + pinctrl-1 = <&P2_09_gpio_pin>; + pinctrl-2 = <&P2_09_gpio_pu_pin>; + pinctrl-3 = <&P2_09_gpio_pd_pin>; + pinctrl-4 = <&P2_09_gpio_input_pin>; + pinctrl-5 = <&P2_09_uart_pin>; + pinctrl-6 = <&P2_09_can_pin>; + pinctrl-7 = <&P2_09_i2c_pin>; + pinctrl-8 = <&P2_09_pru_uart_pin>; + pinctrl-9 = <&P2_09_pruin_pin>; + }; + + /* P2_10 (ZCZ ball R14) */ + P2_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P2_10_default_pin>; + pinctrl-1 = <&P2_10_gpio_pin>; + pinctrl-2 = <&P2_10_gpio_pu_pin>; + pinctrl-3 = <&P2_10_gpio_pd_pin>; + pinctrl-4 = <&P2_10_gpio_input_pin>; + pinctrl-5 = <&P2_10_qep_pin>; + }; + + /* P2_11 (ZCZ ball D16) i2c */ + P2_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_11_default_pin>; + pinctrl-1 = <&P2_11_gpio_pin>; + pinctrl-2 = <&P2_11_gpio_pu_pin>; + pinctrl-3 = <&P2_11_gpio_pd_pin>; + pinctrl-4 = <&P2_11_gpio_input_pin>; + pinctrl-5 = <&P2_11_uart_pin>; + pinctrl-6 = <&P2_11_can_pin>; + pinctrl-7 = <&P2_11_i2c_pin>; + pinctrl-8 = <&P2_11_pru_uart_pin>; + pinctrl-9 = <&P2_11_pruin_pin>; + }; + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) */ + P2_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_17_default_pin>; + pinctrl-1 = <&P2_17_gpio_pin>; + pinctrl-2 = <&P2_17_gpio_pu_pin>; + pinctrl-3 = <&P2_17_gpio_pd_pin>; + pinctrl-4 = <&P2_17_gpio_input_pin>; + }; + + /* P2_18 (ZCZ ball U13) */ + P2_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P2_18_default_pin>; + pinctrl-1 = <&P2_18_gpio_pin>; + pinctrl-2 = <&P2_18_gpio_pu_pin>; + pinctrl-3 = <&P2_18_gpio_pd_pin>; + pinctrl-4 = <&P2_18_gpio_input_pin>; + pinctrl-5 = <&P2_18_qep_pin>; + pinctrl-6 = <&P2_18_pru_ecap_pin>; + pinctrl-7 = <&P2_18_pruin_pin>; + }; + + /* P2_19 (ZCZ ball U12) */ + P2_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_19_default_pin>; + pinctrl-1 = <&P2_19_gpio_pin>; + pinctrl-2 = <&P2_19_gpio_pu_pin>; + pinctrl-3 = <&P2_19_gpio_pd_pin>; + pinctrl-4 = <&P2_19_gpio_input_pin>; + pinctrl-5 = <&P2_19_pwm_pin>; + }; + + /* P2_20 (ZCZ ball T13) */ + P2_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_20_default_pin>; + pinctrl-1 = <&P2_20_gpio_pin>; + pinctrl-2 = <&P2_20_gpio_pu_pin>; + pinctrl-3 = <&P2_20_gpio_pd_pin>; + pinctrl-4 = <&P2_20_gpio_input_pin>; + }; + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) */ + P2_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P2_22_default_pin>; + pinctrl-1 = <&P2_22_gpio_pin>; + pinctrl-2 = <&P2_22_gpio_pu_pin>; + pinctrl-3 = <&P2_22_gpio_pd_pin>; + pinctrl-4 = <&P2_22_gpio_input_pin>; + pinctrl-5 = <&P2_22_qep_pin>; + pinctrl-6 = <&P2_22_pruin_pin>; + }; + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) */ + P2_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_24_default_pin>; + pinctrl-1 = <&P2_24_gpio_pin>; + pinctrl-2 = <&P2_24_gpio_pu_pin>; + pinctrl-3 = <&P2_24_gpio_pd_pin>; + pinctrl-4 = <&P2_24_gpio_input_pin>; + pinctrl-5 = <&P2_24_qep_pin>; + pinctrl-6 = <&P2_24_pruout_pin>; + }; + + /* P2_25 (ZCZ ball E17) spi */ + P2_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c"; + pinctrl-0 = <&P2_25_default_pin>; + pinctrl-1 = <&P2_25_gpio_pin>; + pinctrl-2 = <&P2_25_gpio_pu_pin>; + pinctrl-3 = <&P2_25_gpio_pd_pin>; + pinctrl-4 = <&P2_25_gpio_input_pin>; + pinctrl-5 = <&P2_25_spi_pin>; + pinctrl-6 = <&P2_25_spi_cs_pin>; + pinctrl-7 = <&P2_25_uart_pin>; + pinctrl-8 = <&P2_25_can_pin>; + pinctrl-9 = <&P2_25_i2c_pin>; + }; + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi */ + P2_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c"; + pinctrl-0 = <&P2_27_default_pin>; + pinctrl-1 = <&P2_27_gpio_pin>; + pinctrl-2 = <&P2_27_gpio_pu_pin>; + pinctrl-3 = <&P2_27_gpio_pd_pin>; + pinctrl-4 = <&P2_27_gpio_input_pin>; + pinctrl-5 = <&P2_27_spi_pin>; + pinctrl-6 = <&P2_27_uart_pin>; + pinctrl-7 = <&P2_27_can_pin>; + pinctrl-8 = <&P2_27_i2c_pin>; + }; + + /* P2_28 (ZCZ ball D13) pruin */ + P2_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_28_default_pin>; + pinctrl-1 = <&P2_28_gpio_pin>; + pinctrl-2 = <&P2_28_gpio_pu_pin>; + pinctrl-3 = <&P2_28_gpio_pd_pin>; + pinctrl-4 = <&P2_28_gpio_input_pin>; + pinctrl-5 = <&P2_28_qep_pin>; + pinctrl-6 = <&P2_28_pruout_pin>; + pinctrl-7 = <&P2_28_pruin_pin>; + }; + + /* P2_29 (ZCZ ball C18) spi_sclk */ + P2_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P2_29_default_pin>; + pinctrl-1 = <&P2_29_gpio_pin>; + pinctrl-2 = <&P2_29_gpio_pu_pin>; + pinctrl-3 = <&P2_29_gpio_pd_pin>; + pinctrl-4 = <&P2_29_gpio_input_pin>; + pinctrl-5 = <&P2_29_spi_cs_pin>; + pinctrl-6 = <&P2_29_spi_sclk_pin>; + pinctrl-7 = <&P2_29_uart_pin>; + pinctrl-8 = <&P2_29_pwm_pin>; + pinctrl-9 = <&P2_29_pru_ecap_pin>; + }; + + /* P2_30 (ZCZ ball C12) pruin */ + P2_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_30_default_pin>; + pinctrl-1 = <&P2_30_gpio_pin>; + pinctrl-2 = <&P2_30_gpio_pu_pin>; + pinctrl-3 = <&P2_30_gpio_pd_pin>; + pinctrl-4 = <&P2_30_gpio_input_pin>; + pinctrl-5 = <&P2_30_spi_cs_pin>; + pinctrl-6 = <&P2_30_pwm_pin>; + pinctrl-7 = <&P2_30_pruout_pin>; + pinctrl-8 = <&P2_30_pruin_pin>; + }; + + /* P2_31 (ZCZ ball A15) spi_cs */ + P2_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin"; + pinctrl-0 = <&P2_31_default_pin>; + pinctrl-1 = <&P2_31_gpio_pin>; + pinctrl-2 = <&P2_31_gpio_pu_pin>; + pinctrl-3 = <&P2_31_gpio_pd_pin>; + pinctrl-4 = <&P2_31_gpio_input_pin>; + pinctrl-5 = <&P2_31_spi_cs_pin>; + pinctrl-6 = <&P2_31_pruin_pin>; + }; + + /* P2_32 (ZCZ ball D12) pruin */ + P2_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_32_default_pin>; + pinctrl-1 = <&P2_32_gpio_pin>; + pinctrl-2 = <&P2_32_gpio_pu_pin>; + pinctrl-3 = <&P2_32_gpio_pd_pin>; + pinctrl-4 = <&P2_32_gpio_input_pin>; + pinctrl-5 = <&P2_32_spi_pin>; + pinctrl-6 = <&P2_32_pwm_pin>; + pinctrl-7 = <&P2_32_pruout_pin>; + pinctrl-8 = <&P2_32_pruin_pin>; + }; + + /* P2_33 (ZCZ ball R12) */ + P2_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_33_default_pin>; + pinctrl-1 = <&P2_33_gpio_pin>; + pinctrl-2 = <&P2_33_gpio_pu_pin>; + pinctrl-3 = <&P2_33_gpio_pd_pin>; + pinctrl-4 = <&P2_33_gpio_input_pin>; + pinctrl-5 = <&P2_33_qep_pin>; + pinctrl-6 = <&P2_33_pruout_pin>; + }; + + /* P2_34 (ZCZ ball C13) pruin */ + P2_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_34_default_pin>; + pinctrl-1 = <&P2_34_gpio_pin>; + pinctrl-2 = <&P2_34_gpio_pu_pin>; + pinctrl-3 = <&P2_34_gpio_pd_pin>; + pinctrl-4 = <&P2_34_gpio_input_pin>; + pinctrl-5 = <&P2_34_qep_pin>; + pinctrl-6 = <&P2_34_pruout_pin>; + pinctrl-7 = <&P2_34_pruin_pin>; + }; + + /* P2_35 (ZCZ ball U5) gpio_input */ + P2_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P2_35_default_pin>; + pinctrl-1 = <&P2_35_gpio_pin>; + pinctrl-2 = <&P2_35_gpio_pu_pin>; + pinctrl-3 = <&P2_35_gpio_pd_pin>; + pinctrl-4 = <&P2_35_gpio_input_pin>; + pinctrl-5 = <&P2_35_pruout_pin>; + pinctrl-6 = <&P2_35_pruin_pin>; + }; + + /* P2_36 (ZCZ ball C9) AIN7 */ + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P1_02 { + gpio-name = "P1_02"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P1_04 { + gpio-name = "P1_04"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P1_06 { + gpio-name = "P1_06"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P1_08 { + gpio-name = "P1_08"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P1_10 { + gpio-name = "P1_10"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P1_12 { + gpio-name = "P1_12"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P1_20 { + gpio-name = "P1_20"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P1_26 { + gpio-name = "P1_26"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P1_28 { + gpio-name = "P1_28"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P1_29 { + gpio-name = "P1_29"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P1_30 { + gpio-name = "P1_30"; + gpio = <&gpio1 11 0>; + input; + dir-changeable; + }; + + P1_31 { + gpio-name = "P1_31"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + P1_32 { + gpio-name = "P1_32"; + gpio = <&gpio1 10 0>; + input; + dir-changeable; + }; + + P1_33 { + gpio-name = "P1_33"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P1_34 { + gpio-name = "P1_34"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P1_35 { + gpio-name = "P1_35"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P1_36 { + gpio-name = "P1_36"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P2_01 { + gpio-name = "P2_01"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P2_02 { + gpio-name = "P2_02"; + gpio = <&gpio1 27 0>; + input; + dir-changeable; + }; + + P2_03 { + gpio-name = "P2_03"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P2_04 { + gpio-name = "P2_04"; + gpio = <&gpio1 26 0>; + input; + dir-changeable; + }; + + P2_05 { + gpio-name = "P2_05"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P2_06 { + gpio-name = "P2_06"; + gpio = <&gpio1 25 0>; + input; + dir-changeable; + }; + + P2_07 { + gpio-name = "P2_07"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P2_08 { + gpio-name = "P2_08"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P2_09 { + gpio-name = "P2_09"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P2_10 { + gpio-name = "P2_10"; + gpio = <&gpio1 20 0>; + input; + dir-changeable; + }; + + P2_11 { + gpio-name = "P2_11"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P2_17 { + gpio-name = "P2_17"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P2_18 { + gpio-name = "P2_18"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P2_19 { + gpio-name = "P2_19"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P2_20 { + gpio-name = "P2_20"; + gpio = <&gpio2 0 0>; + input; + dir-changeable; + }; + + P2_22 { + gpio-name = "P2_22"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P2_24 { + gpio-name = "P2_24"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P2_25 { + gpio-name = "P2_25"; + gpio = <&gpio1 9 0>; + input; + dir-changeable; + }; + + P2_27 { + gpio-name = "P2_27"; + gpio = <&gpio1 8 0>; + input; + dir-changeable; + }; + + P2_28 { + gpio-name = "P2_28"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P2_29 { + gpio-name = "P2_29"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P2_30 { + gpio-name = "P2_30"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P2_31 { + gpio-name = "P2_31"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + + P2_32 { + gpio-name = "P2_32"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P2_33 { + gpio-name = "P2_33"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P2_34 { + gpio-name = "P2_34"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P2_35 { + gpio-name = "P2_35"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-pocketbeagle.dts b/arch/arm/boot/dts/am335x-pocketbeagle.dts new file mode 100644 index 0000000000000..ccb9bd9aee8eb --- /dev/null +++ b/arch/arm/boot/dts/am335x-pocketbeagle.dts @@ -0,0 +1,1091 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * Author: Robert Nelson + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-pocketbeagle-common.dtsi" + +&sgx { + status = "okay"; +}; + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.1"; + reg = <1>; + spi-max-frequency = <24000000>; + status = "disabled"; + }; +}; + +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; + +&ocp { + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio_input */ + P1_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_02_default_pin>; + pinctrl-1 = <&P1_02_gpio_pin>; + pinctrl-2 = <&P1_02_gpio_pu_pin>; + pinctrl-3 = <&P1_02_gpio_pd_pin>; + pinctrl-4 = <&P1_02_gpio_input_pin>; + pinctrl-5 = <&P1_02_pruout_pin>; + pinctrl-6 = <&P1_02_pruin_pin>; + }; + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) */ + P1_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_04_default_pin>; + pinctrl-1 = <&P1_04_gpio_pin>; + pinctrl-2 = <&P1_04_gpio_pu_pin>; + pinctrl-3 = <&P1_04_gpio_pd_pin>; + pinctrl-4 = <&P1_04_gpio_input_pin>; + pinctrl-5 = <&P1_04_pruout_pin>; + pinctrl-6 = <&P1_04_pruin_pin>; + }; + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi_cs */ + P1_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_06_default_pin>; + pinctrl-1 = <&P1_06_gpio_pin>; + pinctrl-2 = <&P1_06_gpio_pu_pin>; + pinctrl-3 = <&P1_06_gpio_pd_pin>; + pinctrl-4 = <&P1_06_gpio_input_pin>; + pinctrl-5 = <&P1_06_spi_cs_pin>; + pinctrl-6 = <&P1_06_i2c_pin>; + pinctrl-7 = <&P1_06_pwm_pin>; + pinctrl-8 = <&P1_06_pru_uart_pin>; + }; + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi_sclk */ + P1_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_08_default_pin>; + pinctrl-1 = <&P1_08_gpio_pin>; + pinctrl-2 = <&P1_08_gpio_pu_pin>; + pinctrl-3 = <&P1_08_gpio_pd_pin>; + pinctrl-4 = <&P1_08_gpio_input_pin>; + pinctrl-5 = <&P1_08_spi_sclk_pin>; + pinctrl-6 = <&P1_08_uart_pin>; + pinctrl-7 = <&P1_08_i2c_pin>; + pinctrl-8 = <&P1_08_pwm_pin>; + pinctrl-9 = <&P1_08_pru_uart_pin>; + }; + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi */ + P1_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_10_default_pin>; + pinctrl-1 = <&P1_10_gpio_pin>; + pinctrl-2 = <&P1_10_gpio_pu_pin>; + pinctrl-3 = <&P1_10_gpio_pd_pin>; + pinctrl-4 = <&P1_10_gpio_input_pin>; + pinctrl-5 = <&P1_10_spi_pin>; + pinctrl-6 = <&P1_10_uart_pin>; + pinctrl-7 = <&P1_10_i2c_pin>; + pinctrl-8 = <&P1_10_pwm_pin>; + pinctrl-9 = <&P1_10_pru_uart_pin>; + }; + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi */ + P1_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_12_default_pin>; + pinctrl-1 = <&P1_12_gpio_pin>; + pinctrl-2 = <&P1_12_gpio_pu_pin>; + pinctrl-3 = <&P1_12_gpio_pd_pin>; + pinctrl-4 = <&P1_12_gpio_input_pin>; + pinctrl-5 = <&P1_12_spi_pin>; + pinctrl-6 = <&P1_12_i2c_pin>; + pinctrl-7 = <&P1_12_pwm_pin>; + pinctrl-8 = <&P1_12_pru_uart_pin>; + }; + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) */ + P1_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin"; + pinctrl-0 = <&P1_20_default_pin>; + pinctrl-1 = <&P1_20_gpio_pin>; + pinctrl-2 = <&P1_20_gpio_pu_pin>; + pinctrl-3 = <&P1_20_gpio_pd_pin>; + pinctrl-4 = <&P1_20_gpio_input_pin>; + pinctrl-5 = <&P1_20_pruin_pin>; + }; + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c */ + P1_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_26_default_pin>; + pinctrl-1 = <&P1_26_gpio_pin>; + pinctrl-2 = <&P1_26_gpio_pu_pin>; + pinctrl-3 = <&P1_26_gpio_pd_pin>; + pinctrl-4 = <&P1_26_gpio_input_pin>; + pinctrl-5 = <&P1_26_spi_cs_pin>; + pinctrl-6 = <&P1_26_can_pin>; + pinctrl-7 = <&P1_26_i2c_pin>; + pinctrl-8 = <&P1_26_pru_uart_pin>; + }; + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c */ + P1_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_28_default_pin>; + pinctrl-1 = <&P1_28_gpio_pin>; + pinctrl-2 = <&P1_28_gpio_pu_pin>; + pinctrl-3 = <&P1_28_gpio_pd_pin>; + pinctrl-4 = <&P1_28_gpio_input_pin>; + pinctrl-5 = <&P1_28_spi_cs_pin>; + pinctrl-6 = <&P1_28_can_pin>; + pinctrl-7 = <&P1_28_i2c_pin>; + pinctrl-8 = <&P1_28_pru_uart_pin>; + }; + + /* P1_29 (ZCZ ball A14) pruin */ + P1_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_29_default_pin>; + pinctrl-1 = <&P1_29_gpio_pin>; + pinctrl-2 = <&P1_29_gpio_pu_pin>; + pinctrl-3 = <&P1_29_gpio_pd_pin>; + pinctrl-4 = <&P1_29_gpio_input_pin>; + pinctrl-5 = <&P1_29_qep_pin>; + pinctrl-6 = <&P1_29_pruout_pin>; + pinctrl-7 = <&P1_29_pruin_pin>; + }; + + /* P1_30 (ZCZ ball E16) uart */ + P1_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_30_default_pin>; + pinctrl-1 = <&P1_30_gpio_pin>; + pinctrl-2 = <&P1_30_gpio_pu_pin>; + pinctrl-3 = <&P1_30_gpio_pd_pin>; + pinctrl-4 = <&P1_30_gpio_input_pin>; + pinctrl-5 = <&P1_30_spi_cs_pin>; + pinctrl-6 = <&P1_30_uart_pin>; + pinctrl-7 = <&P1_30_can_pin>; + pinctrl-8 = <&P1_30_i2c_pin>; + pinctrl-9 = <&P1_30_pruout_pin>; + pinctrl-10 = <&P1_30_pruin_pin>; + }; + + /* P1_31 (ZCZ ball B12) pruin */ + P1_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_31_default_pin>; + pinctrl-1 = <&P1_31_gpio_pin>; + pinctrl-2 = <&P1_31_gpio_pu_pin>; + pinctrl-3 = <&P1_31_gpio_pd_pin>; + pinctrl-4 = <&P1_31_gpio_input_pin>; + pinctrl-5 = <&P1_31_qep_pin>; + pinctrl-6 = <&P1_31_pruout_pin>; + pinctrl-7 = <&P1_31_pruin_pin>; + }; + + /* P1_32 (ZCZ ball E15) uart */ + P1_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_32_default_pin>; + pinctrl-1 = <&P1_32_gpio_pin>; + pinctrl-2 = <&P1_32_gpio_pu_pin>; + pinctrl-3 = <&P1_32_gpio_pd_pin>; + pinctrl-4 = <&P1_32_gpio_input_pin>; + pinctrl-5 = <&P1_32_spi_cs_pin>; + pinctrl-6 = <&P1_32_uart_pin>; + pinctrl-7 = <&P1_32_can_pin>; + pinctrl-8 = <&P1_32_i2c_pin>; + pinctrl-9 = <&P1_32_pruout_pin>; + pinctrl-10 = <&P1_32_pruin_pin>; + }; + + /* P1_33 (ZCZ ball B13) pruin */ + P1_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_33_default_pin>; + pinctrl-1 = <&P1_33_gpio_pin>; + pinctrl-2 = <&P1_33_gpio_pu_pin>; + pinctrl-3 = <&P1_33_gpio_pd_pin>; + pinctrl-4 = <&P1_33_gpio_input_pin>; + pinctrl-5 = <&P1_33_spi_pin>; + pinctrl-6 = <&P1_33_pwm_pin>; + pinctrl-7 = <&P1_33_pruout_pin>; + pinctrl-8 = <&P1_33_pruin_pin>; + }; + + /* P1_34 (ZCZ ball T11) */ + P1_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P1_34_default_pin>; + pinctrl-1 = <&P1_34_gpio_pin>; + pinctrl-2 = <&P1_34_gpio_pu_pin>; + pinctrl-3 = <&P1_34_gpio_pd_pin>; + pinctrl-4 = <&P1_34_gpio_input_pin>; + pinctrl-5 = <&P1_34_pwm_pin>; + }; + + /* P1_35 (ZCZ ball V5) pruin */ + P1_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_35_default_pin>; + pinctrl-1 = <&P1_35_gpio_pin>; + pinctrl-2 = <&P1_35_gpio_pu_pin>; + pinctrl-3 = <&P1_35_gpio_pd_pin>; + pinctrl-4 = <&P1_35_gpio_input_pin>; + pinctrl-5 = <&P1_35_pruout_pin>; + pinctrl-6 = <&P1_35_pruin_pin>; + }; + + /* P1_36 (ZCZ ball A13) pwm */ + P1_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_36_default_pin>; + pinctrl-1 = <&P1_36_gpio_pin>; + pinctrl-2 = <&P1_36_gpio_pu_pin>; + pinctrl-3 = <&P1_36_gpio_pd_pin>; + pinctrl-4 = <&P1_36_gpio_input_pin>; + pinctrl-5 = <&P1_36_spi_sclk_pin>; + pinctrl-6 = <&P1_36_pwm_pin>; + pinctrl-7 = <&P1_36_pruout_pin>; + pinctrl-8 = <&P1_36_pruin_pin>; + }; + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) pwm */ + P2_01_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_01_default_pin>; + pinctrl-1 = <&P2_01_gpio_pin>; + pinctrl-2 = <&P2_01_gpio_pu_pin>; + pinctrl-3 = <&P2_01_gpio_pd_pin>; + pinctrl-4 = <&P2_01_gpio_input_pin>; + pinctrl-5 = <&P2_01_pwm_pin>; + }; + + /* P2_02 (ZCZ ball V17) */ + P2_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_02_default_pin>; + pinctrl-1 = <&P2_02_gpio_pin>; + pinctrl-2 = <&P2_02_gpio_pu_pin>; + pinctrl-3 = <&P2_02_gpio_pd_pin>; + pinctrl-4 = <&P2_02_gpio_input_pin>; + }; + + /* P2_03 (ZCZ ball T10) */ + P2_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_03_default_pin>; + pinctrl-1 = <&P2_03_gpio_pin>; + pinctrl-2 = <&P2_03_gpio_pu_pin>; + pinctrl-3 = <&P2_03_gpio_pd_pin>; + pinctrl-4 = <&P2_03_gpio_input_pin>; + pinctrl-5 = <&P2_03_pwm_pin>; + }; + + /* P2_04 (ZCZ ball T16) */ + P2_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_04_default_pin>; + pinctrl-1 = <&P2_04_gpio_pin>; + pinctrl-2 = <&P2_04_gpio_pu_pin>; + pinctrl-3 = <&P2_04_gpio_pd_pin>; + pinctrl-4 = <&P2_04_gpio_input_pin>; + }; + + /* P2_05 (ZCZ ball T17) uart */ + P2_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_05_default_pin>; + pinctrl-1 = <&P2_05_gpio_pin>; + pinctrl-2 = <&P2_05_gpio_pu_pin>; + pinctrl-3 = <&P2_05_gpio_pd_pin>; + pinctrl-4 = <&P2_05_gpio_input_pin>; + pinctrl-5 = <&P2_05_uart_pin>; + }; + + /* P2_06 (ZCZ ball U16) */ + P2_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_06_default_pin>; + pinctrl-1 = <&P2_06_gpio_pin>; + pinctrl-2 = <&P2_06_gpio_pu_pin>; + pinctrl-3 = <&P2_06_gpio_pd_pin>; + pinctrl-4 = <&P2_06_gpio_input_pin>; + }; + + /* P2_07 (ZCZ ball U17) uart */ + P2_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_07_default_pin>; + pinctrl-1 = <&P2_07_gpio_pin>; + pinctrl-2 = <&P2_07_gpio_pu_pin>; + pinctrl-3 = <&P2_07_gpio_pd_pin>; + pinctrl-4 = <&P2_07_gpio_input_pin>; + pinctrl-5 = <&P2_07_uart_pin>; + }; + + /* P2_08 (ZCZ ball U18) */ + P2_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_08_default_pin>; + pinctrl-1 = <&P2_08_gpio_pin>; + pinctrl-2 = <&P2_08_gpio_pu_pin>; + pinctrl-3 = <&P2_08_gpio_pd_pin>; + pinctrl-4 = <&P2_08_gpio_input_pin>; + }; + + /* P2_09 (ZCZ ball D15) i2c */ + P2_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_09_default_pin>; + pinctrl-1 = <&P2_09_gpio_pin>; + pinctrl-2 = <&P2_09_gpio_pu_pin>; + pinctrl-3 = <&P2_09_gpio_pd_pin>; + pinctrl-4 = <&P2_09_gpio_input_pin>; + pinctrl-5 = <&P2_09_uart_pin>; + pinctrl-6 = <&P2_09_can_pin>; + pinctrl-7 = <&P2_09_i2c_pin>; + pinctrl-8 = <&P2_09_pru_uart_pin>; + pinctrl-9 = <&P2_09_pruin_pin>; + }; + + /* P2_10 (ZCZ ball R14) */ + P2_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P2_10_default_pin>; + pinctrl-1 = <&P2_10_gpio_pin>; + pinctrl-2 = <&P2_10_gpio_pu_pin>; + pinctrl-3 = <&P2_10_gpio_pd_pin>; + pinctrl-4 = <&P2_10_gpio_input_pin>; + pinctrl-5 = <&P2_10_qep_pin>; + }; + + /* P2_11 (ZCZ ball D16) i2c */ + P2_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_11_default_pin>; + pinctrl-1 = <&P2_11_gpio_pin>; + pinctrl-2 = <&P2_11_gpio_pu_pin>; + pinctrl-3 = <&P2_11_gpio_pd_pin>; + pinctrl-4 = <&P2_11_gpio_input_pin>; + pinctrl-5 = <&P2_11_uart_pin>; + pinctrl-6 = <&P2_11_can_pin>; + pinctrl-7 = <&P2_11_i2c_pin>; + pinctrl-8 = <&P2_11_pru_uart_pin>; + pinctrl-9 = <&P2_11_pruin_pin>; + }; + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) */ + P2_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_17_default_pin>; + pinctrl-1 = <&P2_17_gpio_pin>; + pinctrl-2 = <&P2_17_gpio_pu_pin>; + pinctrl-3 = <&P2_17_gpio_pd_pin>; + pinctrl-4 = <&P2_17_gpio_input_pin>; + }; + + /* P2_18 (ZCZ ball U13) */ + P2_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P2_18_default_pin>; + pinctrl-1 = <&P2_18_gpio_pin>; + pinctrl-2 = <&P2_18_gpio_pu_pin>; + pinctrl-3 = <&P2_18_gpio_pd_pin>; + pinctrl-4 = <&P2_18_gpio_input_pin>; + pinctrl-5 = <&P2_18_qep_pin>; + pinctrl-6 = <&P2_18_pru_ecap_pin>; + pinctrl-7 = <&P2_18_pruin_pin>; + }; + + /* P2_19 (ZCZ ball U12) */ + P2_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_19_default_pin>; + pinctrl-1 = <&P2_19_gpio_pin>; + pinctrl-2 = <&P2_19_gpio_pu_pin>; + pinctrl-3 = <&P2_19_gpio_pd_pin>; + pinctrl-4 = <&P2_19_gpio_input_pin>; + pinctrl-5 = <&P2_19_pwm_pin>; + }; + + /* P2_20 (ZCZ ball T13) */ + P2_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_20_default_pin>; + pinctrl-1 = <&P2_20_gpio_pin>; + pinctrl-2 = <&P2_20_gpio_pu_pin>; + pinctrl-3 = <&P2_20_gpio_pd_pin>; + pinctrl-4 = <&P2_20_gpio_input_pin>; + }; + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) */ + P2_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P2_22_default_pin>; + pinctrl-1 = <&P2_22_gpio_pin>; + pinctrl-2 = <&P2_22_gpio_pu_pin>; + pinctrl-3 = <&P2_22_gpio_pd_pin>; + pinctrl-4 = <&P2_22_gpio_input_pin>; + pinctrl-5 = <&P2_22_qep_pin>; + pinctrl-6 = <&P2_22_pruin_pin>; + }; + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) */ + P2_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_24_default_pin>; + pinctrl-1 = <&P2_24_gpio_pin>; + pinctrl-2 = <&P2_24_gpio_pu_pin>; + pinctrl-3 = <&P2_24_gpio_pd_pin>; + pinctrl-4 = <&P2_24_gpio_input_pin>; + pinctrl-5 = <&P2_24_qep_pin>; + pinctrl-6 = <&P2_24_pruout_pin>; + }; + + /* P2_25 (ZCZ ball E17) spi */ + P2_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c"; + pinctrl-0 = <&P2_25_default_pin>; + pinctrl-1 = <&P2_25_gpio_pin>; + pinctrl-2 = <&P2_25_gpio_pu_pin>; + pinctrl-3 = <&P2_25_gpio_pd_pin>; + pinctrl-4 = <&P2_25_gpio_input_pin>; + pinctrl-5 = <&P2_25_spi_pin>; + pinctrl-6 = <&P2_25_spi_cs_pin>; + pinctrl-7 = <&P2_25_uart_pin>; + pinctrl-8 = <&P2_25_can_pin>; + pinctrl-9 = <&P2_25_i2c_pin>; + }; + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi */ + P2_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c"; + pinctrl-0 = <&P2_27_default_pin>; + pinctrl-1 = <&P2_27_gpio_pin>; + pinctrl-2 = <&P2_27_gpio_pu_pin>; + pinctrl-3 = <&P2_27_gpio_pd_pin>; + pinctrl-4 = <&P2_27_gpio_input_pin>; + pinctrl-5 = <&P2_27_spi_pin>; + pinctrl-6 = <&P2_27_uart_pin>; + pinctrl-7 = <&P2_27_can_pin>; + pinctrl-8 = <&P2_27_i2c_pin>; + }; + + /* P2_28 (ZCZ ball D13) pruin */ + P2_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_28_default_pin>; + pinctrl-1 = <&P2_28_gpio_pin>; + pinctrl-2 = <&P2_28_gpio_pu_pin>; + pinctrl-3 = <&P2_28_gpio_pd_pin>; + pinctrl-4 = <&P2_28_gpio_input_pin>; + pinctrl-5 = <&P2_28_qep_pin>; + pinctrl-6 = <&P2_28_pruout_pin>; + pinctrl-7 = <&P2_28_pruin_pin>; + }; + + /* P2_29 (ZCZ ball C18) spi_sclk */ + P2_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P2_29_default_pin>; + pinctrl-1 = <&P2_29_gpio_pin>; + pinctrl-2 = <&P2_29_gpio_pu_pin>; + pinctrl-3 = <&P2_29_gpio_pd_pin>; + pinctrl-4 = <&P2_29_gpio_input_pin>; + pinctrl-5 = <&P2_29_spi_cs_pin>; + pinctrl-6 = <&P2_29_spi_sclk_pin>; + pinctrl-7 = <&P2_29_uart_pin>; + pinctrl-8 = <&P2_29_pwm_pin>; + pinctrl-9 = <&P2_29_pru_ecap_pin>; + }; + + /* P2_30 (ZCZ ball C12) pruin */ + P2_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_30_default_pin>; + pinctrl-1 = <&P2_30_gpio_pin>; + pinctrl-2 = <&P2_30_gpio_pu_pin>; + pinctrl-3 = <&P2_30_gpio_pd_pin>; + pinctrl-4 = <&P2_30_gpio_input_pin>; + pinctrl-5 = <&P2_30_spi_cs_pin>; + pinctrl-6 = <&P2_30_pwm_pin>; + pinctrl-7 = <&P2_30_pruout_pin>; + pinctrl-8 = <&P2_30_pruin_pin>; + }; + + /* P2_31 (ZCZ ball A15) spi_cs */ + P2_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin"; + pinctrl-0 = <&P2_31_default_pin>; + pinctrl-1 = <&P2_31_gpio_pin>; + pinctrl-2 = <&P2_31_gpio_pu_pin>; + pinctrl-3 = <&P2_31_gpio_pd_pin>; + pinctrl-4 = <&P2_31_gpio_input_pin>; + pinctrl-5 = <&P2_31_spi_cs_pin>; + pinctrl-6 = <&P2_31_pruin_pin>; + }; + + /* P2_32 (ZCZ ball D12) pruin */ + P2_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_32_default_pin>; + pinctrl-1 = <&P2_32_gpio_pin>; + pinctrl-2 = <&P2_32_gpio_pu_pin>; + pinctrl-3 = <&P2_32_gpio_pd_pin>; + pinctrl-4 = <&P2_32_gpio_input_pin>; + pinctrl-5 = <&P2_32_spi_pin>; + pinctrl-6 = <&P2_32_pwm_pin>; + pinctrl-7 = <&P2_32_pruout_pin>; + pinctrl-8 = <&P2_32_pruin_pin>; + }; + + /* P2_33 (ZCZ ball R12) */ + P2_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_33_default_pin>; + pinctrl-1 = <&P2_33_gpio_pin>; + pinctrl-2 = <&P2_33_gpio_pu_pin>; + pinctrl-3 = <&P2_33_gpio_pd_pin>; + pinctrl-4 = <&P2_33_gpio_input_pin>; + pinctrl-5 = <&P2_33_qep_pin>; + pinctrl-6 = <&P2_33_pruout_pin>; + }; + + /* P2_34 (ZCZ ball C13) pruin */ + P2_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_34_default_pin>; + pinctrl-1 = <&P2_34_gpio_pin>; + pinctrl-2 = <&P2_34_gpio_pu_pin>; + pinctrl-3 = <&P2_34_gpio_pd_pin>; + pinctrl-4 = <&P2_34_gpio_input_pin>; + pinctrl-5 = <&P2_34_qep_pin>; + pinctrl-6 = <&P2_34_pruout_pin>; + pinctrl-7 = <&P2_34_pruin_pin>; + }; + + /* P2_35 (ZCZ ball U5) gpio_input */ + P2_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P2_35_default_pin>; + pinctrl-1 = <&P2_35_gpio_pin>; + pinctrl-2 = <&P2_35_gpio_pu_pin>; + pinctrl-3 = <&P2_35_gpio_pd_pin>; + pinctrl-4 = <&P2_35_gpio_input_pin>; + pinctrl-5 = <&P2_35_pruout_pin>; + pinctrl-6 = <&P2_35_pruin_pin>; + }; + + /* P2_36 (ZCZ ball C9) AIN7 */ + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P1_02 { + gpio-name = "P1_02"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P1_04 { + gpio-name = "P1_04"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P1_06 { + gpio-name = "P1_06"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P1_08 { + gpio-name = "P1_08"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P1_10 { + gpio-name = "P1_10"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P1_12 { + gpio-name = "P1_12"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P1_20 { + gpio-name = "P1_20"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P1_26 { + gpio-name = "P1_26"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P1_28 { + gpio-name = "P1_28"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P1_29 { + gpio-name = "P1_29"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P1_30 { + gpio-name = "P1_30"; + gpio = <&gpio1 11 0>; + input; + dir-changeable; + }; + + P1_31 { + gpio-name = "P1_31"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + P1_32 { + gpio-name = "P1_32"; + gpio = <&gpio1 10 0>; + input; + dir-changeable; + }; + + P1_33 { + gpio-name = "P1_33"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P1_34 { + gpio-name = "P1_34"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P1_35 { + gpio-name = "P1_35"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P1_36 { + gpio-name = "P1_36"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P2_01 { + gpio-name = "P2_01"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P2_02 { + gpio-name = "P2_02"; + gpio = <&gpio1 27 0>; + input; + dir-changeable; + }; + + P2_03 { + gpio-name = "P2_03"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P2_04 { + gpio-name = "P2_04"; + gpio = <&gpio1 26 0>; + input; + dir-changeable; + }; + + P2_05 { + gpio-name = "P2_05"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P2_06 { + gpio-name = "P2_06"; + gpio = <&gpio1 25 0>; + input; + dir-changeable; + }; + + P2_07 { + gpio-name = "P2_07"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P2_08 { + gpio-name = "P2_08"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P2_09 { + gpio-name = "P2_09"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P2_10 { + gpio-name = "P2_10"; + gpio = <&gpio1 20 0>; + input; + dir-changeable; + }; + + P2_11 { + gpio-name = "P2_11"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P2_17 { + gpio-name = "P2_17"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P2_18 { + gpio-name = "P2_18"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P2_19 { + gpio-name = "P2_19"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P2_20 { + gpio-name = "P2_20"; + gpio = <&gpio2 0 0>; + input; + dir-changeable; + }; + + P2_22 { + gpio-name = "P2_22"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P2_24 { + gpio-name = "P2_24"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P2_25 { + gpio-name = "P2_25"; + gpio = <&gpio1 9 0>; + input; + dir-changeable; + }; + + P2_27 { + gpio-name = "P2_27"; + gpio = <&gpio1 8 0>; + input; + dir-changeable; + }; + + P2_28 { + gpio-name = "P2_28"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P2_29 { + gpio-name = "P2_29"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P2_30 { + gpio-name = "P2_30"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P2_31 { + gpio-name = "P2_31"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + + P2_32 { + gpio-name = "P2_32"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P2_33 { + gpio-name = "P2_33"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P2_34 { + gpio-name = "P2_34"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P2_35 { + gpio-name = "P2_35"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-roboticscape.dtsi b/arch/arm/boot/dts/am335x-roboticscape.dtsi new file mode 100644 index 0000000000000..81557df9d4157 --- /dev/null +++ b/arch/arm/boot/dts/am335x-roboticscape.dtsi @@ -0,0 +1,489 @@ +/* + * @file am335x-roboticscape.dtsi + * + * used by: + * am335x-boneblack-roboticscape.dts + * am335x-boneblack-wireless-roboticscape.dts + * + * + * @file am335x-boneblack-roboticscape.dtb + * + * based on am335x-boneblack.dtb + * + * This device tree serves to replace the need for an overlay when using the + * RoboticsCape. It is similar to the boneblue tree but preserves pin config for + * the black. + * + * This file was updated in April 2018 to support LED driver like in BB Blue + * This goes in sync with V0.4.0 of the Robotics Cape library + * + * @author James Strawson + * @date April 19, 2018 + */ + +&aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + d_can0 = &dcan0; + d_can1 = &dcan1; + usb0 = &usb0; + usb1 = &usb1; + phy0 = &usb0_phy; + phy1 = &usb1_phy; + ethernet0 = &cpsw_emac0; + ethernet1 = &cpsw_emac1; + /* librobotcontrol expects /dev/spidev1 */ + spi0 = &spi0; + spi1 = &spi1; + /delete-property/ spi2; +}; + +/******************************************************************************* +* Pin Muxing +*******************************************************************************/ +&am33xx_pinmux { + +/******************************************************************************* +* Static Pinmux +*******************************************************************************/ + mux_helper_pins: pins { + pinctrl-single,pins = < + + /* GPIO Input Pullup */ + 0x09c 0x37 /*P8.9 T6 PAUSE_BTN */ + 0x098 0x37 /*P8.10 U6 MODE_BTN */ + 0x1AC 0x37 /*P9.25 A14 IMU_INT */ + + /* Motor Control GPIO Out*/ + 0x0cc 0x0F /*P8.34 MDIR_2B different from blue!!*/ + 0x0a8 0x0F /*P8.43 MDIR_3B*/ + 0x0ac 0x0F /*P8.44 MDIR_3A*/ + 0x0a0 0x0F /*P8.45 MDIR_4A*/ + 0x0a4 0x0F /*P8.46 MDIR_4B*/ + 0x078 0x0F /*P9.12 MDIR_1A different from blue!!*/ + 0x074 0x0F /*P9.13 MDIR_1B*/ + 0x040 0x0F /*P9.15 MDIR_2A*/ + 0x1b4 0x0F /*P9.41 MOT_STBY*/ + + /* PRU encoder input */ + 0x03c 0x36 /* P8_15,PRU0_r31_15,MODE6 */ + 0x038 0x36 /* P8_16,PRU0_r31_16,MODE6 */ + + /* PRU Servo output */ + 0x0e0 0x05 /*pru1_pru_r30_8, MODE5*/ + 0x0e8 0x05 /*pru1_pru_r30_10, MODE5 */ + 0x0e4 0x05 /*pr1_pru1_pru_r30_9, MODE5 */ + 0x0ec 0x05 /*pru1_pru_r30_11, MODE5 */ + 0x0b8 0x05 /*pru1_pru_r30_6, MODE5 */ + 0x0bc 0x05 /*pru1_pru_r30_7, MODE5 */ + 0x0b0 0x05 /*pru1_pru_r30_4, MODE5 */ + 0x0b4 0x05 /*pru1_pru_r30_5, MODE5 */ + 0x0C8 0x0F /*P8.36, SERVO_PWR GPIO OUT*/ + + /* I2C1 */ + 0x15C 0x32 /* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */ + 0x158 0x32 /* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */ + + /* I2C2 */ + //0x17c 0x73 /* P9.19, i2c2_sda, mode 3 */ + //0x178 0x73 /* P9.20, i2c2_sda, mode 3 */ + + /* UART5 */ + 0x0C4 0x34 /* P8.38,uart5_rxd,MODE4 */ + 0x0C0 0x14 /* P8.37,uart5_txd,MODE4 */ + + >; + }; + + + led_pins: pinmux_led_pins { + pinctrl-single,pins = < + 0x090 (PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ + 0x094 (PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ + 0x02C (PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ + 0x08c (PIN_OUTPUT | MUX_MODE7) /* (V12) gpio2.1 - P8.18 BATT_LED_2 diff from BLUE! */ + 0x07C (PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ + 0x028 (PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ + >; + }; + +}; + + +/******************************************************************************* +* apply static and dynamic pinmux modes listed above. Configurable pins get the +* modes from am335x-boneblack-common-universal-pins.dtsi +*******************************************************************************/ +&ocp { + /* activate the static pinmux helper list of pin modes above */ + test_helper: helper { + compatible = "bone-pinmux-helper"; + pinctrl-names = "default"; + pinctrl-0 = <&mux_helper_pins>; + + status = "okay"; + }; + + /* UART4 RX DSM */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_uart_pin>; + }; + + /* UART 2 TX GPS*/ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_21_uart_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_spi_pin>; + pinctrl-5 = <&P9_21_uart_pin>; + pinctrl-6 = <&P9_21_i2c_pin>; + pinctrl-7 = <&P9_21_pwm_pin>; + }; + + /* UART 2 RX GPS */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_22_uart_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_spi_pin>; + pinctrl-5 = <&P9_22_uart_pin>; + pinctrl-6 = <&P9_22_i2c_pin>; + pinctrl-7 = <&P9_22_pwm_pin>; + }; + + /* SPI MISO */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_29_spi_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_pwm_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pruout_pin>; + pinctrl-7 = <&P9_29_pruin_pin>; + }; + + /* SPI MOSI */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_30_spi_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_pwm_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pruout_pin>; + pinctrl-7 = <&P9_30_pruin_pin>; + }; + + /* SPI SCK */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin"; + pinctrl-0 = <&P9_31_spi_sclk_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_pwm_pin>; + pinctrl-5 = <&P9_31_spi_pin>; + pinctrl-6 = <&P9_31_spi_sclk_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* SPI SS1 GPIO3_17*/ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_spi_cs_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_pwm_pin>; + pinctrl-5 = <&P9_28_spi_pin>; + pinctrl-6 = <&P9_28_spi_cs_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* SPI SS1 GPIO1_17*/ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm"; + pinctrl-0 = <&P9_23_gpio_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_pwm_pin>; + }; + + /* UART 1 TX */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_24_uart_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_uart_pin>; + pinctrl-5 = <&P9_24_can_pin>; + pinctrl-6 = <&P9_24_i2c_pin>; + pinctrl-7 = <&P9_24_pruin_pin>; + }; + + /* UART 1 RX */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_26_uart_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_uart_pin>; + pinctrl-5 = <&P9_26_can_pin>; + pinctrl-6 = <&P9_26_i2c_pin>; + pinctrl-7 = <&P9_26_pruin_pin>; + }; + + +}; + + +/******************************************************************************* +* PWMSS +*******************************************************************************/ +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; +}; + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_19_pwm_pin &P8_13_pwm_pin>; + status = "okay"; +}; + +&ehrpwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&P9_14_pwm_pin &P9_16_pwm_pin>; + status = "okay"; +}; + +/******************************************************************************* +* EQEP +*******************************************************************************/ +&eqep0 { + pinctrl-names = "default"; + pinctrl-0 = <&P9_92_qep_pin &P9_27_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep1 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_33_qep_pin &P8_35_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep2 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_12_qep_pin &P8_11_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + + +/******************************************************************************* +* UART +*******************************************************************************/ +&uart1 { + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&uart4 { + status = "okay"; +}; + +&uart5 { + status = "okay"; +}; + + +/******************************************************************************* +* PRU Encoder and Servos +*******************************************************************************/ +&pruss_soc_bus { + status = "okay"; + + pruss: pruss@0 { + status = "okay"; + }; +}; + + +/******************************************************************************* +* I2C +*******************************************************************************/ +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; +}; + +/******************************************************************************* +* SPI +*******************************************************************************/ +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/******************************************************************************* +* LEDs +*******************************************************************************/ +/ { + leds { + pinctrl-names = "default"; + pinctrl-0 = <&led_pins>; + compatible = "gpio-leds"; + + red_led { + label = "red"; + gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + green_led { + label = "green"; + gpios = <&gpio2 3 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_1_led { + label = "bat25"; + gpios = <&gpio0 27 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_2_led { + label = "bat50"; + gpios = <&gpio2 1 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_3_led { + label = "bat75"; + gpios = <&gpio1 29 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_4_led { + label = "bat100"; + gpios = <&gpio0 26 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + +}; + +&tscadc { + status = "okay"; + adc { + ti,adc-channels = <0 1 2 3 4 5 6 7>; + ti,chan-step-avg = <0x16 0x16 0x16 0x16 0x16 0x16 0x16 0x16>; + ti,chan-step-opendelay = <0x98 0x98 0x98 0x98 0x98 0x98 0x98 0x98>; + ti,chan-step-sampledelay = <0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe-common.dtsi b/arch/arm/boot/dts/am335x-sancloud-bbe-common.dtsi new file mode 100644 index 0000000000000..cfcb365a39746 --- /dev/null +++ b/arch/arm/boot/dts/am335x-sancloud-bbe-common.dtsi @@ -0,0 +1,585 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/ { + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x10000000>; /* 256 MB */ + }; + + chosen { + stdout-path = &uart0; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led2 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led4 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led5 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&usb_hub_ctrl>; + + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + +// i2c2_pins: pinmux_i2c2_pins { +// pinctrl-single,pins = < +// AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_ctsn.i2c2_sda */ +// AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_rtsn.i2c2_scl */ +// >; +// }; + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ + AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + 0x114 (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txen.rgmii1_tctl */ + 0x118 (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ + 0x11c (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd3.rgmii1_td3 */ + 0x120 (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd2.rgmii1_td2 */ + 0x124 (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd1.rgmii1_td1 */ + 0x128 (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd0.rgmii1_td0 */ + 0x12c (PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txclk.rgmii1_tclk */ + 0x130 (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxclk.rgmii1_rclk */ + 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd3.rgmii1_rd3 */ + 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd2.rgmii1_rd2 */ + 0x13c (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd1.rgmii1_rd1 */ + 0x140 (PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd0.rgmii1_rd0 */ + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + 0x114 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x118 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x11c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x120 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x124 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x128 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x12c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x130 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x13c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x140 (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ + AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + mmc1_pins: pinmux_mmc1_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat0.mmc0_dat0 */ + AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat1.mmc0_dat1 */ + AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat2.mmc0_dat2 */ + AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat3.mmc0_dat3 */ + AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_cmd.mmc0_cmd */ + AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_clk.mmc0_clk */ + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; + + usb_hub_ctrl: usb_hub_ctrl { + pinctrl-single,pins = < + 0x144 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* mcasp0_ahclkr.gpio3_17 */ + >; + }; + + mpu6050_pins: pinmux_mpu6050_pins { + pinctrl-single,pins = < + 0x168 (PIN_INPUT | MUX_MODE7) /* spi0_sclk.gpio0_2 */ + >; + }; + + lps3331ap_pins: pinmux_lps3331ap_pins { + pinctrl-single,pins = < + 0x6C (PIN_INPUT | MUX_MODE7) /* conf_gpmc_a11.gpio1_27 */ + >; + }; + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + + status = "okay"; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; +}; + +&usb1_phy { + status = "okay"; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + interrupts-extended = <&intc 18 &tps 0>; + interrupt-names = "mc", "vbus"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; + + lps331ap: lps331ap@5C { + compatible = "st,lps331ap"; + reg = <0x5C>; + interrupts = <0>, <1>; + }; + + mpu6050: mpu6050@68 { + compatible = "inv,mpu6050"; + reg = <0x68>; + orientation = <0xff 0 0 0 1 0 0 0 0xff>; + interrupts = <2 1>; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + //pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; + + status = "okay"; + clock-frequency = <100000>; + + cape_eeprom0: cape_eeprom0@54 { + compatible = "atmel,24c256"; + reg = <0x54>; + #address-cells = <1>; + #size-cells = <1>; + cape0_data: cape_data@0 { + reg = <0 0x100>; + }; + }; + + cape_eeprom1: cape_eeprom1@55 { + compatible = "atmel,24c256"; + reg = <0x55>; + #address-cells = <1>; + #size-cells = <1>; + cape1_data: cape_data@0 { + reg = <0 0x100>; + }; + }; + + cape_eeprom2: cape_eeprom2@56 { + compatible = "atmel,24c256"; + reg = <0x56>; + #address-cells = <1>; + #size-cells = <1>; + cape2_data: cape_data@0 { + reg = <0 0x100>; + }; + }; + + cape_eeprom3: cape_eeprom3@57 { + compatible = "atmel,24c256"; + reg = <0x57>; + #address-cells = <1>; + #size-cells = <1>; + cape3_data: cape_data@0 { + reg = <0 0x100>; + }; + }; +}; + + +/include/ "tps65217.dtsi" + +&tps { + /* + * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only + * mode") at poweroff. Most BeagleBone versions do not support RTC-only + * mode and risk hardware damage if this mode is entered. + * + * For details, see linux-omap mailing list May 2015 thread + * [PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller + * In particular, messages: + * http://www.spinics.net/lists/linux-omap/msg118585.html + * http://www.spinics.net/lists/linux-omap/msg118615.html + * + * You can override this later with + * &tps { /delete-property/ ti,pmic-shutdown-controller; } + * if you want to use RTC-only mode and made sure you are not affected + * by the hardware problems. (Tip: double-check by performing a current + * measurement after shutdown: it should be less than 1 mA.) + */ + + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "rgmii-txid"; +}; + +&mac { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + slaves = <1>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; +}; + +&mmc1 { + status = "okay"; + bus-width = <0x4>; + pinctrl-names = "default"; + pinctrl-0 = <&mmc1_pins>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + clocks = <&clk_32768_ck>, <&clkdiv32k_ick>; + clock-names = "ext-clk", "int-clk"; + system-power-controller; +}; + +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@0 { +// status = "okay"; +// }; +//}; + +/* the cape manager */ +/ { + bone_capemgr { + compatible = "ti,bone-capemgr"; + status = "okay"; + + nvmem-cells = <&baseboard_data &cape0_data &cape1_data &cape2_data &cape3_data>; + nvmem-cell-names = "baseboard", "slot0", "slot1", "slot2", "slot3"; + #slots = <4>; + + /* map board revisions to compatible definitions */ + baseboardmaps { + baseboard_beaglebone: board@0 { + board-name = "A335BONE"; + compatible-name = "ti,beaglebone"; + }; + + baseboard_beaglebone_black: board@1 { + board-name = "A335BNLT"; + compatible-name = "ti,beaglebone-black"; + }; + }; + }; +}; + +&ocp { + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe-uboot-univ.dts b/arch/arm/boot/dts/am335x-sancloud-bbe-uboot-univ.dts new file mode 100644 index 0000000000000..360c5f2897893 --- /dev/null +++ b/arch/arm/boot/dts/am335x-sancloud-bbe-uboot-univ.dts @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-sancloud-bbe-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "SanCloud BeagleBone Enhanced"; + compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe-uboot.dts b/arch/arm/boot/dts/am335x-sancloud-bbe-uboot.dts new file mode 100644 index 0000000000000..7776c01476cf9 --- /dev/null +++ b/arch/arm/boot/dts/am335x-sancloud-bbe-uboot.dts @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-sancloud-bbe-common.dtsi" + +/ { + model = "SanCloud BeagleBone Enhanced"; + compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe.dts b/arch/arm/boot/dts/am335x-sancloud-bbe.dts new file mode 100644 index 0000000000000..4855005f8526c --- /dev/null +++ b/arch/arm/boot/dts/am335x-sancloud-bbe.dts @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-boneblack-common.dtsi" +#include + +/ { + model = "SanCloud BeagleBone Enhanced"; + compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-sancloud-bbe.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&mmc2 { + ti,vcc-aux-disable-is-sleep; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&usb_hub_ctrl>; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txen.rgmii1_tctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd3.rgmii1_td3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd2.rgmii1_td2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd1.rgmii1_td1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd0.rgmii1_td0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txclk.rgmii1_tclk */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxclk.rgmii1_rclk */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd3.rgmii1_rd3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd2.rgmii1_rd2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd1.rgmii1_rd1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd0.rgmii1_rd0 */ + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + usb_hub_ctrl: usb_hub_ctrl { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* rmii1_refclk.gpio0_29 */ + >; + }; + + mpu6050_pins: pinmux_mpu6050_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT, MUX_MODE7) /* uart0_ctsn.gpio1_8 */ + >; + }; + + lps3331ap_pins: pinmux_lps3331ap_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_A10, PIN_INPUT, MUX_MODE7) /* gpmc_a10.gpio1_26 */ + >; + }; +}; + +&mac { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "rgmii-txid"; +}; + +&i2c0 { + lps331ap: lps331ap@5C { + compatible = "st,lps331ap"; + st,drdy-int-pin = <1>; + reg = <0x5C>; + interrupt-parent = <&gpio1>; + interrupts = <27 IRQ_TYPE_EDGE_RISING>; + }; + + mpu6050: mpu6050@68 { + compatible = "invensense,mpu6050"; + reg = <0x68>; + interrupt-parent = <&gpio0>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + //orientation = <0xff 0 0 0 1 0 0 0 0xff>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-som-common.dtsi b/arch/arm/boot/dts/am335x-som-common.dtsi new file mode 100644 index 0000000000000..fb4399b6fb333 --- /dev/null +++ b/arch/arm/boot/dts/am335x-som-common.dtsi @@ -0,0 +1,465 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/ { + + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_fixed>; + }; + }; + + memory { + device_type = "memory"; + reg = <0x80000000 0x20000000>; /* 512 MB */ + }; + + ocp { + uart0: serial@44e09000 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + + status = "okay"; + }; + uart1: serial@48022000 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_pins>; + status = "okay"; + + }; + uart4: serial@481a8000 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins>; + status = "okay"; + }; + + epwmss0: epwmss@48300000 { + status = "okay"; + + ecap0: ecap@48300100 { + status = "okay"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&ecap0_pins_default>; + pinctrl-1 = <&ecap0_pins_sleep>; + }; + }; + + musb: usb@47400000 { + status = "okay"; + + control@44e10000 { + status = "okay"; + }; + + usb-phy@47401300 { + status = "okay"; + }; + + usb-phy@47401b00 { + status = "okay"; + }; + + usb@47401000 { + status = "okay"; + dr_mode = "otg"; + }; + + usb@47401800 { + status = "okay"; + dr_mode = "host"; + }; + + dma-controller@07402000 { + status = "okay"; + }; + }; + + i2c0: i2c@44e0b000 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + status = "okay"; + clock-frequency = <100000>; + + tps: tps@24 { + reg = <0x24>; + }; + }; + }; + + vmmcsd_fixed: fixedregulator@0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + dcdc2_fixed: fixedregulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.325V with +/-4% tolerance */ + compatible = "regulator-fixed"; + regulator-name = "dcdc2_fixed"; + + regulator-min-microvolt = <1378000>; + regulator-max-microvolt = <1378000>; + regulator-boot-on; + regulator-always-on; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led@1 { + label = "led1:green:heartbeat"; + gpios = <&gpio0 19 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + + led@2 { + label = "led2:red:heartbeat"; + gpios = <&gpio3 20 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + + led@3 { + label = "led3:yello:heartbeat"; + gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + + led@4 { + label = "bkl"; + gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "default-on"; + }; + }; + + backlight { + compatible = "pwm-backlight"; + pwms = <&ecap0 0 500000 1>; + brightness-levels = < + 0 1 2 3 4 5 6 7 8 9 + 10 11 12 13 14 15 16 17 18 19 + 20 21 22 23 24 25 26 27 28 29 + 30 31 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 48 49 + 50 51 52 53 54 55 56 57 58 59 + 60 61 62 63 64 65 66 67 68 69 + 70 71 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 88 89 + 90 91 92 93 94 95 96 97 98 99 + 100 + >; + default-brightness-level = <50>; + }; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&clkout2_pin>; + + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + 0x1b0 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* xdma_event_intr0.gpio0_19 */ + 0x198 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* mcasp0_axr0.gpio3_20 */ + 0x1a8 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* mcasp0_axr1.gpio3_21 */ + 0x1a4 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* mcasp0_fsr.gpio3[19], INPUT_PULLDOWN | MODE7 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + 0x188 (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ + 0x18c (PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + 0x170 (PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ + 0x174 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ + >; + }; + + uart1_pins: pinmux_uart1_pins { + pinctrl-single,pins = < + 0x168 (PIN_INPUT_PULLUP | MUX_MODE1) + 0x16c (PIN_OUTPUT_PULLDOWN | MUX_MODE1) + >; + }; + + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + 0x180 (PIN_INPUT_PULLUP | MUX_MODE0) + 0x184 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) + >; + }; + + + + clkout2_pin: pinmux_clkout2_pin { + pinctrl-single,pins = < + 0x1b4 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* xdma_event_intr1.clkout2 */ + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + 0x110 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */ + 0x114 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */ + 0x118 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */ + 0x11c (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */ + 0x120 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */ + 0x124 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */ + 0x128 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */ + 0x12c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */ + 0x130 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */ + 0x134 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */ + 0x138 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */ + 0x13c (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */ + 0x140 (PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */ + + 0x040 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gpmc_a0.gmii2_txen, OUTPUT_PULLDOWN | MODE1 */ + 0x044 (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a1.gmii2_rxdv, INPUT_PULLDOWN | MODE1 */ + 0x048 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gpmc_a2.gmii2_txd3, OUTPUT_PULLDOWN | MODE1 */ + 0x04c (PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gpmc_a3.gmii2_txd2, OUTPUT_PULLDOWN | MODE1 */ + 0x050 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gpmc_a4.gmii2_txd1, OUTPUT_PULLDOWN | MODE1 */ + 0x054 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gpmc_a5.gmii2_txd0, OUTPUT_PULLDOWN | MODE1 */ + 0x058 (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a6.gmii2_txclk, INPUT_PULLDOWN | MODE1 */ + 0x05c (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a7.gmii2_rxclk, INPUT_PULLDOWN | MODE1 */ + 0x060 (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a8.gmii2_rxd3, INPUT_PULLDOWN | MODE1 */ + 0x064 (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a9.gmii2_rxd2, INPUT_PULLDOWN | MODE1 */ + 0x068 (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a10.gmii2_rxd1, INPUT_PULLDOWN | MODE1 */ + 0x06c (PIN_INPUT_PULLDOWN | MUX_MODE1 ) /* gpmc_a11.gmii2_rxd0, INPUT_PULLDOWN | MODE1 */ + 0x070 (PIN_INPUT_PULLUP | MUX_MODE1 ) /* gpmc_wait0.gmii2_crs, INPUT_PULLUP | MODE1 */ + 0x074 (PIN_INPUT_PULLUP | MUX_MODE1 ) /* gpmc_wpn.gmii2_rxer, INPUT_PULLUP | MODE1 */ + 0x078 (PIN_INPUT_PULLUP | MUX_MODE1 ) /* gpmc_ben1.gmii2_col, INPUT_PULLUP | MODE1 */ + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + 0x110 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x114 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x118 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x11c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x120 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x124 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x128 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x12c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x130 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x134 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x138 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x13c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x140 (PIN_INPUT_PULLDOWN | MUX_MODE7) + + 0x40 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x44 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x48 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x4c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x50 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x54 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x58 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x5c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x60 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x64 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x68 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x6c (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x070 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x074 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x078 (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + 0x148 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ + 0x14c (PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + 0x148 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x14c (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + mmc1_pins_default: pinmux_mmc1_pins { + pinctrl-single,pins = < + 0x0F0 (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat3.mmc0_dat3 */ + 0x0F4 (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat2.mmc0_dat2 */ + 0x0F8 (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat1.mmc0_dat1 */ + 0x0FC (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat0.mmc0_dat0 */ + 0x100 (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_clk.mmc0_clk */ + 0x104 (PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_cmd.mmc0_cmd */ + 0x1A0 (PIN_INPUT_PULLUP | MUX_MODE7) /* mcasp0_aclkr.gpio3_18 */ + 0x160 (PIN_INPUT | MUX_MODE7) /* spi0_cs1.gpio0_6 */ + >; + }; + + mmc1_pins_sleep: pinmux_mmc1_pins_sleep { + pinctrl-single,pins = < + 0x0F0 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x0F4 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x0F8 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x0FC (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x100 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x104 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x1A0 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x160 (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + 0x80 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + 0x84 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + 0x00 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + 0x04 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + 0x08 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + 0x0c (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + 0x10 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + 0x14 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + 0x18 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + 0x1c (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; + + ecap0_pins_default: backlight_pins { + pinctrl-single,pins = < + 0x164 0x0 /* eCAP0_in_PWM0_out.eCAP0_in_PWM0_out MODE0 */ + >; + }; + + ecap0_pins_sleep: ecap0_pins_sleep { + pinctrl-single,pins = < + 0x164 (PULL_DISABLE | MUX_MODE7) /* eCAP0_in_PWM0_out.eCAP0_in_PWM0_out */ + >; + }; + dcan0_default: dcan0_default_pins { + pinctrl-single,pins = < + 0x178 0x0a /* uart1_ctsn.dcan0_tx_mux2, OUTPUT | MODE2 */ + 0x17c 0x2a /* uart1_rtsn.dcan0_rx_mux2, INPUT | MODE2 */ + >; + }; + }; + +&tps { + compatible = "ti,tps65217"; + regulators { + #address-cells = <1>; + #size-cells = <0>; + + dcdc1_reg: regulator@0 { + reg = <0>; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + reg = <1>; + /* VDD_MPU voltage limits 0.95V - 1.325V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1378000>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + reg = <2>; + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + reg = <3>; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + reg = <4>; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + reg = <5>; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + reg = <6>; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "mii"; +}; + +&cpsw_emac1 { + phy_id = <&davinci_mdio>, <1>; + phy-mode = "mii"; +}; + +&mac { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + slaves = <2>; + dual_emac = <1>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; +}; + +&mmc1 { + status = "okay"; + bus-width = <0x4>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&mmc1_pins_default>; + pinctrl-1 = <&mmc1_pins_sleep>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_HIGH>; + cd-inverted; +}; + +&dcan0 { + pinctrl-names = "default"; + pinctrl-0 = <&dcan0_default>; + status = "okay"; +}; + +&tscadc { + status = "okay"; + tsc { + ti,wires = <4>; + ti,x-plate-resistance = <200>; + ti,coordinate-readouts = <5>; + ti,wire-config = <0x00 0x11 0x22 0x33>; + }; + + adc { + ti,adc-channels = <0 1 2 3>; + }; +}; diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index a048086d10b86..af54fe3796f5c 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -18,7 +18,7 @@ #size-cells = <1>; chosen { }; - aliases { + aliases: aliases { i2c0 = &i2c0; i2c1 = &i2c1; i2c2 = &i2c2; @@ -36,8 +36,8 @@ phy1 = &usb1_phy; ethernet0 = &cpsw_emac0; ethernet1 = &cpsw_emac1; - spi0 = &spi0; - spi1 = &spi1; + spi1 = &spi0; + spi2 = &spi1; }; cpus { @@ -168,7 +168,7 @@ * for the moment, just use a fake OCP bus entry to represent * the whole bus hierarchy. */ - ocp { + ocp: ocp { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; @@ -319,6 +319,39 @@ #interrupt-cells = <2>; reg = <0x44e07000 0x1000>; interrupts = <96>; + gpio-line-names = + "MDIO_DATA", // 0 + "MDIO_CLK", // 1 + "SPI0_SCLK", // 2 + "SPI0_D0", // 3 + "SPI0_D1", // 4 + "SPI0_CS0", // 5 + "SPI0_CS1", // 6 + "ECAP0_IN_PWM0_OUT", // 7 + "LCD_DATA12", // 8 + "LCD_DATA13", // 9 + "LCD_DATA14", // 10 + "LCD_DATA15", // 11 + "UART1_CTSN", // 12 + "UART1_RTSN", // 13 + "UART1_RXD", // 14 + "UART1_TXD", // 15 + "GMII1_TXD3", // 16 + "GMII1_TXD2", // 17 + "USB0_DRVVBUS", // 18 + "XDMA_EVENT_INTR0", // 19 + "XDMA_EVENT_INTR1", // 20 + "GMII1_TXD1", // 21 + "GPMC_AD8", // 22 + "GPMC_AD9", // 23 + "NC", // 24 + "NC", // 25 + "GPMC_AD10", // 26 + "GPMC_AD11", // 27 + "GMII1_TXD0", // 28 + "RMII1_REFCLK", // 29 + "GPMC_WAIT0", // 30 + "GPMC_WPN"; // 31 }; gpio1: gpio@4804c000 { @@ -330,6 +363,39 @@ #interrupt-cells = <2>; reg = <0x4804c000 0x1000>; interrupts = <98>; + gpio-line-names = + "GPMC_AD0", // 0 + "GPMC_AD1", // 1 + "GPMC_AD2", // 2 + "GPMC_AD3", // 3 + "GPMC_AD4", // 4 + "GPMC_AD5", // 5 + "GPMC_AD6", // 6 + "GPMC_AD7", // 7 + "UART0_CTSN", // 8 + "UART0_RTSN", // 9 + "UART0_RXD", // 10 + "UART0_TXD", // 11 + "GPMC_AD12", // 12 + "GPMC_AD13", // 13 + "GPMC_AD14", // 14 + "GPMC_AD15", // 15 + "GPMC_A0", // 16 + "GPMC_A1", // 17 + "GPMC_A2", // 18 + "GPMC_A3", // 19 + "GPMC_A4", // 20 + "GPMC_A5", // 21 + "GPMC_A6", // 22 + "GPMC_A7", // 23 + "GPMC_A8", // 24 + "GPMC_A9", // 25 + "GPMC_A10", // 26 + "GPMC_A11", // 27 + "GPMC_BE1N", // 28 + "GPMC_CSN0", // 29 + "GPMC_CSN1", // 30 + "GPMC_CSN2"; // 31 }; gpio2: gpio@481ac000 { @@ -341,6 +407,39 @@ #interrupt-cells = <2>; reg = <0x481ac000 0x1000>; interrupts = <32>; + gpio-line-names = + "GPMC_CSN3", // 0 + "GPMC_CLK", // 1 + "GPMC_ADVN_ALE", // 2 + "GPMC_OEN_REN", // 3 + "GPMC_WEN", // 4 + "GPMC_BE0N_CLE", // 5 + "LCD_DATA0", // 6 + "LCD_DATA1", // 7 + "LCD_DATA2", // 8 + "LCD_DATA3", // 9 + "LCD_DATA4", // 10 + "LCD_DATA5", // 11 + "LCD_DATA6", // 12 + "LCD_DATA7", // 13 + "LCD_DATA8", // 14 + "LCD_DATA9", // 15 + "LCD_DATA10", // 16 + "LCD_DATA11", // 17 + "GMII1_RXD3", // 18 + "GMII1_RXD2", // 19 + "GMII1_RXD1", // 20 + "GMII1_RXD0", // 21 + "LCD_VSYNC", // 22 + "LCD_HSYNC", // 23 + "LCD_PCLK", // 24 + "LCD_AC_BIAS_EN", // 25 + "MMC0_DAT3", // 26 + "MMC0_DAT2", // 27 + "MMC0_DAT1", // 28 + "MMC0_DAT0", // 29 + "MMC0_CLK", // 30 + "MMC0_CMD"; // 31 }; gpio3: gpio@481ae000 { @@ -352,6 +451,39 @@ #interrupt-cells = <2>; reg = <0x481ae000 0x1000>; interrupts = <62>; + gpio-line-names = + "GMII1_COL", // 0 + "GMII1_CRS", // 1 + "GMII1_RXER", // 2 + "GMII1_TXEN", // 3 + "GMII1_RXDV", // 4 + "I2C0_SDA", // 5 + "I2C0_SCL", // 6 + "EMU0", // 7 + "EMU1", // 8 + "GMII1_TXCLK", // 9 + "GMII1_RXCLK", // 10 + "NC", // 11 + "NC", // 12 + "USB1_DRVVBUS", // 13 + "MCASP0_ACLKX", // 14 + "MCASP0_FSX", // 15 + "MCASP0_AXR0", // 16 + "MCASP0_AHCLKR", // 17 + "MCASP0_ACLKR", // 18 + "MCASP0_FSR", // 19 + "MCASP0_AXR1", // 20 + "MCASP0_AHCLKX", // 21 + "NC", // 22 + "NC", // 23 + "NC", // 24 + "NC", // 25 + "NC", // 26 + "NC", // 27 + "NC", // 28 + "NC", // 29 + "NC", // 30 + "NC"; // 31 }; uart0: serial@44e09000 { @@ -777,6 +909,16 @@ status = "disabled"; }; + eqep0: eqep@0x48300180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48300180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <79>; + status = "disabled"; + }; + ehrpwm0: pwm@48300200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -811,6 +953,17 @@ status = "disabled"; }; + + eqep1: eqep@0x48302180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48302180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <88>; + status = "disabled"; + }; + ehrpwm1: pwm@48302200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -845,6 +998,16 @@ status = "disabled"; }; + eqep2: eqep@0x48304180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48304180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <89>; + status = "disabled"; + }; + ehrpwm2: pwm@48304200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; diff --git a/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts new file mode 100644 index 0000000000000..3bf365ea42c37 --- /dev/null +++ b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include "am5729-beagleboneai.dts" + +/ { + chosen { + base_dtb = "am5729-beagleboneai-roboticscape.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + rcleds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&cape_pins_rc>; + + red_led { + label = "red"; + gpios = <&gpio6 5 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + green_led { + label = "green"; + gpios = <&gpio6 6 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_1_led { + label = "bat25"; + gpios = <&gpio8 18 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_2_led { + label = "bat50"; + gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_3_led { + label = "bat75"; + gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_4_led { + label = "bat100"; + gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; +}; + +&cape_pins { + status = "disabled"; +}; + +&dra7_pmx_core { + cape_pins_rc: cape_pins_rc { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, MUX_MODE15) /* AB8: P8.3: mmc3_dat6.off */ + DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE15) /* AB5: P8.4: mmc3_dat7.off */ + DRA7XX_CORE_IOPAD(0x378C, MUX_MODE15) /* AC9: P8.5: mmc3_dat2.off */ + DRA7XX_CORE_IOPAD(0x3790, MUX_MODE15) /* AC3: P8.6: mmc3_dat3.off */ + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT | MUX_MODE14) /* G14: mcasp1_axr14.gpio6_5 - P8_7 - RED */ + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT | MUX_MODE14) /* F14: mcasp1_axr15.gpio6_6 - P8_8 - GREEN */ + DRA7XX_CORE_IOPAD(0x3698, PIN_INPUT_PULLUP | MUX_MODE14) /* E17: P8.9: xref_clk1.gpio6_18 - PAUSE */ + DRA7XX_CORE_IOPAD(0x36E8, PIN_INPUT_PULLUP | MUX_MODE14) /* A13: P8.10: mcasp1_axr13.gpio6_4 - MODE */ + DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE10) /* AH4: P8.11: vin1a_d7.eQEP2B_in - QEP2B */ + DRA7XX_CORE_IOPAD(0x350C, PIN_INPUT | MUX_MODE10) /* AG6: P8.12: vin1a_d6.eQEP2A_in - QEP2A */ + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT | MUX_MODE10) /* D3: P8.13: vin2a_d10.ehrpwm2B - PWM2B */ + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | MUX_MODE14) /* D5: vin2a_d12.gpio4_13 - P8_14 - BAT100 */ + DRA7XX_CORE_IOPAD(0x3570, MUX_MODE15) /* D1: P8.15a: vin2a_d2.off */ + DRA7XX_CORE_IOPAD(0x35B4, PIN_INPUT | MUX_MODE12) /* A3: P8.15b: vin2a_d19.pr1_pru1_gpi16 - QEP4B */ + DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE12) /* B4: P8.16: vin2a_d21.pr1_pru1_gpi18 - QEP4A */ + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT | MUX_MODE14) /* A7: vout1_d18.gpio8_18 - P8_17 - BAT25 */ + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT | MUX_MODE14) /* F5: vin2a_d8.gpio4_9 - P8_18 - BAT50 */ + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT | MUX_MODE10) /* E6: P8.19: vin2a_d9.ehrpwm2A - PWM2A */ + DRA7XX_CORE_IOPAD(0x3780, MUX_MODE15) /* AC4: P8.20: mmc3_cmd.off */ + DRA7XX_CORE_IOPAD(0x377C, MUX_MODE15) /* AD4: P8.21: mmc3_clk.off */ + DRA7XX_CORE_IOPAD(0x3798, MUX_MODE15) /* AD6: P8.22: mmc3_dat5.off */ + DRA7XX_CORE_IOPAD(0x3794, MUX_MODE15) /* AC8: P8.23: mmc3_dat4.off */ + DRA7XX_CORE_IOPAD(0x3788, MUX_MODE15) /* AC6: P8.24: mmc3_dat1.off */ + DRA7XX_CORE_IOPAD(0x3784, MUX_MODE15) /* AC7: P8.25: mmc3_dat0.off */ + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT | MUX_MODE14) /* B3: vin2a_d20.gpio4_28 - P8_26 - BAT75 */ + DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE15) /* E11: P8.27a: vout1_vsync.off */ + DRA7XX_CORE_IOPAD(0x3628, PIN_OUTPUT | MUX_MODE13) /* A8: P8.27b: vout1_d19.pr2_pru0_gpo16 - SVO1 */ + DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE15) /* D11: P8.28a: vout1_clk.off */ + DRA7XX_CORE_IOPAD(0x362C, PIN_OUTPUT | MUX_MODE13) /* C9: P8.28b: vout1_d20.pr2_pru0_gpo17 - SVO2 */ + DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE15) /* C11: P8.29a: vout1_hsync.off */ + DRA7XX_CORE_IOPAD(0x3630, PIN_OUTPUT | MUX_MODE13) /* A9: P8.29b: vout1_d21.pr2_pru0_gpo18 - SVO3 */ + DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE15) /* B10: P8.30a: vout1_de.off */ + DRA7XX_CORE_IOPAD(0x3634, PIN_OUTPUT | MUX_MODE13) /* B9: P8.30b: vout1_d22.pr2_pru0_gpo19 - SVO4 */ + DRA7XX_CORE_IOPAD(0x3614, MUX_MODE15) /* C8: P8.31a: vout1_d14.off */ + DRA7XX_CORE_IOPAD(0x373C, MUX_MODE15) /* G16: P8.31b: mcasp4_axr0.off */ + DRA7XX_CORE_IOPAD(0x3618, MUX_MODE15) /* C7: P8.32a: vout1_d15.off */ + DRA7XX_CORE_IOPAD(0x3740, MUX_MODE15) /* D17: P8.32b: mcasp4_axr1.off */ + DRA7XX_CORE_IOPAD(0x3610, MUX_MODE15) /* C6: P8.33a: vout1_d13.off */ + DRA7XX_CORE_IOPAD(0x34E8, PIN_INPUT | MUX_MODE10) /* AF9: P8.33b: vin1a_fld0.eQEP1B_in - QEP1B */ + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT | MUX_MODE14) /* D8: P8.34a: vout1_d11.gpio8_11 - MDIR2B */ + DRA7XX_CORE_IOPAD(0x3564, MUX_MODE15) /* G6: P8.34b: vin2a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x360C, MUX_MODE15) /* A5: P8.35a: vout1_d12.off */ + DRA7XX_CORE_IOPAD(0x34E4, PIN_INPUT | MUX_MODE10) /* AD9: P8.35b: vin1a_de0.eQEP1A_in - QEP1A */ + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT | MUX_MODE14) /* D7: P8.36a: vout1_d10.gpio8_10 - SERVO_EN */ + DRA7XX_CORE_IOPAD(0x3568, MUX_MODE15) /* F2: P8.36b: vin2a_d0.off */ + DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE15) /* E8: P8.37a: vout1_d8.off */ + DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT | MUX_MODE3) /* A21: P8.37b: mcasp4_fsx.uart8_txd - UART5_TX */ + DRA7XX_CORE_IOPAD(0x3600, MUX_MODE15) /* D9: P8.38a: vout1_d9.off */ + DRA7XX_CORE_IOPAD(0x3734, PIN_INPUT | MUX_MODE3) /* C18: P8.38b: mcasp4_aclkx.uart8_rxd - UART5_RX */ + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT | MUX_MODE13) /* F8: P8.39: vout1_d6.pr2_pru0_gpo3 - SVO5 */ + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT | MUX_MODE13) /* E7: P8.40: vout1_d7.pr2_pru0_gpo4 - SVO6 */ + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT | MUX_MODE13) /* E9: P8.41: vout1_d4.pr2_pru0_gpo1 - SVO7 */ + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT | MUX_MODE13) /* F9: P8.42: vout1_d5.pr2_pru0_gpo2 - SVO8 */ + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT | MUX_MODE14 ) /* F10: P8.43: vout1_d2.gpio8_2 - MDIR3B */ + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT | MUX_MODE14 ) /* G11: P8.44: vout1_d3.gpio8_3 - MDIR3A */ + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT | MUX_MODE14 ) /* F11: P8.45a: vout1_d0.gpio8_0 - MDIR4A */ + DRA7XX_CORE_IOPAD(0x361C, MUX_MODE15) /* B7: P8.45b: vout1_d16.off */ + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT | MUX_MODE14 ) /* G10: P8.46a: vout1_d1.gpio8_1 - MDIR4B */ + DRA7XX_CORE_IOPAD(0x3638, MUX_MODE15) /* A10: P8.46b: vout1_d23.off */ + DRA7XX_CORE_IOPAD(0x372C, PIN_INPUT | MUX_MODE4) /* B19: P9.11a: mcasp3_axr0.uart5_rxd - UART4_RX */ + DRA7XX_CORE_IOPAD(0x3620, MUX_MODE15) /* B8: P9.11b: vout1_d17.off */ + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT | MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.gpio5_0 - MDIR1A */ + DRA7XX_CORE_IOPAD(0x3730, PIN_OUTPUT | MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off - MDIR1B */ + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT | MUX_MODE10) /* D6: P9.14: vin2a_d17.ehrpwm3A - PWM1A */ + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT | MUX_MODE14) /* AG4: P9.15: vin1a_d8.gpio3_12 - MDIR2A */ + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT | MUX_MODE10) /* C5: P9.16: vin2a_d18.ehrpwm3B - PWM1B */ + DRA7XX_CORE_IOPAD(0x37CC, PIN_INPUT | MUX_MODE14) /* B24: P9.17a: spi2_cs0.gpio7_17 */ + DRA7XX_CORE_IOPAD(0x36B8, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* F12: P9.17b: mcasp1_axr1.i2c5_scl - I2C1_SCL */ + DRA7XX_CORE_IOPAD(0x37C8, PIN_INPUT | MUX_MODE14) /* G17: P9.18a: spi2_d0.gpio7_16 */ + DRA7XX_CORE_IOPAD(0x36B4, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* G12: P9.18b: mcasp1_axr0.i2c5_sda - I2C1_SDA */ + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl - I2C2_SCL */ + DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */ + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda - I2C2_SDA */ + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */ + DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE15) /* AF8: P9.21a: vin1a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT | MUX_MODE1) /* B22: P9.21b: spi2_d1.uart3_txd - UART2_TX */ + DRA7XX_CORE_IOPAD(0x369C, MUX_MODE15) /* B26: P9.22a: xref_clk2.off */ + DRA7XX_CORE_IOPAD(0x37C0, PIN_INPUT | MUX_MODE1) /* A26: P9.22b: spi2_sclk.uart3_rxd - UART2_RX */ + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT | MUX_MODE14) /* A22: P9.23: spi1_cs1.gpio7_11 - SPI1_SS2 */ + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT | MUX_MODE3) /* F20: P9.24: gpio6_15.uart10_txd - UART1_TX */ + DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE14) /* D18: P9.25: xref_clk0.gpio6_17 - IMU_INT */ + DRA7XX_CORE_IOPAD(0x3688, PIN_INPUT | MUX_MODE3) /* E21: P9.26a: gpio6_14.uart10_rxd - UART1_RX */ + DRA7XX_CORE_IOPAD(0x3544, MUX_MODE15) /* AE2: P9.26b: vin1a_d20.off */ + DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE10) /* C3: P9.27a: vin2a_d14.eQEP3B_in - QEP0B */ + DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE15) /* J14: P9.27b: mcasp1_fsr.off */ + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT | MUX_MODE3) /* A12: P9.28: mcasp1_axr11.spi3_cs0 - SPI1_SS1 */ + DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE3) /* A11: P9.29a: mcasp1_axr9.spi3_d1 - SPI1_MISO */ + DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE15) /* D14: P9.29b: mcasp1_fsx.off */ + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT | MUX_MODE3) /* B13: P9.30: mcasp1_axr10.spi3_d0 - SPI1_MOSI */ + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT | MUX_MODE3) /* B12: P9.31a: mcasp1_axr8.spi3_sclk - SPI1_SCK */ + DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE15) /* C14: P9.31b: mcasp1_aclkx.off */ + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT | MUX_MODE14) /* C23: P9.41a: xref_clk3.gpio6_20 - MOT_STBY */ + DRA7XX_CORE_IOPAD(0x3580, MUX_MODE15) /* C1: P9.41b: vin2a_d6.off */ + DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE15) /* E14: P9.42a: mcasp1_axr12.off */ + DRA7XX_CORE_IOPAD(0x359C, PIN_INPUT | MUX_MODE10) /* C2: P9.42b: vin2a_d13.eQEP3A_in - QEP0A */ + >; + }; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <100000>; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm1 { + status = "okay"; +}; + +&ehrpwm2 { + status = "okay"; +}; + +&uart8 { + status = "okay"; +}; + +&uart5 { + status = "okay"; +}; + +&uart3 { + status = "okay"; +}; + +&uart10 { + status = "okay"; +}; + +&mcspi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + cs-gpios = <0>, <&gpio7 11 0>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; diff --git a/arch/arm/boot/dts/am5729-beagleboneai.dts b/arch/arm/boot/dts/am5729-beagleboneai.dts new file mode 100644 index 0000000000000..eeb81909bb9be --- /dev/null +++ b/arch/arm/boot/dts/am5729-beagleboneai.dts @@ -0,0 +1,1153 @@ +/* + * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "dra74x.dtsi" +#include "dra74x-mmc-iodelay.dtsi" +#include "dra74-ipu-dsp-common.dtsi" +#include +#include +#include + +/ { + model = "BeagleBoard.org BeagleBone AI"; + compatible = "beagleboard.org,am5729-beagleboneai", "ti,am5728", "ti,dra742", "ti,dra74", "ti,dra7"; + + aliases { + rtc0 = &tps659038_rtc; + rtc1 = &rtc; + display0 = &hdmi_conn; + }; + + chosen { + stdout-path = &uart1; + base_dtb = "am5729-beagleboneai.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + memory@0 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 { + reg = <0x0 0x40500000 0x0 0x100000>; + no-map; + status = "okay"; + }; + + ipu2_memory_region: ipu2-memory@95800000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x95800000 0x0 0x3800000>; + reusable; + status = "okay"; + }; + + dsp1_memory_region: dsp1-memory@99000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x99000000 0x0 0x4000000>; + reusable; + status = "okay"; + }; + + ipu1_memory_region: ipu1-memory@9d000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x9d000000 0x0 0x2000000>; + reusable; + status = "okay"; + }; + + dsp2_memory_region: dsp2-memory@9f000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x9f000000 0x0 0x800000>; + reusable; + status = "okay"; + }; + + cmem_block_mem_0: cmem_block_mem@a0000000 { + reg = <0x0 0xa0000000 0x0 0x18000000>; + no-map; + status = "okay"; + }; + }; + + cmem { + compatible = "ti,cmem"; + #address-cells = <1>; + #size-cells = <0>; + #pool-size-cells = <2>; + status = "okay"; + + cmem_block_0: cmem_block@0 { + reg = <0>; + memory-region = <&cmem_block_mem_0>; + cmem-buf-pools = <1 0x0 0x18000000>; + }; + + cmem_block_1: cmem_block@1 { + reg = <1>; + memory-region = <&cmem_block_mem_1_ocmc3>; + }; + }; + + vdd_adc: gpioregulator-vdd_adc { + compatible = "regulator-gpio"; + regulator-name = "vdd_adc"; + vin-supply = <&vdd_5v>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + gpios = <&gpio3 27 GPIO_ACTIVE_HIGH>; + states = <1800000 0 + 3300000 1>; + }; + + vdd_5v: fixedregulator-vdd_5v { + compatible = "regulator-fixed"; + regulator-name = "vdd_5v"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + regulator-boot-on; + }; + + vtt_fixed: fixedregulator-vtt { + /* TPS51200 */ + compatible = "regulator-fixed"; + regulator-name = "vtt_fixed"; + vin-supply = <&vdd_ddr>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&led_pins_default>; + + led0 { + label = "beaglebone:green:usr0"; + gpios = <&gpio3 17 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led1 { + label = "beaglebone:green:usr1"; + gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led2 { + label = "beaglebone:green:usr2"; + gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr3"; + gpios = <&gpio3 14 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + + led4 { + label = "beaglebone:green:usr4"; + gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "netdev"; + default-state = "off"; + }; + }; + + hdmi_conn: connector@0 { + compatible = "hdmi-connector"; + label = "hdmi"; + type = "a"; + + port { + hdmi_connector_in: endpoint { + remote-endpoint = <&hdmi_encoder_out>; + }; + }; + }; + + hdmi_enc: encoder@0 { + compatible = "ti,tpd12s016", "ti,tpd12s015"; + gpios = <0>, + <0>, + <&gpio7 12 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <0x1>; + #size-cells = <0x0>; + + port@0 { + reg = <0x0>; + + hdmi_encoder_in: endpoint@0 { + remote-endpoint = <&hdmi_out>; + }; + }; + + port@1 { + reg = <0x1>; + + hdmi_encoder_out: endpoint@0 { + remote-endpoint = <&hdmi_connector_in>; + }; + }; + }; + }; + + emmc_pwrseq: emmc_pwrseq { + compatible = "mmc-pwrseq-emmc"; + reset-gpios = <&gpio5 7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pwrseq_pins_default>; + }; + + brcmf_pwrseq: brcmf_pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&brcmf_pwrseq_pins_default>; + reset-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>, // BT-REG-ON + <&gpio3 18 GPIO_ACTIVE_LOW>; // WL-REG-ON + }; + + unused_pins: unused_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&unused_pins_default>; + }; + + cape_pins: cape_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&cape_pins_default>; + }; + + extcon_usb1: extcon_usb1 { + compatible = "linux,extcon-usb-gpio"; + ti,enable-id-detection; + id-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&extcon_usb1_pins_default>; + }; +}; + +&ocp { + pruss1_shmem: pruss_shmem@4b200000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b200000 0x020000>; + }; + + pruss2_shmem: pruss_shmem@4b280000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b280000 0x020000>; + }; +}; + +&dra7_pmx_core { + extcon_usb1_pins_default: extcon_usb1_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3518, PIN_INPUT | MUX_MODE14) /* AG2: vin1a_d9.gpio3_13 - USR0 */ + >; + }; + + led_pins_default: led_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3528, PIN_OUTPUT | MUX_MODE14) /* AF6: vin1a_d13.gpio3_17 - USR0 */ + DRA7XX_CORE_IOPAD(0x36c0, PIN_OUTPUT | MUX_MODE14) /* J11: mcasp1_axr3.gpio5_5 - USR1 */ + DRA7XX_CORE_IOPAD(0x3520, PIN_OUTPUT | MUX_MODE14) /* AG5: vin1a_d12.gpio3_15 - USR2 */ + DRA7XX_CORE_IOPAD(0x351c, PIN_OUTPUT | MUX_MODE14) /* AG3: vin1a_d10.gpio3_14 - USR3 */ + DRA7XX_CORE_IOPAD(0x3500, PIN_OUTPUT | MUX_MODE14) /* AH6: vin1a_d3.gpio3_7 - USR4 */ + >; + }; + + microsd_extra_pins_default: microsd_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT_PULLUP | MUX_MODE14) /* W7: mmc1_sdcd.gpio6_27 - MMC1_SDCD */ + >; + }; + + ethphy_extra_pins_default: ethphy_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34c4, PIN_OUTPUT_PULLUP | MUX_MODE14) /* N1: gpmc_advn_ale.gpio2_23 - RGMII_RST */ + DRA7XX_CORE_IOPAD(0x364c, PIN_INPUT | MUX_MODE14) /* Y1: uart3_txd.gpio5_19 - MII0_INT */ + >; + }; + + emmc_pwrseq_pins_default: emmc_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36c8, PIN_OUTPUT_PULLUP | MUX_MODE14) /* F13: mcasp1_axr5.gpio5_7 - eMMC_RSTn */ + >; + }; + + brcmf_pwrseq_pins_default: brcmf_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x352c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AF3: vin1a_d14.gpio3_18 - WL_REG_ON */ + DRA7XX_CORE_IOPAD(0x353c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AE5: vin1a_d18.gpio3_22 - BT_REG_ON */ + >; + }; + + wifibt_extra_pins_default: wifibt_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3540, PIN_INPUT | MUX_MODE14) /* AE1: vin1a_d19.gpio3_23 - WL_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3450, PIN_INPUT | MUX_MODE8) /* P6: vin1a_d20.uart6_rxd - UART6_RXD */ + DRA7XX_CORE_IOPAD(0x3454, PIN_INPUT | MUX_MODE8) /* R9: vin1a_d21.uart6_txd - UART6_TXD */ + DRA7XX_CORE_IOPAD(0x3458, PIN_INPUT | MUX_MODE8) /* R5: vin1a_d22.uart6_ctsn - UART6_CTSN */ + DRA7XX_CORE_IOPAD(0x345c, PIN_INPUT | MUX_MODE8) /* P5: vin1a_d23.uart6_rtsn - UART6_RTSN */ + DRA7XX_CORE_IOPAD(0x3534, PIN_INPUT_PULLDOWN | MUX_MODE14) /* AF1: vin1a_d16.gpio3_20 - BT_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3538, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* AE3: vin1a_d6.gpio3_21 - BT_WAKE */ + >; + }; + + adc_pins_default: adc_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3550, PIN_OUTPUT | MUX_MODE14) /* AD3: vin1a_d23.gpio3_27 - VDD_ADC_SEL */ + DRA7XX_CORE_IOPAD(0x34DC, PIN_INPUT_PULLUP | MUX_MODE14) /* AG8: vin1a_clk0.gpio2_30 - INT_ADC */ + >; + }; + + pmic_pins_default: pmic_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3690, PIN_INPUT_PULLUP | MUX_MODE14) /* F21: gpio6_16.gpio6_16 - PMIC_INT */ + >; + }; + + hdmi_pins_default: hdmi_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* C25: i2c2_sda.hdmi1_ddc_scl - HDMI_DDC_SCL */ + DRA7XX_CORE_IOPAD(0x380C, PIN_INPUT | MUX_MODE1) /* F17: i2c2_scl.hdmi1_ddc_sda - HDMI_DDC_SDA */ + DRA7XX_CORE_IOPAD(0x37BC, PIN_INPUT | MUX_MODE6) /* B20: spi1_cs3.hdmi1_cec - HDMI_DDC_CEC */ +#if 0 + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE6) /* B21: spi1_cs2.hdmi1_hpd - HDMI_DDC_HPD */ +#else + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE14) /* B21: spi1_cs2.gpio7_12 - HDMI_DDC_HPD */ +#endif + >; + }; + + unused_pins_default: unused_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3400, MUX_MODE15) /* M6: sysboot0 */ + DRA7XX_CORE_IOPAD(0x3404, MUX_MODE15) /* M2: sysboot1 */ + DRA7XX_CORE_IOPAD(0x3408, MUX_MODE15) /* L5: sysboot2 */ + DRA7XX_CORE_IOPAD(0x340C, MUX_MODE15) /* M1: sysboot3 */ + DRA7XX_CORE_IOPAD(0x3410, MUX_MODE15) /* L6: sysboot4 */ + DRA7XX_CORE_IOPAD(0x3414, MUX_MODE15) /* L4: sysboot5 */ + DRA7XX_CORE_IOPAD(0x3418, MUX_MODE15) /* L3: sysboot6 */ + DRA7XX_CORE_IOPAD(0x341C, MUX_MODE15) /* L2: sysboot7 */ + DRA7XX_CORE_IOPAD(0x3420, MUX_MODE15) /* L1: sysboot8 */ + DRA7XX_CORE_IOPAD(0x3424, MUX_MODE15) /* K2: sysboot9 */ + DRA7XX_CORE_IOPAD(0x3428, MUX_MODE15) /* J1: sysboot10 */ + DRA7XX_CORE_IOPAD(0x342C, MUX_MODE15) /* J2: sysboot11 */ + DRA7XX_CORE_IOPAD(0x3430, MUX_MODE15) /* H1: sysboot12 */ + DRA7XX_CORE_IOPAD(0x3434, MUX_MODE15) /* J3: sysboot13 */ + DRA7XX_CORE_IOPAD(0x3438, MUX_MODE15) /* H2: sysboot14 */ + DRA7XX_CORE_IOPAD(0x343C, MUX_MODE15) /* H3: sysboot15 */ + + DRA7XX_CORE_IOPAD(0x3448, MUX_MODE15) /* T6: N/C */ + DRA7XX_CORE_IOPAD(0x344C, MUX_MODE15) /* T7: N/C */ + + DRA7XX_CORE_IOPAD(0x3460, MUX_MODE15) /* N7: N/C */ + DRA7XX_CORE_IOPAD(0x3464, MUX_MODE15) /* R4: N/C */ + DRA7XX_CORE_IOPAD(0x3468, MUX_MODE15) /* N9: N/C */ + DRA7XX_CORE_IOPAD(0x346C, MUX_MODE15) /* P9: N/C */ + DRA7XX_CORE_IOPAD(0x3470, MUX_MODE15) /* P4: N/C */ + DRA7XX_CORE_IOPAD(0x3474, MUX_MODE15) /* R3: N/C */ + DRA7XX_CORE_IOPAD(0x3478, MUX_MODE15) /* T2: N/C */ + DRA7XX_CORE_IOPAD(0x347C, MUX_MODE15) /* U2: N/C */ + DRA7XX_CORE_IOPAD(0x3480, MUX_MODE15) /* U1: N/C */ + DRA7XX_CORE_IOPAD(0x3484, MUX_MODE15) /* P3: N/C */ + DRA7XX_CORE_IOPAD(0x3488, MUX_MODE15) /* R2: N/C */ + + DRA7XX_CORE_IOPAD(0x34B4, MUX_MODE15) /* T1: N/C */ + DRA7XX_CORE_IOPAD(0x34B8, MUX_MODE15) /* P2: N/C */ + DRA7XX_CORE_IOPAD(0x34BC, MUX_MODE15) /* P1: N/C */ + DRA7XX_CORE_IOPAD(0x34C0, MUX_MODE15) /* P7: N/C */ + DRA7XX_CORE_IOPAD(0x34C4, MUX_MODE15) /* N1: N/C */ + DRA7XX_CORE_IOPAD(0x34C8, MUX_MODE15) /* M5: N/C */ + DRA7XX_CORE_IOPAD(0x34CC, MUX_MODE15) /* M3: N/C */ + DRA7XX_CORE_IOPAD(0x34D0, MUX_MODE15) /* N6: N/C */ + DRA7XX_CORE_IOPAD(0x34D4, MUX_MODE15) /* M4: N/C */ + DRA7XX_CORE_IOPAD(0x34D8, MUX_MODE15) /* N2: N/C */ + + DRA7XX_CORE_IOPAD(0x34E0, MUX_MODE15) /* AH7: N/C */ + + DRA7XX_CORE_IOPAD(0x34EC, MUX_MODE15) /* AE9: N/C */ + + DRA7XX_CORE_IOPAD(0x34F4, MUX_MODE15) /* AE8: N/C */ + DRA7XX_CORE_IOPAD(0x34F8, MUX_MODE15) /* AD8: N/C */ + DRA7XX_CORE_IOPAD(0x34FC, MUX_MODE15) /* AG7: N/C */ + + DRA7XX_CORE_IOPAD(0x3504, MUX_MODE15) /* AH3: N/C */ + DRA7XX_CORE_IOPAD(0x3508, MUX_MODE15) /* AH5: N/C */ + + DRA7XX_CORE_IOPAD(0x3524, MUX_MODE15) /* AF2: N/C */ + + DRA7XX_CORE_IOPAD(0x3530, MUX_MODE15) /* AF4: N/C */ + + DRA7XX_CORE_IOPAD(0x354C, MUX_MODE15) /* AD2: N/C */ + + DRA7XX_CORE_IOPAD(0x3554, MUX_MODE15) /* E1: N/C */ + DRA7XX_CORE_IOPAD(0x3558, MUX_MODE15) /* G2: N/C */ + DRA7XX_CORE_IOPAD(0x355C, MUX_MODE15) /* H7: N/C */ + DRA7XX_CORE_IOPAD(0x3560, MUX_MODE15) /* G1: N/C */ + + DRA7XX_CORE_IOPAD(0x356C, MUX_MODE15) /* F3: N/C */ + + DRA7XX_CORE_IOPAD(0x3574, MUX_MODE15) /* E2: N/C */ + + DRA7XX_CORE_IOPAD(0x3584, MUX_MODE15) /* E4: N/C */ + + DRA7XX_CORE_IOPAD(0x3594, MUX_MODE15) /* F6: N/C */ + + DRA7XX_CORE_IOPAD(0x35A4, MUX_MODE15) /* C4: N/C */ + DRA7XX_CORE_IOPAD(0x35A8, MUX_MODE15) /* B2: N/C */ + + DRA7XX_CORE_IOPAD(0x35C0, MUX_MODE15) /* B5: N/C */ + DRA7XX_CORE_IOPAD(0x35C4, MUX_MODE15) /* A4: N/C */ + + DRA7XX_CORE_IOPAD(0x35D0, MUX_MODE15) /* B11: N/C */ + + DRA7XX_CORE_IOPAD(0x3644, MUX_MODE15) /* U3: N/C */ + + DRA7XX_CORE_IOPAD(0x3680, MUX_MODE15) /* AB10: N/C */ + + DRA7XX_CORE_IOPAD(0x36BC, MUX_MODE15) /* G13: N/C */ + + DRA7XX_CORE_IOPAD(0x36C4, MUX_MODE15) /* E12: N/C */ + + DRA7XX_CORE_IOPAD(0x36CC, MUX_MODE15) /* C12: N/C */ + DRA7XX_CORE_IOPAD(0x36D0, MUX_MODE15) /* D12: N/C */ + + DRA7XX_CORE_IOPAD(0x36FC, MUX_MODE15) /* E15: N/C */ + DRA7XX_CORE_IOPAD(0x3700, MUX_MODE15) /* A20: N/C */ + + DRA7XX_CORE_IOPAD(0x370C, MUX_MODE15) /* C15: N/C */ + DRA7XX_CORE_IOPAD(0x3710, MUX_MODE15) /* A16: N/C */ + DRA7XX_CORE_IOPAD(0x3714, MUX_MODE15) /* D15: N/C */ + DRA7XX_CORE_IOPAD(0x3718, MUX_MODE15) /* B16: N/C */ + DRA7XX_CORE_IOPAD(0x371C, MUX_MODE15) /* B17: N/C */ + DRA7XX_CORE_IOPAD(0x3720, MUX_MODE15) /* A17: N/C */ + DRA7XX_CORE_IOPAD(0x3724, MUX_MODE15) /* B18: N/C */ + DRA7XX_CORE_IOPAD(0x3728, MUX_MODE15) /* F15: N/C */ + + DRA7XX_CORE_IOPAD(0x3744, MUX_MODE15) /* AA3: N/C */ + DRA7XX_CORE_IOPAD(0x3748, MUX_MODE15) /* AB9: N/C */ + DRA7XX_CORE_IOPAD(0x374C, MUX_MODE15) /* AB3: N/C */ + DRA7XX_CORE_IOPAD(0x3750, MUX_MODE15) /* AA4: N/C */ + + DRA7XX_CORE_IOPAD(0x3770, MUX_MODE15) /* Y9: N/C */ + DRA7XX_CORE_IOPAD(0x3774, MUX_MODE15) /* AC5: N/C */ + DRA7XX_CORE_IOPAD(0x3778, MUX_MODE15) /* AB4: N/C */ + + DRA7XX_CORE_IOPAD(0x37A4, MUX_MODE15) /* A25: N/C */ + DRA7XX_CORE_IOPAD(0x37A8, MUX_MODE15) /* F16: N/C */ + DRA7XX_CORE_IOPAD(0x37AC, MUX_MODE15) /* B25: N/C */ + DRA7XX_CORE_IOPAD(0x37B0, MUX_MODE15) /* A24: N/C */ + + DRA7XX_CORE_IOPAD(0x37D0, MUX_MODE15) /* G20: N/C */ + DRA7XX_CORE_IOPAD(0x37D4, MUX_MODE15) /* G19: N/C */ + + DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15) /* AD17: N/C */ + DRA7XX_CORE_IOPAD(0x381C, MUX_MODE15) /* AC17: N/C */ + DRA7XX_CORE_IOPAD(0x3820, MUX_MODE15) /* AB16: N/C */ + DRA7XX_CORE_IOPAD(0x3824, MUX_MODE15) /* AC16: N/C */ + >; + }; + + cape_pins_default: cape_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, MUX_MODE14) /* AB8: P8.3: mmc3_dat6.off */ + DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE14) /* AB5: P8.4: mmc3_dat7.off */ + DRA7XX_CORE_IOPAD(0x378C, MUX_MODE14) /* AC9: P8.5: mmc3_dat2.off */ + DRA7XX_CORE_IOPAD(0x3790, MUX_MODE14) /* AC3: P8.6: mmc3_dat3.off */ + DRA7XX_CORE_IOPAD(0x36EC, MUX_MODE14) /* G14: P8.7: mcasp1_axr14.off */ + DRA7XX_CORE_IOPAD(0x36F0, MUX_MODE14) /* F14: P8.8: mcasp1_axr15.off */ + DRA7XX_CORE_IOPAD(0x3698, MUX_MODE14) /* E17: P8.9: xref_clk1.off */ + DRA7XX_CORE_IOPAD(0x36E8, MUX_MODE14) /* A13: P8.10: mcasp1_axr13.off */ + DRA7XX_CORE_IOPAD(0x3510, MUX_MODE14) /* AH4: P8.11: vin1a_d7.off */ + DRA7XX_CORE_IOPAD(0x350C, MUX_MODE14) /* AG6: P8.12: vin1a_d6.off */ + DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE12) /* D3: P8.13: vin2a_d10.off */ + DRA7XX_CORE_IOPAD(0x3598, MUX_MODE14) /* D5: P8.14: vin2a_d12.off */ + DRA7XX_CORE_IOPAD(0x3570, MUX_MODE14) /* D1: P8.15a: vin2a_d2.off */ + DRA7XX_CORE_IOPAD(0x35B4, MUX_MODE13) /* A3: P8.15b: vin2a_d19.off */ + DRA7XX_CORE_IOPAD(0x35BC, MUX_MODE13) /* B4: P8.16: vin2a_d21.off */ + DRA7XX_CORE_IOPAD(0x3624, MUX_MODE14) /* A7: P8.17: vout1_d18.off */ + DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE12) /* F5: P8.18: vin2a_d8.off */ + DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE12) /* E6: P8.19: vin2a_d9.off */ + DRA7XX_CORE_IOPAD(0x3780, MUX_MODE14) /* AC4: P8.20: mmc3_cmd.off */ + DRA7XX_CORE_IOPAD(0x377C, MUX_MODE14) /* AD4: P8.21: mmc3_clk.off */ + DRA7XX_CORE_IOPAD(0x3798, MUX_MODE14) /* AD6: P8.22: mmc3_dat5.off */ + DRA7XX_CORE_IOPAD(0x3794, MUX_MODE14) /* AC8: P8.23: mmc3_dat4.off */ + DRA7XX_CORE_IOPAD(0x3788, MUX_MODE14) /* AC6: P8.24: mmc3_dat1.off */ + DRA7XX_CORE_IOPAD(0x3784, MUX_MODE14) /* AC7: P8.25: mmc3_dat0.off */ + DRA7XX_CORE_IOPAD(0x35B8, MUX_MODE13) /* B3: P8.26: vin2a_d20.off */ + DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE14) /* E11: P8.27a: vout1_vsync.off */ + DRA7XX_CORE_IOPAD(0x3628, MUX_MODE14) /* A8: P8.27b: vout1_d19.off */ + DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE14) /* D11: P8.28a: vout1_clk.off */ + DRA7XX_CORE_IOPAD(0x362C, MUX_MODE14) /* C9: P8.28b: vout1_d20.off */ + DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE14) /* C11: P8.29a: vout1_hsync.off */ + DRA7XX_CORE_IOPAD(0x3630, MUX_MODE14) /* A9: P8.29b: vout1_d21.off */ + DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE14) /* B10: P8.30a: vout1_de.off */ + DRA7XX_CORE_IOPAD(0x3634, MUX_MODE14) /* B9: P8.30b: vout1_d22.off */ + DRA7XX_CORE_IOPAD(0x3614, MUX_MODE14) /* C8: P8.31a: vout1_d14.off */ + DRA7XX_CORE_IOPAD(0x373C, MUX_MODE14) /* G16: P8.31b: mcasp4_axr0.off */ + DRA7XX_CORE_IOPAD(0x3618, MUX_MODE14) /* C7: P8.32a: vout1_d15.off */ + DRA7XX_CORE_IOPAD(0x3740, MUX_MODE14) /* D17: P8.32b: mcasp4_axr1.off */ + DRA7XX_CORE_IOPAD(0x3610, MUX_MODE14) /* C6: P8.33a: vout1_d13.off */ + DRA7XX_CORE_IOPAD(0x34E8, MUX_MODE14) /* AF9: P8.33b: vin1a_fld0.off */ + DRA7XX_CORE_IOPAD(0x3608, MUX_MODE14) /* D8: P8.34a: vout1_d11.off */ + DRA7XX_CORE_IOPAD(0x3564, MUX_MODE14) /* G6: P8.34b: vin2a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x360C, MUX_MODE14) /* A5: P8.35a: vout1_d12.off */ + DRA7XX_CORE_IOPAD(0x34E4, MUX_MODE14) /* AD9: P8.35b: vin1a_de0.off */ + DRA7XX_CORE_IOPAD(0x3604, MUX_MODE14) /* D7: P8.36a: vout1_d10.off */ + DRA7XX_CORE_IOPAD(0x3568, MUX_MODE14) /* F2: P8.36b: vin2a_d0.off */ + DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE14) /* E8: P8.37a: vout1_d8.off */ + DRA7XX_CORE_IOPAD(0x3738, MUX_MODE14) /* A21: P8.37b: mcasp4_fsx.off */ + DRA7XX_CORE_IOPAD(0x3600, MUX_MODE14) /* D9: P8.38a: vout1_d9.off */ + DRA7XX_CORE_IOPAD(0x3734, MUX_MODE14) /* C18: P8.38b: mcasp4_aclkx.off */ + DRA7XX_CORE_IOPAD(0x35F4, MUX_MODE14) /* F8: P8.39: vout1_d6.off */ + DRA7XX_CORE_IOPAD(0x35F8, MUX_MODE14) /* E7: P8.40: vout1_d7.off */ + DRA7XX_CORE_IOPAD(0x35EC, MUX_MODE14) /* E9: P8.41: vout1_d4.off */ + DRA7XX_CORE_IOPAD(0x35F0, MUX_MODE14) /* F9: P8.42: vout1_d5.off */ + DRA7XX_CORE_IOPAD(0x35E4, MUX_MODE14) /* F10: P8.43: vout1_d2.off */ + DRA7XX_CORE_IOPAD(0x35E8, MUX_MODE14) /* G11: P8.44: vout1_d3.off */ + DRA7XX_CORE_IOPAD(0x35DC, MUX_MODE14) /* F11: P8.45a: vout1_d0.off */ + DRA7XX_CORE_IOPAD(0x361C, MUX_MODE14) /* B7: P8.45b: vout1_d16.off */ + DRA7XX_CORE_IOPAD(0x35E0, MUX_MODE14) /* G10: P8.46a: vout1_d1.off */ + DRA7XX_CORE_IOPAD(0x3638, MUX_MODE14) /* A10: P8.46b: vout1_d23.off */ + DRA7XX_CORE_IOPAD(0x372C, MUX_MODE14) /* B19: P9.11a: mcasp3_axr0.off */ + DRA7XX_CORE_IOPAD(0x3620, MUX_MODE14) /* B8: P9.11b: vout1_d17.off */ + DRA7XX_CORE_IOPAD(0x36AC, MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.off */ + DRA7XX_CORE_IOPAD(0x3730, MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off */ + DRA7XX_CORE_IOPAD(0x35AC, MUX_MODE10) /* D6: P9.14: vin2a_d17.off */ + DRA7XX_CORE_IOPAD(0x3514, MUX_MODE14) /* AG4: P9.15: vin1a_d8.off */ + DRA7XX_CORE_IOPAD(0x35B0, MUX_MODE13) /* C5: P9.16: vin2a_d18.off */ + DRA7XX_CORE_IOPAD(0x37CC, MUX_MODE14) /* B24: P9.17a: spi2_cs0.off */ + DRA7XX_CORE_IOPAD(0x36B8, MUX_MODE14) /* F12: P9.17b: mcasp1_axr1.off */ + DRA7XX_CORE_IOPAD(0x37C8, MUX_MODE14) /* G17: P9.18a: spi2_d0.off */ + DRA7XX_CORE_IOPAD(0x36B4, MUX_MODE14) /* G12: P9.18b: mcasp1_axr0.off */ + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT_PULLUP | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl */ + DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT_PULLUP | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */ + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT_PULLUP | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda */ + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT_PULLUP | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */ + DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE14) /* AF8: P9.21a: vin1a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x37C4, MUX_MODE14) /* B22: P9.21b: spi2_d1.off */ + DRA7XX_CORE_IOPAD(0x369C, MUX_MODE14) /* B26: P9.22a: xref_clk2.off */ + DRA7XX_CORE_IOPAD(0x37C0, MUX_MODE14) /* A26: P9.22b: spi2_sclk.off */ + DRA7XX_CORE_IOPAD(0x37B4, MUX_MODE14) /* A22: P9.23: spi1_cs1.off */ + DRA7XX_CORE_IOPAD(0x368C, MUX_MODE14) /* F20: P9.24: gpio6_15.off */ + DRA7XX_CORE_IOPAD(0x3694, MUX_MODE14) /* D18: P9.25: xref_clk0.off */ + DRA7XX_CORE_IOPAD(0x3688, MUX_MODE14) /* E21: P9.26a: gpio6_14.off */ + DRA7XX_CORE_IOPAD(0x3544, MUX_MODE14) /* AE2: P9.26b: vin1a_d20.off */ + DRA7XX_CORE_IOPAD(0x35A0, MUX_MODE14) /* C3: P9.27a: vin2a_d14.off */ + DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE14) /* J14: P9.27b: mcasp1_fsr.off */ + DRA7XX_CORE_IOPAD(0x36E0, MUX_MODE14) /* A12: P9.28: mcasp1_axr11.off */ + DRA7XX_CORE_IOPAD(0x36D8, MUX_MODE14) /* A11: P9.29a: mcasp1_axr9.off */ + DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE14) /* D14: P9.29b: mcasp1_fsx.off */ + DRA7XX_CORE_IOPAD(0x36DC, MUX_MODE14) /* B13: P9.30: mcasp1_axr10.off */ + DRA7XX_CORE_IOPAD(0x36D4, MUX_MODE14) /* B12: P9.31a: mcasp1_axr8.off */ + DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE14) /* C14: P9.31b: mcasp1_aclkx.off */ + DRA7XX_CORE_IOPAD(0x36A0, MUX_MODE14) /* C23: P9.41a: xref_clk3.off */ + DRA7XX_CORE_IOPAD(0x3580, MUX_MODE14) /* C1: P9.41b: vin2a_d6.off */ + DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE14) /* E14: P9.42a: mcasp1_axr12.off */ + DRA7XX_CORE_IOPAD(0x359C, MUX_MODE14) /* C2: P9.42b: vin2a_d13.off */ + >; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + + tps659038: tps659038@58 { + compatible = "ti,tps659038"; + reg = <0x58>; + interrupt-parent = <&gpio6>; + interrupts = <16 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default"; + pinctrl-0 = <&pmic_pins_default>; + + #interrupt-cells = <2>; + interrupt-controller; + + ti,system-power-controller; + ti,palmas-override-powerhold; + + tps659038_pmic { + compatible = "ti,tps659038-pmic"; + + smps12-in-supply = <&vdd_5v>; + smps3-in-supply = <&vdd_5v>; + smps45-in-supply = <&vdd_5v>; + smps6-in-supply = <&vdd_5v>; + smps7-in-supply = <&vdd_5v>; + mps3-in-supply = <&vdd_5v>; + smps8-in-supply = <&vdd_5v>; + smps9-in-supply = <&vdd_5v>; + ldo1-in-supply = <&vdd_5v>; + ldo2-in-supply = <&vdd_5v>; + ldo3-in-supply = <&vdd_5v>; + ldo4-in-supply = <&vdd_5v>; + ldo9-in-supply = <&vdd_5v>; + ldoln-in-supply = <&vdd_5v>; + ldousb-in-supply = <&vdd_5v>; + ldortc-in-supply = <&vdd_5v>; + + regulators { + vdd_mpu: smps12 { + /* VDD_MPU */ + regulator-name = "smps12"; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <1250000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_ddr: smps3 { + /* VDD_DDR EMIF1 EMIF2 */ + regulator-name = "smps3"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_dspeve: smps45 { + /* VDD_DSPEVE on AM572 */ + regulator-name = "smps45"; + regulator-min-microvolt = < 850000>; + regulator-max-microvolt = <1250000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_gpu: smps6 { + /* VDD_GPU */ + regulator-name = "smps6"; + regulator-min-microvolt = < 850000>; + regulator-max-microvolt = <1250000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_core: smps7 { + /* VDD_CORE */ + regulator-name = "smps7"; + regulator-min-microvolt = < 850000>; /*** 1.15V */ + regulator-max-microvolt = <1150000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_iva: smps8 { + /* VDD_IVAHD */ /*** 1.06V */ + regulator-name = "smps8"; + }; + + vdd_3v3: smps9 { + /* VDD_3V3 */ + regulator-name = "smps9"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_sd: ldo1 { + /* VDDSHV8 - VSDMMC */ + regulator-name = "ldo1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + vdd_1v8: ldo2 { + /* VDDSH18V */ + regulator-name = "ldo2"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_1v8_phy_ldo3: ldo3 { + /* R1.3a 572x V1_8PHY_LDO3: USB, SATA */ + regulator-name = "ldo3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_1v8_phy_ldo4: ldo4 { + /* R1.3a 572x V1_8PHY_LDO4: PCIE, HDMI*/ + regulator-name = "ldo4"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + /* LDO5-8 unused */ + + vdd_rtc: ldo9 { + /* VDD_RTC */ + regulator-name = "ldo9"; + regulator-min-microvolt = < 840000>; + regulator-max-microvolt = <1160000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_1v8_pll: ldoln { + /* VDDA_1V8_PLL */ + regulator-name = "ldoln"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + ldousb_reg: ldousb { + /* VDDA_3V_USB: VDDA_USBHS33 */ + regulator-name = "ldousb"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + ldortc_reg: ldortc { + /* VDDA_RTC */ + regulator-name = "ldortc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + regen1: regen1 { + /* VDD_3V3_ON */ + regulator-name = "regen1"; + regulator-boot-on; + regulator-always-on; + }; + + regen2: regen2 { + /* Needed for PMIC internal resource */ + regulator-name = "regen2"; + regulator-boot-on; + regulator-always-on; + }; + }; + }; + + tps659038_rtc: tps659038_rtc { + compatible = "ti,palmas-rtc"; + interrupt-parent = <&tps659038>; + interrupts = <8 IRQ_TYPE_EDGE_FALLING>; + wakeup-source; + }; + + tps659038_pwr_button: tps659038_pwr_button { + compatible = "ti,palmas-pwrbutton"; + interrupt-parent = <&tps659038>; + interrupts = <1 IRQ_TYPE_EDGE_FALLING>; + wakeup-source; + ti,palmas-long-press-seconds = <12>; + }; + + tps659038_gpio: tps659038_gpio { + compatible = "ti,palmas-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + }; + + /* STMPE811 touch screen controller */ + stmpe811@41 { + compatible = "st,stmpe811"; + reg = <0x41>; + interrupts = <30 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&gpio2>; + interrupt-controller; + id = <0>; + blocks = <0x5>; + irq-trigger = <0x1>; + st,mod-12b = <1>; /* 12-bit ADC */ + st,ref-sel = <0>; /* internal ADC reference */ + st,adc-freq = <1>; /* 3.25 MHz ADC clock speed */ + st,sample-time = <4>; /* ADC converstion time: 80 clocks */ + + pinctrl-names = "default"; + pinctrl-0 = <&adc_pins_default>; + + stmpe_adc { + compatible = "st,stmpe-adc"; + st,norequest-mask = <0x00>; /* mask any channels to be used by touchscreen */ + adc0: iio-device@0 { + #io-channel-cells = <1>; + iio-channels = <&adc0 4>, <&adc0 1>, <&adc0 2>, <&adc0 3>, <&adc0 4>, <&adc0 5>, <&adc0 6>; + iio-channel-names = "AIN0_P9_39", "AIN1_P9_40", "AIN2_P9_37", "AIN3_P9_38", + "AIN4_P9_33", "AIN5_P9_36", "AIN6_P9_35"; + }; + }; + + stmpe_touchscreen { + status = "disabled"; + compatible = "st,stmpe-ts"; + }; + + stmpe_gpio { + compatible = "st,stmpe-gpio"; + }; + + stmpe_pwm { + compatible = "st,stmpe-pwm"; + #pwm-cells = <2>; + }; + }; +}; + +&cpu0 { + vdd-supply = <&vdd_mpu>; + voltage-tolerance = <1>; +}; + +&uart1 { + status = "okay"; +}; + +&davinci_mdio { + reset-gpios = <&gpio2 23 GPIO_ACTIVE_LOW>; + reset-delay-us = <2>; + + phy0: ethernet-phy@1 { + reg = <4>; + compatible = "ethernet-phy-id004d.d072", + "ethernet-phy-ieee802.3-c22"; + eee-broken-100tx; + eee-broken-1000t; + //interrupt-parent = <&gpio5>; + //interrupts = <19 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&mac { + slaves = <1>; + status = "okay"; + //dual_emac; +}; + +&cpsw_emac0 { + phy-handle = <&phy0>; + phy-mode = "rgmii"; + //dual_emac_res_vlan = <1>; +}; + +&mmc1 { + status = "okay"; + vmmc-supply = <&vdd_3v3>; + vqmmc-supply = <&vdd_sd>; + bus-width = <4>; + cd-gpios = <&gpio6 27 GPIO_ACTIVE_LOW>; /* gpio 219 */ + + ti,needs-special-reset; + ti,dual-volt; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-ddr50; + sd-uhs-sdr104; + mmc-hs200-1_8v; + cap-sd-highspeed; + cap-mmc-highspeed; + dmas = <&sdma_xbar 61>, <&sdma_xbar 62>; + dma-names = "tx", "rx"; + + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104"; + pinctrl-0 = <&mmc1_pins_default µsd_extra_pins_default>; + pinctrl-1 = <&mmc1_pins_hs µsd_extra_pins_default>; + pinctrl-2 = <&mmc1_pins_sdr12 µsd_extra_pins_default>; + pinctrl-3 = <&mmc1_pins_sdr25 µsd_extra_pins_default>; + pinctrl-4 = <&mmc1_pins_sdr50 µsd_extra_pins_default>; + pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_rev20_conf µsd_extra_pins_default>; + pinctrl-6 = <&mmc1_pins_sdr104 &mmc1_iodelay_sdr104_rev20_conf µsd_extra_pins_default>; +}; + +&mmc2 { + status = "okay"; + vmmc-supply = <&vdd_1v8>; + vqmmc-supply = <&vdd_1v8>; + bus-width = <8>; + ti,non-removable; + non-removable; + mmc-pwrseq = <&emmc_pwrseq>; + + ti,needs-special-reset; + dmas = <&sdma_xbar 47>, <&sdma_xbar 48>; + dma-names = "tx", "rx"; + + pinctrl-names = "default", "hs", "ddr_1_8v", "hs200_1_8v"; + pinctrl-0 = <&mmc2_pins_default>; + pinctrl-1 = <&mmc2_pins_hs>; + pinctrl-2 = <&mmc2_pins_ddr_rev20>; + pinctrl-3 = <&mmc2_pins_hs200>; +}; + +&mmc4 { + /* DS: Default speed (DS) up to 25 MHz, including 1- and 4-bit modes (3.3 V signaling). */ + /* HS: High speed up to 50 MHz (3.3 V signaling). */ + /* SDR12: SDR up to 25 MHz (1.8 V signaling). */ + /* SDR25: SDR up to 50 MHz (1.8 V signaling). */ + /* SDR50: SDR up to 100 MHz (1.8 V signaling). */ + /* SDR104: SDR up to 208 MHz (1.8 V signaling) */ + /* DDR50: DDR up to 50 MHz (1.8 V signaling). */ + status = "okay"; + + pinctrl-names = "default", "hs"; + pinctrl-0 = <&mmc4_pins_default &wifibt_extra_pins_default>; + pinctrl-1 = <&mmc4_pins_hs &wifibt_extra_pins_default>; + + ti,needs-special-reset; + vmmc-supply = <&vdd_3v3>; + cap-power-off-card; + keep-power-in-suspend; + bus-width = <4>; + ti,non-removable; + non-removable; + no-1-8-v; + max-frequency = <24000000>; + + #address-cells = <1>; + #size-cells = <0>; + mmc-pwrseq = <&brcmf_pwrseq>; + + brcmf: wifi@1 { + status = "okay"; + reg = <1>; + compatible = "brcm,bcm4329-fmac"; + + /* + * Not Mainline: brcm,sd_head_align/brcm,sd_sgentry_align + * From: Cypress Linux WiFi Driver Release (FMAC) [2019-05-08] + * https://community.cypress.com/docs/DOC-17441 + */ + brcm,sd_head_align = <4>; + brcm,sd_sgentry_align = <512>; + + interrupt-parent = <&gpio3>; + interrupts = <23 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "host-wake"; + }; +}; + +&usb2_phy1 { + phy-supply = <&ldousb_reg>; +}; + +&usb2_phy2 { + phy-supply = <&ldousb_reg>; +}; + +&usb1 { + status = "okay"; + dr_mode = "otg"; +}; + +&omap_dwc3_1 { + extcon = <&extcon_usb1>; +}; + +&usb2 { + status = "okay"; + dr_mode = "host"; +}; + +&gpu { + status = "okay"; +}; + +&bb2d { + status = "okay"; +}; + +&dss { + status = "okay"; + vdda_video-supply = <&vdd_1v8_pll>; +}; + +&hdmi { + status = "okay"; + vdda-supply = <&vdd_1v8_phy_ldo4>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_pins_default>; + + port { + hdmi_out: endpoint { + remote-endpoint = <&hdmi_encoder_in>; + //lanes = <1 0 3 2 5 4 7 6>; + }; + }; +}; + +&bandgap { + status = "okay"; +}; + +&mailbox1 { + status = "okay"; +}; + +&mailbox2 { + status = "okay"; +}; + +&mailbox3 { + status = "okay"; +}; + +&mailbox4 { + status = "okay"; +}; + +&mailbox7 { + status = "okay"; +}; + +&mailbox8 { + status = "okay"; +}; + +&mailbox9 { + status = "okay"; +}; + +&mailbox10 { + status = "okay"; +}; + +&mailbox11 { + status = "okay"; +}; + +&mailbox12 { + status = "okay"; +}; + +&mailbox13 { + status = "okay"; +}; + +&ipu1 { + status = "okay"; + memory-region = <&ipu1_memory_region>; +}; + +&ipu2 { + status = "okay"; + memory-region = <&ipu2_memory_region>; +}; + +&dsp1 { + status = "okay"; + memory-region = <&dsp1_memory_region>; +}; + +&dsp2 { + status = "okay"; + memory-region = <&dsp2_memory_region>; +}; + +&pruss_soc_bus1 { + status = "okay"; +}; + +&pruss1 { + status = "okay"; +}; + +&pruss_soc_bus2 { + status = "okay"; +}; + +&pruss2 { + status = "okay"; +}; + +&cpu_alert0 { + temperature = <55000>; /* milliCelsius */ +}; + +&cpu_crit { + temperature = <85000>; /* milliCelsius */ +}; + +&gpu_crit { + temperature = <85000>; /* milliCelsius */ +}; + +&core_crit { + temperature = <85000>; /* milliCelsius */ +}; + +&dspeve_crit { + temperature = <85000>; /* milliCelsius */ +}; + +&iva_crit { + temperature = <85000>; /* milliCelsius */ +}; + +&sata { + status = "disabled"; +}; + +&sata_phy { + status = "disabled"; +}; + +/* bluetooth */ +&uart6 { + status = "okay"; +}; + +/* cape header stuff */ +&i2c4 { + status = "okay"; + clock-frequency = <400000>; +}; + +/* thermal hacks */ +&cpu0_opp_table { + opp_slow-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <860000 850000 1100000>, + <860000 850000 1100000>; + opp-supported-hw = <0xFF 0x01>; + opp-suspend; + }; +}; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi index ff74e319de01a..d2ee755f1a485 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi +++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi @@ -596,6 +596,10 @@ vdda_video-supply = <&ldoln_reg>; }; +//&bb2d { +// status = "okay"; +//}; + &bb2d { status = "okay"; }; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-gssi.dts b/arch/arm/boot/dts/am57xx-beagle-x15-gssi.dts new file mode 100644 index 0000000000000..1d0d0dd6d9577 --- /dev/null +++ b/arch/arm/boot/dts/am57xx-beagle-x15-gssi.dts @@ -0,0 +1,38 @@ +#include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" +#include "gssi-27-009-gpio.dtsi" +#include "gssi-27-009-rts-cyclone.dtsi" +#include "gssi-27-009-rts-hv.dtsi" +#include "gssi-27-009-rts-pru.dtsi" +#include "gssi-27-009-serial.dtsi" +#include "gssi-27-009-wl1837.dtsi" + +/ { + model = "TI AM5728 BeagleBoard-X15 rev C on GSSI mainboard"; +}; + +&tpd12s015 { + gpios = <&gpio7 10 GPIO_ACTIVE_HIGH>, /* gpio7_10, CT CP HPD */ + <&gpio2 30 GPIO_ACTIVE_HIGH>, /* gpio2_30, LS OE */ + <&gpio7 12 GPIO_ACTIVE_HIGH>; /* gpio7_12/sp1_cs2, HPD */ +}; + +&mmc1 { + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104"; + pinctrl-0 = <&mmc1_pins_default>; + pinctrl-1 = <&mmc1_pins_hs>; + pinctrl-2 = <&mmc1_pins_sdr12>; + pinctrl-3 = <&mmc1_pins_sdr25>; + pinctrl-4 = <&mmc1_pins_sdr50>; + pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_rev20_conf>; + pinctrl-6 = <&mmc1_pins_sdr104 &mmc1_iodelay_sdr104_rev20_conf>; + vmmc-supply = <&vdd_3v3>; + vqmmc-supply = <&ldo1_reg>; +}; + +&mmc2 { + pinctrl-names = "default", "hs", "ddr_1_8v"; + pinctrl-0 = <&mmc2_pins_default>; + pinctrl-1 = <&mmc2_pins_hs>; + pinctrl-2 = <&mmc2_pins_ddr_rev20>; +}; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts index 5a77b334923d0..318c2c6724c53 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts @@ -1,15 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { model = "TI AM5728 BeagleBoard-X15 rev B1"; + + chosen { + base_dtb = "am57xx-beagle-x15-revb1.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts index 17c41da3b55f1..ace38636db86d 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts @@ -1,15 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { model = "TI AM5728 BeagleBoard-X15 rev C"; + + chosen { + base_dtb = "am57xx-beagle-x15-revc.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts index c393de8a23912..01b635ca07e80 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts @@ -1,16 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { /* NOTE: This describes the "original" pre-production A2 revision */ model = "TI AM5728 BeagleBoard-X15"; + + chosen { + base_dtb = "am57xx-beagle-x15.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-cmem.dtsi b/arch/arm/boot/dts/am57xx-cmem.dtsi new file mode 100644 index 0000000000000..09f32e4310c0f --- /dev/null +++ b/arch/arm/boot/dts/am57xx-cmem.dtsi @@ -0,0 +1,42 @@ +/* https://git.yoctoproject.org/cgit/cgit.cgi/meta-ti/tree/recipes-kernel/linux/files/dra7xx/cmem.dtsi?h=rocko */ + +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + cmem_block_mem_0: cmem_block_mem@a0000000 { + reg = <0x0 0xa0000000 0x0 0x18000000>; + no-map; + status = "okay"; + }; + + cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 { + reg = <0x0 0x40500000 0x0 0x100000>; + no-map; + status = "okay"; + }; + }; + + cmem { + compatible = "ti,cmem"; + #address-cells = <1>; + #size-cells = <0>; + + #pool-size-cells = <2>; + + status = "okay"; + + cmem_block_0: cmem_block@0 { + reg = <0>; + memory-region = <&cmem_block_mem_0>; + cmem-buf-pools = <1 0x0 0x18000000>; + }; + + cmem_block_1: cmem_block@1 { + reg = <1>; + memory-region = <&cmem_block_mem_1_ocmc3>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am57xx-evm-common.dtsi b/arch/arm/boot/dts/am57xx-evm-common.dtsi new file mode 100644 index 0000000000000..32162a9dc9994 --- /dev/null +++ b/arch/arm/boot/dts/am57xx-evm-common.dtsi @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +/ { + compatible = "ti,am5728-evm", "ti,am572x-beagle-x15", "ti,am5728", "ti,dra742", "ti,dra74", "ti,dra7"; + + gpio_keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + USER1 { + label = "Up"; + linux,code = ; + gpios = <&gpio2 23 GPIO_ACTIVE_LOW>; + }; + + USER2 { + label = "Down"; + linux,code = ; + gpios = <&gpio2 25 GPIO_ACTIVE_LOW>; + }; + + USER3 { + label = "Left"; + linux,code = ; + gpios = <&gpio2 28 GPIO_ACTIVE_LOW>; + }; + + USER4 { + label = "Right"; + linux,code = ; + gpios = <&gpio2 24 GPIO_ACTIVE_LOW>; + }; + + USER5 { + label = "Home"; + linux,code = ; + gpios = <&gpio2 20 GPIO_ACTIVE_LOW>; + }; + }; + + aliases { + display0 = &lcd0; + display1 = &hdmi0; + }; + + lcd0: display { + compatible = "osddisplays,osd070t1718-19ts", "panel-dpi"; + + label = "lcd"; + + backlight = <&lcd_bl>; + + enable-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>; + + panel-timing { + clock-frequency = <33000000>; + hactive = <800>; + vactive = <480>; + hfront-porch = <210>; + hback-porch = <16>; + hsync-len = <30>; + vback-porch = <10>; + vfront-porch = <22>; + vsync-len = <13>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <1>; + pixelclk-active = <1>; + }; + + port { + lcd_in: endpoint { + remote-endpoint = <&dpi_out>; + }; + }; + }; + + lcd_bl: backlight { + compatible = "pwm-backlight"; + pwms = <&ehrpwm1 0 50000 0>; + + /* Anything lower than 241 is no longer visible */ + brightness-levels = <0 243 245 247 249 251 252 253 255>; + + default-brightness-level = <8>; + }; + + vmmcwl_fixed: fixedregulator-mmcwl { + compatible = "regulator-fixed"; + regulator-name = "vmmcwl_fixed"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&gpio5 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&dss { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port { + reg = <0>; + + dpi_out: endpoint { + remote-endpoint = <&lcd_in>; + data-lines = <24>; + }; + }; + }; +}; + +&epwmss1 { + status = "okay"; +}; + +&ehrpwm1 { + status = "okay"; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <400000>; + + pixcir_ts@5c { + compatible = "pixcir,pixcir_tangoc"; + reg = <0x5c>; + interrupt-parent = <&gpio2>; + interrupts = <4 0>; + + attb-gpio = <&gpio2 4 GPIO_ACTIVE_HIGH>; + reset-gpio = <&gpio2 6 GPIO_ACTIVE_HIGH>; + + touchscreen-size-x = <1024>; + touchscreen-size-y = <600>; + }; +}; + +&uart8 { + status = "okay"; +}; + +&mmc3 { + status = "okay"; + vmmc-supply = <&vmmcwl_fixed>; + bus-width = <4>; + pinctrl-names = "default"; + cap-power-off-card; + keep-power-in-suspend; + ti,non-removable; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@2 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio5>; + interrupts = <7 IRQ_TYPE_LEVEL_HIGH>; + }; +}; diff --git a/arch/arm/boot/dts/am57xx-evm-reva3.dts b/arch/arm/boot/dts/am57xx-evm-reva3.dts new file mode 100644 index 0000000000000..711acadf96863 --- /dev/null +++ b/arch/arm/boot/dts/am57xx-evm-reva3.dts @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-evm-common.dtsi" +#include "am57xx-industrial-grade.dtsi" +#include "am57xx-cmem.dtsi" + +/ { + model = "TI AM572x EVM Rev A3"; +}; + +&tpd12s015 { + gpios = <&gpio7 10 GPIO_ACTIVE_HIGH>, /* gpio7_10, CT CP HPD */ + <&gpio2 30 GPIO_ACTIVE_HIGH>, /* gpio2_30, LS OE */ + <&gpio7 12 GPIO_ACTIVE_HIGH>; /* gpio7_12/sp1_cs2, HPD */ +}; + +&mmc1 { + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104"; + pinctrl-0 = <&mmc1_pins_default>; + pinctrl-1 = <&mmc1_pins_hs>; + pinctrl-2 = <&mmc1_pins_sdr12>; + pinctrl-3 = <&mmc1_pins_sdr25>; + pinctrl-4 = <&mmc1_pins_sdr50>; + pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_rev20_conf>; + pinctrl-6 = <&mmc1_pins_sdr104 &mmc1_iodelay_sdr104_rev20_conf>; + vmmc-supply = <&vdd_3v3>; + vqmmc-supply = <&ldo1_reg>; +}; + +&mmc2 { + pinctrl-names = "default", "hs", "ddr_1_8v"; + pinctrl-0 = <&mmc2_pins_default>; + pinctrl-1 = <&mmc2_pins_hs>; + pinctrl-2 = <&mmc2_pins_ddr_rev20>; +}; + +&mmc3 { + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50"; + pinctrl-0 = <&mmc3_pins_default>; + pinctrl-1 = <&mmc3_pins_hs>; + pinctrl-2 = <&mmc3_pins_sdr12>; + pinctrl-3 = <&mmc3_pins_sdr25>; + pinctrl-4 = <&mmc3_pins_sdr50 &mmc3_iodelay_manual1_rev20_conf>; +}; diff --git a/arch/arm/boot/dts/am57xx-evm.dts b/arch/arm/boot/dts/am57xx-evm.dts new file mode 100644 index 0000000000000..a8129b6677ba6 --- /dev/null +++ b/arch/arm/boot/dts/am57xx-evm.dts @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "am57xx-beagle-x15.dts" +#include "am57xx-evm-common.dtsi" +#include "am57xx-commercial-grade.dtsi" +#include "am57xx-cmem.dtsi" + +/ { + model = "TI AM5728 EVM"; +}; + +&mmc3 { + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50"; + pinctrl-0 = <&mmc3_pins_default>; + pinctrl-1 = <&mmc3_pins_hs>; + pinctrl-2 = <&mmc3_pins_sdr12>; + pinctrl-3 = <&mmc3_pins_sdr25>; + pinctrl-4 = <&mmc3_pins_sdr50 &mmc3_iodelay_manual1_rev11_conf>; +}; diff --git a/arch/arm/boot/dts/dra7-pru-iodelay.dtsi b/arch/arm/boot/dts/dra7-pru-iodelay.dtsi new file mode 100644 index 0000000000000..8f46d7ea7b9c5 --- /dev/null +++ b/arch/arm/boot/dts/dra7-pru-iodelay.dtsi @@ -0,0 +1,114 @@ +#define IODELAY_IN( index, a_delay, g_delay ) \ + ( 12 * (index) + 0 ) A_DELAY_PS( a_delay ) G_DELAY_PS( g_delay ) +#define IODELAY_OUT( index, a_delay, g_delay ) \ + ( 12 * (index) + 4 ) A_DELAY_PS( a_delay ) G_DELAY_PS( g_delay ) \ + ( 12 * (index) + 8 ) A_DELAY_PS( a_delay ) G_DELAY_PS( g_delay ) +#define IODELAY( index, in_a_delay, in_g_delay, out_a_delay, out_g_delay ) \ + IODELAY_IN( index, in_a_delay, in_g_delay ) \ + IODELAY_OUT( index, out_a_delay, out_g_delay ) + +&dra7_iodelay_core { + pruss1_pru0_iodelay: pruss1_pru0 { + pinctrl-pin-array = < + IODELAY( 202, 0, 600, 0, 1600 ) // P19.52 AH6 core 0 gpio 0 + IODELAY( 203, 0, 0, 0, 2800 ) // P19.45 AH3 core 0 gpio 1 + IODELAY( 204, 0, 900, 0, 0 ) // P19.48 AH5 core 0 gpio 2 + IODELAY( 205, 0, 400, 0, 0 ) // P19.53 AH6 core 0 gpio 3 + IODELAY( 206, 0, 500, 0, 0 ) // P19.14 AH4 core 0 gpio 4 + IODELAY( 207, 0, 0, 0, 0 ) // P19.15 AG4 core 0 gpio 5 + IODELAY( 208, 0, 0, 0, 0 ) // x AG2 core 0 gpio 6 + IODELAY( 186, 0, 0, 0, 600 ) // P19.46 AG3 core 0 gpio 7 + IODELAY( 187, 0, 300, 0, 0 ) // P19.49 AG5 core 0 gpio 8 + IODELAY( 188, 0, 800, 0, 2700 ) // P19.42 AF2 core 0 gpio 9 + IODELAY( 189, 0, 0, 0, 200 ) // x AF6 core 0 gpio 10 + IODELAY( 190, 0, 600, 0, 800 ) // P19.43 AF3 core 0 gpio 11 + IODELAY( 191, 0,1100, 0, 0 ) // x AF4 core 0 gpio 12 + IODELAY( 192, 0, 800, 0, 100 ) // P19.11 AF1 core 0 gpio 13 + IODELAY( 193, 0,1000, 0, 300 ) // x AE3 core 0 gpio 14 + IODELAY( 194, 0,1100, 0, 0 ) // x AE5 core 0 gpio 15 + IODELAY( 195, 0,2800, 0, 400 ) // P19.39 AE1 core 0 gpio 16 + IODELAY( 197, 0, 900, 0, 300 ) // P19.12 AE2 core 0 gpio 17 + IODELAY( 198, 0, 800, 0, 500 ) // x AE6 core 0 gpio 18 + IODELAY( 199, 0,1400, 0, 0 ) // P19.40 AD2 core 0 gpio 19 + IODELAY( 200, 0,1001, 0, 500 ) // x AD3 core 0 gpio 20 + >; + }; + + pruss1_pru1_iodelay: pruss1_pru1 { + pinctrl-pin-array = < + IODELAY( 232, 0, 2100, 0, 3900 ) // P18.42 E2 core 1 gpio 0 + IODELAY( 233, 0, 1000, 0, 5100 ) // P18.13 D2 core 1 gpio 1 + IODELAY( 234, 0, 1700, 0, 0 ) // P18.08 F4 core 1 gpio 2 + IODELAY( 235, 0, 700, 0, 2700 ) // P18.43 C1 core 1 gpio 3 + IODELAY( 236, 0, 1300, 0, 2600 ) // P18.39 E4 core 1 gpio 4 + IODELAY( 237, 0, 1700, 0, 2500 ) // P18.38 F5 core 1 gpio 5 + IODELAY( 238, 0, 1600, 0, 1900 ) // P18.07 E6 core 1 gpio 6 + IODELAY( 216, 0, 1600, 0, 1900 ) // P18.44 D3 core 1 gpio 7 + IODELAY( 217, 0, 1000, 0, 2700 ) // P18.06 F6 core 1 gpio 8 + IODELAY( 218, 0, 1400, 0, 3400 ) // eth1 D5 core 1 gpio 9 + IODELAY( 219, 0, 800, 0, 3200 ) // eth1 C2 core 1 gpio 10 + IODELAY( 220, 0, 0, 0, 3000 ) // eth1 C3 core 1 gpio 11 + IODELAY( 221, 0, 1600, 0, 2900 ) // eth1 C4 core 1 gpio 12 + IODELAY( 222, 0, 1200, 0, 2700 ) // eth1 B2 core 1 gpio 13 + IODELAY( 223, 0, 1500, 0, 3000 ) // eth1 D6 core 1 gpio 14 + IODELAY( 224, 0, 1000, 0, 2200 ) // eth1 C5 core 1 gpio 15 + IODELAY( 225, 0, 1100, 0, 2300 ) // eth1 A3 core 1 gpio 16 + IODELAY( 227, 0, 700, 0, 1800 ) // eth1 B3 core 1 gpio 17 + IODELAY( 228, 0, 1300, 0, 1900 ) // eth1 B4 core 1 gpio 18 + IODELAY( 229, 0, 1400, 0, 1400 ) // eth1 B5 core 1 gpio 19 + IODELAY( 230, 0, 1300, 0, 1900 ) // eth1 A4 core 1 gpio 20 + >; + }; + + pruss2_pru0_iodelay: pruss2_pru0 { + pinctrl-pin-array = < + IODELAY( 261, 0, 226, 0, 1300 ) // P18.26 G11 core 0 gpio 0 + IODELAY( 262, 0, 0, 0, 2500 ) // P18.48 E09 core 0 gpio 1 + IODELAY( 263, 0, 365, 0, 950 ) // P18.49 F09 core 0 gpio 2 + IODELAY( 264, 0, 0, 0, 800 ) // P18.45 F08 core 0 gpio 3 + IODELAY( 265, 0, 218, 0, 600 ) // P18.46 E07 core 0 gpio 4 + IODELAY( 266, 0, 186, 0, 500 ) // P18.47 E08 core 0 gpio 5 + IODELAY( 267, 0, 308, 0, 500 ) // P18.56 D09 core 0 gpio 6 + IODELAY( 245, 0, 0, 0, 0 ) // P18.16 D07 core 0 gpio 7 + IODELAY( 246, 0, 756, 0, 600 ) // P18.18 D08 core 0 gpio 8 + IODELAY( 247, 0, 531, 0, 2300 ) // P18.20 A05 core 0 gpio 9 + IODELAY( 248, 0, 180, 0, 450 ) // P18.15 C06 core 0 gpio 10 + IODELAY( 249, 0, 334, 0, 600 ) // P18.17 C08 core 0 gpio 11 + IODELAY( 250, 0, 1361, 0, 500 ) // P18.14 C07 core 0 gpio 12 + IODELAY( 251, 0, 488, 0, 100 ) // P18.50 B07 core 0 gpio 13 + IODELAY( 252, 0, 321, 0, 300 ) // P18.51 B08 core 0 gpio 14 + IODELAY( 253, 0, 254, 0, 700 ) // P18.21 A07 core 0 gpio 15 + IODELAY( 254, 0, 500, 0, 700 ) // P18.22 A08 core 0 gpio 16 + IODELAY( 256, 0, 716, 0, 900 ) // P18.19 C09 core 0 gpio 17 + IODELAY( 257, 0, 0, 0, 900 ) // P18.23 A09 core 0 gpio 18 + IODELAY( 258, 0, 404, 0, 300 ) // P18.52 B09 core 0 gpio 19 + IODELAY( 259, 0, 290, 0, 300 ) // P18.24 A10 core 0 gpio 20 + >; + }; + + pruss2_pru1_iodelay: pruss2_pru1 { + pinctrl-pin-array = < + IODELAY( 119, 0, 2422, 0, 3200 ) // ethmdio V01 core 1 gpio 0 + IODELAY( 118, 0, 3200, 0, 3900 ) // ethmdio U04 core 1 gpio 1 + IODELAY( 0, 0, 2100, 0, 2500 ) // x U03 core 1 gpio 2 + IODELAY( 180, 0, 1900, 0, 3400 ) // P18.59 V02 core 1 gpio 3 + IODELAY( 181, 0, 1900, 0, 3000 ) // P18.29 Y01 core 1 gpio 4 + IODELAY( 150, 0, 2488, 0, 3300 ) // eth0 W09 core 1 gpio 5 + IODELAY( 151, 0, 2263, 0, 2700 ) // eth0 V09 core 1 gpio 6 + IODELAY( 155, 0, 2400, 0, 3200 ) // eth0 V07 core 1 gpio 7 + IODELAY( 154, 0, 2800, 0, 3000 ) // eth0 U07 core 1 gpio 8 + IODELAY( 153, 0, 2900, 0, 2500 ) // eth0 V06 core 1 gpio 9 + IODELAY( 152, 0, 2292, 0, 2900 ) // eth0 U06 core 1 gpio 10 + IODELAY( 144, 0, 1904, 0, 2600 ) // eth0 U05 core 1 gpio 11 + IODELAY( 145, 0, 3629, 0, 2800 ) // eth0 V05 core 1 gpio 12 + IODELAY( 149, 0, 3363, 0, 2700 ) // eth0 V04 core 1 gpio 13 + IODELAY( 148, 0, 2900, 0, 2600 ) // eth0 V03 core 1 gpio 14 + IODELAY( 147, 0, 3100, 0, 2700 ) // eth0 Y02 core 1 gpio 15 + IODELAY( 146, 0, 2800, 0, 2800 ) // eth0 W02 core 1 gpio 16 + IODELAY( 271, 0, 0, 0, 1200 ) // P18.28 E11 core 1 gpio 17 + IODELAY( 244, 0, 1020, 0, 600 ) // P18.25 F11 core 1 gpio 18 + IODELAY( 255, 0, 976, 0, 0 ) // P18.55 G10 core 1 gpio 19 + IODELAY( 260, 0, 946, 0, 300 ) // P18.54 F10 core 1 gpio 20 + >; + }; +}; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 12731012c72e4..5d60a13fed27f 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -144,7 +144,7 @@ * the moment, just use a fake OCP bus entry to represent the whole bus * hierarchy. */ - ocp { + ocp: ocp { compatible = "ti,dra7-l3-noc", "simple-bus"; #address-cells = <1>; #size-cells = <1>; @@ -2087,7 +2087,7 @@ }; bb2d: bb2d@59000000 { - compatible = "ti,dra7-bb2d"; + compatible = "ti,dra7-bb2d","vivante,gc"; reg = <0x59000000 0x0700>; interrupts = ; ti,hwmods = "bb2d"; @@ -2096,6 +2096,16 @@ status = "disabled"; }; + //bb2d: bb2d@59000000 { + // compatible = "ti,dra7-bb2d","vivante,gc"; + // reg = <0x59000000 0x0700>; + // interrupts = ; + // ti,hwmods = "bb2d"; + // clocks = <&dpll_core_h24x2_ck>; + // clock-names = "fclk"; + // status = "disabled"; + //}; + dss: dss@58000000 { compatible = "ti,dra7-dss"; /* 'reg' defined in dra72x.dtsi and dra74x.dtsi */ @@ -2155,6 +2165,15 @@ status = "disabled"; }; + eqep0: eqep@0x4843e180 { + compatible = "ti,am33xx-eqep"; + reg = <0x4843e180 0x80>; + clocks = <&l4_root_clk_div>; + clock-names = "fck"; + interrupt-parent = <&crossbar_mpu>; + interrupts = ; + }; + ecap0: ecap@4843e100 { compatible = "ti,dra746-ecap", "ti,am3352-ecap"; @@ -2185,6 +2204,15 @@ status = "disabled"; }; + eqep1: eqep@0x48440180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48440180 0x80>; + clocks = <&l4_root_clk_div>; + clock-names = "fck"; + interrupt-parent = <&crossbar_mpu>; + interrupts = ; + }; + ecap1: ecap@48440100 { compatible = "ti,dra746-ecap", "ti,am3352-ecap"; @@ -2215,6 +2243,15 @@ status = "disabled"; }; + eqep2: eqep@0x48442180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48442180 0x80>; + clocks = <&l4_root_clk_div>; + clock-names = "fck"; + interrupt-parent = <&crossbar_mpu>; + interrupts = ; + }; + ecap2: ecap@48442100 { compatible = "ti,dra746-ecap", "ti,am3352-ecap"; @@ -2370,6 +2407,12 @@ }; }; + gpu-subsystem { + compatible = "vivante,gc-gpu-subsystem"; + cores = <&bb2d>; + status = "okay"; + }; + thermal_zones: thermal-zones { #include "omap4-cpu-thermal.dtsi" #include "omap5-gpu-thermal.dtsi" diff --git a/arch/arm/boot/dts/gssi-27-009-gpio.dtsi b/arch/arm/boot/dts/gssi-27-009-gpio.dtsi new file mode 100644 index 0000000000000..072dc755ac8a4 --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-gpio.dtsi @@ -0,0 +1,49 @@ +/ { + gpio_keys { + compatible = "gpio-keys"; + + marker { + label = "GSSI Marker Switch"; + pinctrl-names = "default"; + pinctrl-0 = <&power_switch_pins>; + gpios = <&gpio2 31 GPIO_ACTIVE_HIGH>; + debounce-interval = <5>; + interrupts = <0x1 0x4 0x7>; + }; + }; + + gpio_poweroff { + compatible = "gpio-poweroff"; + gpios = <&gpio1 31 GPIO_ACTIVE_HIGH>; + }; + + sir4k_cable { + compatible = "extcon-gpio"; + gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>; + }; + + power_leds { + compatible = "gpio-leds"; + + pinctrl-names = "default"; + pinctrl-0 = <&power_switch_pins>; + + power_led0 { + label = "gssi:led2_7"; + gpios = <&gpio2 7 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "default-on"; + default-state = "on"; + }; + }; +}; + +&dra7_pmx_core { + power_switch_pins: pinmux_power_switch_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3464, PIN_INPUT_PULLDOWN | MUX_MODE14) /* R04 gpmc_a9 gpio1_31 power sw */ + DRA7XX_CORE_IOPAD(0x3484, PIN_OUTPUT_PULLUP | MUX_MODE14) /* P03 gpmc_a17 gpio2_7 power LED */ + DRA7XX_CORE_IOPAD(0x34B8, PIN_INPUT_PULLDOWN | MUX_MODE14) /* P02 gpmc_cs2 gpio2_20 sir4K */ + DRA7XX_CORE_IOPAD(0x34E0, PIN_INPUT_PULLDOWN | MUX_MODE14) /* AH7 vin1b_clik1 gpio2_32 marker*/ + >; + }; +}; diff --git a/arch/arm/boot/dts/gssi-27-009-rts-cyclone.dtsi b/arch/arm/boot/dts/gssi-27-009-rts-cyclone.dtsi new file mode 100644 index 0000000000000..8b4729ffbc145 --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-rts-cyclone.dtsi @@ -0,0 +1,45 @@ +/ { + fpga_region0: fpga-region0 { + compatible = "fpga-region"; + #address-cells = <2>; + #size-cells = <2>; + fpga-mgr = <&fpga_prog>; + }; +}; + +&dra7_pmx_core { + altera_ps_pins: pinmux_altera_ps_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3470, PIN_INPUT_PULLDOWN | MUX_MODE14) /* P04 gpmc_a12 gpio2_2 */ + DRA7XX_CORE_IOPAD(0x3474, PIN_INPUT_PULLDOWN | MUX_MODE14) /* R03 gpmc_a13 gpio2_3 */ + DRA7XX_CORE_IOPAD(0x347c, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* U02 gpmc_a13 gpio2_3 */ + >; + }; + + mcspi3_pins: pinmux_mcspi3_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36d4, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* B12 mcasp1_axr8 spi3_sclk */ + DRA7XX_CORE_IOPAD(0x36d8, PIN_INPUT | MUX_MODE3 ) /* A11 mcasp1_axr9 spi3_d1 */ + DRA7XX_CORE_IOPAD(0x36dc, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* B13 mcasp1_axr10 spi3_d0 */ + DRA7XX_CORE_IOPAD(0x36e0, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* A12 mcasp1_axr11 spi3_cs0 */ + >; + }; +}; + +&mcspi3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&mcspi3_pins>; + ti,pindir-d0-out-d1-in; + + fpga_prog: fpga@0 { + compatible = "altr,fpga-passive-serial"; + spi-max-frequency = <20000000>; + reg = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&altera_ps_pins>; + nconfig-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>; + nstat-gpios = <&gpio2 3 GPIO_ACTIVE_LOW>; + confd-gpios = <&gpio2 2 GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/arm/boot/dts/gssi-27-009-rts-hv.dtsi b/arch/arm/boot/dts/gssi-27-009-rts-hv.dtsi new file mode 100644 index 0000000000000..4c02658911aa9 --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-rts-hv.dtsi @@ -0,0 +1,44 @@ +/ { + rts_hv_regulator: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "gpr-high-voltage-supply"; + regulator-min-microvolt = <50000000>; + regulator-max-microvolt = <120000000>; + enable-gpio = <&gpio2 20 GPIO_ACTIVE_HIGH>; + + gpios = < + &gpio2 25 GPIO_ACTIVE_HIGH + &gpio2 23 GPIO_ACTIVE_HIGH + &gpio2 28 GPIO_ACTIVE_HIGH + >; + + states = < + 50000000 0x7 + 60000000 0x6 + 70000000 0x5 + 80000000 0x4 + 90000000 0x3 + 100000000 0x2 + 110000000 0x1 + 120000000 0x0>; + startup-delay-us = <100000>; + enable-active-high; + }; + + rts_bistatic_enable@5 { + compatible = "regulator-fixed"; + regulator-name = "rts-bistatic-enable"; + pinctrl-names = "default"; + pinctrl-0 = <&rts_bistatic_pins>; + gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&dra7_pmx_core { + rts_bistatic_pins: pinmux_rts_bistatic_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34B4, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* T01 gpmc_cs0 gpio2_19 */ + >; + }; +}; diff --git a/arch/arm/boot/dts/gssi-27-009-rts-pru.dtsi b/arch/arm/boot/dts/gssi-27-009-rts-pru.dtsi new file mode 100644 index 0000000000000..149a85a79dadb --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-rts-pru.dtsi @@ -0,0 +1,117 @@ +#include "dra7-pru-iodelay.dtsi" + +#define PINCONF( index, config ) \ + DRA7XX_CORE_IOPAD( 0x3400 + 4 * (index), config ) + +#define PIN_PRU_GPI (PIN_INPUT_PULLDOWN | MODE_SELECT | MUX_MODE12) +#define PIN_PRU_GPO (PIN_OUTPUT | MODE_SELECT | MUX_MODE13) + +/ { + rts_lvds_enable@4 { + compatible = "regulator-fixed"; + regulator-name = "rts-lvds-enable"; + regulator-always-on; + pinctrl-names = "default"; + pinctrl-0 = <&rts_lvds_pins>; + gpio = <&gpio4 8 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&dra7_pmx_core { + pruss1_pru0_pins: pruss1 { + pinctrl-single,pins = < + PINCONF( 64, PIN_PRU_GPI ) // P19.52 core 0 gpi 0 + PINCONF( 65, PIN_PRU_GPI ) // P19.45 core 0 gpi 1 + PINCONF( 66, PIN_PRU_GPI ) // P19.48 core 0 gpi 2 + PINCONF( 67, PIN_PRU_GPI ) // P19.53 core 0 gpi 3 + PINCONF( 68, PIN_PRU_GPI ) // P19.14 core 0 gpi 4 + PINCONF( 69, PIN_PRU_GPI ) // P19.15 core 0 gpi 5 + PINCONF( 71, PIN_PRU_GPI ) // P19.46 core 0 gpi 7 + PINCONF( 72, PIN_PRU_GPI ) // P19.49 core 0 gpi 8 + PINCONF( 73, PIN_PRU_GPI ) // P19.42 core 0 gpi 9 + PINCONF( 75, PIN_PRU_GPI ) // P19.43 core 0 gpi 11 + PINCONF( 77, PIN_PRU_GPI ) // P19.11 core 0 gpi 13 + PINCONF( 80, PIN_PRU_GPI ) // P19.39 core 0 gpi 16 + PINCONF( 81, PIN_PRU_GPI ) // P19.12 core 0 gpi 17 + PINCONF( 83, PIN_PRU_GPI ) // P19.40 core 0 gpi 19 + >; + }; + + pruss1_pru1_pins: pruss1 { + pinctrl-single,pins = < + PINCONF( 93, PIN_PRU_GPI ) // P18.42 core 1 gpi 0 + PINCONF( 94, PIN_PRU_GPI ) // P18.13 core 1 gpi 1 + PINCONF( 95, PIN_PRU_GPI ) // P18.08 core 1 gpi 2 + PINCONF( 96, PIN_PRU_GPI ) // P18.43 core 1 gpi 3 + PINCONF( 97, PIN_PRU_GPI ) // P18.39 core 1 gpi 4 + PINCONF( 98, PIN_PRU_GPI ) // P18.38 core 1 gpi 5 + PINCONF( 99, PIN_PRU_GPI ) // P18.07 core 1 gpi 6 + PINCONF(100, PIN_PRU_GPI ) // P18.44 core 1 gpi 7 + PINCONF(101, PIN_PRU_GPI ) // P18.06 core 1 gpi 8 + >; + }; + + pruss2_pru0_adapter_pins: pruss2 { + pinctrl-single,pins = < + PINCONF(122, PIN_PRU_GPI ) // P18.26 core 0 gpi 0 + PINCONF(123, PIN_PRU_GPI ) // P18.48 core 0 gpi 1 + PINCONF(124, PIN_PRU_GPI ) // P18.49 core 0 gpi 2 + PINCONF(125, PIN_PRU_GPI ) // P18.45 core 0 gpi 3 + PINCONF(126, PIN_PRU_GPI ) // P18.46 core 0 gpi 4 + PINCONF(127, PIN_PRU_GPI ) // P18.47 core 0 gpi 5 + PINCONF(128, PIN_PRU_GPI ) // P18.56 core 0 gpi 6 + PINCONF(129, PIN_PRU_GPI ) // P18.16 core 0 gpi 7 + PINCONF(130, PIN_PRU_GPI ) // P18.18 core 0 gpi 8 + + PINCONF(131, PIN_PRU_GPO ) // P18.20 core 0 gpo 9 + PINCONF(132, PIN_PRU_GPO ) // P18.15 core 0 gpo 10 + >; + }; + + pruss2_pru0_mainboard_pins: pruss2 { + pinctrl-single,pins = < + PINCONF(122, PIN_PRU_GPO ) // P18.26 G11 core 0 gpio 0 + PINCONF(123, PIN_PRU_GPO ) // P18.48 E09 core 0 gpio + + PINCONF(129, PIN_PRU_GPI ) // P18.16 D07 core 0 gpio 7 + PINCONF(130, PIN_PRU_GPI ) // P18.18 D08 core 0 gpio 8 + PINCONF(131, PIN_PRU_GPI ) // P18.20 A05 core 0 gpio 9 + PINCONF(132, PIN_PRU_GPI ) // P18.15 C06 core 0 gpio 10 + PINCONF(133, PIN_PRU_GPI ) // P18.17 C08 core 0 gpio 11 + PINCONF(134, PIN_PRU_GPI ) // P18.14 C07 core 0 gpio 12 + PINCONF(135, PIN_PRU_GPI ) // P18.50 B07 core 0 gpio 13 + PINCONF(136, PIN_PRU_GPI ) // P18.51 B08 core 0 gpio 14 + PINCONF(137, PIN_PRU_GPI ) // P18.21 A07 core 0 gpio 15 + PINCONF(138, PIN_PRU_GPI ) // P18.22 A08 core 0 gpio 16 + >; + }; + + rts_lvds_pins: pinmux_rts_lvds_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3584, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* E04 vin2a_d7 gpio4_8 */ + >; + }; + + rts_reset_pins: pinmux_rts_reset_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x355c, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* H07 vin2a_fld0 gpio3_30 */ + >; + }; + + rts_debug_pins: pinmux_rts_debug_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3580, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* C01 vin2a_d6 gpio4_7 */ + >; + }; +}; + +&pruss2 { + compatible = "ti,pruss-v2"; + ti,pintc-offset = <0x20000>; + pruss-instance = "pruss2"; + + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pruss2_pru0_mainboard_pins>, <&pruss2_pru0_iodelay>; +}; diff --git a/arch/arm/boot/dts/gssi-27-009-serial.dtsi b/arch/arm/boot/dts/gssi-27-009-serial.dtsi new file mode 100644 index 0000000000000..7c4fd19a5e947 --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-serial.dtsi @@ -0,0 +1,72 @@ +&dra7_pmx_core { + battery_pins: pinmux_battery_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36b0, PIN_INPUT_PULLUP | MUX_MODE10) /* J14 mcasp1_fsr i2c4_scl */ + DRA7XX_CORE_IOPAD(0x36ac, PIN_INPUT_PULLUP | MUX_MODE10) /* B14 mcasp1_aclkr i2c4_sda */ + >; + }; + + eqep1_pins: pinmux_eqep1_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350c, PIN_INPUT | MUX_MODE10) /* AG6 vin1a_d6 eQEP2A_in */ + DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE10) /* AH4 vin1a_d7 eQEP2B_in */ + >; + }; + + uart8_pins: pinmux_uart8_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3734, PIN_INPUT_PULLUP | MUX_MODE3 ) /* C18 mcasp4_aclkx uart8_rxd */ + DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* A21 mcasp4_fsx uart8_txd */ + >; + }; +}; + +&epwmss1 { + status = "okay"; + ranges = <0x48440000 0x48440000 0x1000>; + + module { + reg = <0x48440000 0x1000>; + compatible = "uio"; + symlink = "uio/pwmss1/module"; + }; +}; + +&eqep1 { + compatible = "uio"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&eqep1_pins>; + symlink = "uio/pwmss1/qep"; + + count_mode = <0>; + swap_inputs = <0>; + invert_qa = <0>; + invert_qb = <0>; + invert_qi = <0>; + invert_qs = <0>; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&battery_pins>; + status = "okay"; + + battery: sbs-battery@b { + compatible = "sbs,sbs-battery"; + reg = <0x0b>; + sbs,poll-retry-count = <100>; + sbs,i2c-retry-count = <100>; + }; + + eeprom_ant: eeprom-ant@50 { + compatible = "atmel,24c32"; + reg = <0x50>; + }; +}; + +&uart8 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart8_pins>; +}; diff --git a/arch/arm/boot/dts/gssi-27-009-wl1837.dtsi b/arch/arm/boot/dts/gssi-27-009-wl1837.dtsi new file mode 100644 index 0000000000000..520cd6abac6ad --- /dev/null +++ b/arch/arm/boot/dts/gssi-27-009-wl1837.dtsi @@ -0,0 +1,80 @@ +/ { + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us = <70000>; + gpio = <&gpio5 4 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&dra7_pmx_core { + wl18xx_pins_default: pinmux_wl18xx_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36bc, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* G13 mcasp1_axr2 gpio5_4 */ + DRA7XX_CORE_IOPAD(0x36c4, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* E12 mcasp1_axr4 gpio5_6 */ + DRA7XX_CORE_IOPAD(0x36d0, PIN_INPUT_PULLDOWN | MUX_MODE14) /* D12 mcasp1_axr7 gpio5_9 */ + >; + }; + + wl18xx_pins_sleep: pinmux_wl18xx_pins_sleep { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36bc, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* G13 mcasp1_axr2 gpio5_4 */ + DRA7XX_CORE_IOPAD(0x36c4, PIN_OUTPUT_PULLUP | MUX_MODE14) /* E12 mcasp1_axr4 gpio5_6 */ + DRA7XX_CORE_IOPAD(0x36d0, PIN_INPUT_PULLDOWN | MUX_MODE14) /* D12 mcasp1_axr7 gpio5_9 */ + >; + }; + + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36c0, PIN_OUTPUT_PULLUP | MUX_MODE14) /* J11 mcasp1_axr3 gpio5_5 */ + >; + }; + + uart9_pins: pinmux_uart9_pins { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3744, PIN_INPUT_PULLUP | MUX_MODE3 ) /* AA3 mcasp5_aclkx uart9_rxd */ + DRA7XX_CORE_IOPAD(0x3748, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* AB9 mcasp5_fsx uart9_txd */ + DRA7XX_CORE_IOPAD(0x374c, PIN_INPUT | MUX_MODE3 ) /* AB3 mcasp5_axr0 uart9_ctsn */ + DRA7XX_CORE_IOPAD(0x3750, PIN_OUTPUT_PULLDOWN | MUX_MODE3 ) /* AA4 mcasp5_axr1 uart9_rtsn */ + >; + }; +}; + +&mmc3 { + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50"; + pinctrl-0 = <&mmc3_pins_default &wl18xx_pins_default>; + pinctrl-1 = <&mmc3_pins_hs &wl18xx_pins_default>; + pinctrl-2 = <&mmc3_pins_sdr12 &wl18xx_pins_default>; + pinctrl-3 = <&mmc3_pins_sdr25 &wl18xx_pins_default>; + pinctrl-4 = <&mmc3_pins_sdr50 &wl18xx_pins_default>; + ti,non-removable; + ti,needs-special-hs-handling; + cap-power-off-card; + keep-power-in-suspend; + + #address-cells = <1>; + #size-cells = <0>; + wlcore@0 { + compatible = "ti,wl1837"; + reg = <2>; + interrupt-parent = <&gpio5>; + interrupts = <9 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart9 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart9_pins &bt_pins>; + + bluetooth { + compatible = "ti,wl1837-st"; + enable-gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>; + }; +}; diff --git a/arch/arm/configs/bb.org_defconfig b/arch/arm/configs/bb.org_defconfig new file mode 100644 index 0000000000000..c408bb038af74 --- /dev/null +++ b/arch/arm/configs/bb.org_defconfig @@ -0,0 +1,2508 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZO=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ_IDLE=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_MEMCG=y +CONFIG_MEMCG_SWAP=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_SYSCTL_SYSCALL=y +CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y +CONFIG_USERFAULTFD=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB_FREELIST_RANDOM=y +CONFIG_PROFILING=y +CONFIG_OPROFILE=m +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_CC_STACKPROTECTOR_STRONG=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_COMPRESS=y +CONFIG_MODULE_COMPRESS_XZ=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_WBT=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_MQ_IOSCHED_KYBER=m +CONFIG_IOSCHED_BFQ=m +CONFIG_BFQ_GROUP_IOSCHED=y +CONFIG_OMAP_RESET_CLOCKS=y +CONFIG_SOC_OMAP5=y +CONFIG_SOC_AM33XX=y +CONFIG_SOC_AM43XX=y +CONFIG_SOC_DRA7XX=y +CONFIG_SOC_HAS_OMAP2_SDRC=y +CONFIG_ARM_THUMBEE=y +CONFIG_CACHE_L2X0_PMU=y +CONFIG_PL310_ERRATA_588369=y +CONFIG_PL310_ERRATA_727915=y +CONFIG_SMP=y +CONFIG_NR_CPUS=2 +CONFIG_ARM_PSCI=y +CONFIG_PREEMPT=y +CONFIG_HZ_250=y +CONFIG_ARM_MODULE_PLTS=y +CONFIG_KSM=y +CONFIG_FRONTSWAP=y +CONFIG_ZSWAP=y +CONFIG_ZBUD=y +CONFIG_ZSMALLOC=m +CONFIG_SECCOMP=y +# CONFIG_ATAGS is not set +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_EFI=y +# CONFIG_DMI is not set +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPUFREQ_DT=y +# CONFIG_ARM_OMAP2PLUS_CPUFREQ is not set +CONFIG_ARM_TI_CPUFREQ=y +CONFIG_BINFMT_MISC=m +CONFIG_HIBERNATION=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y +CONFIG_APM_EMULATION=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=m +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=m +CONFIG_XFRM_USER=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_WIREGUARD=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_MROUTE_MULTIPLE_TABLES=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_NET_IPVTI=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_XFRM_MODE_TRANSPORT=m +CONFIG_INET_XFRM_MODE_TUNNEL=m +CONFIG_INET_XFRM_MODE_BEET=m +CONFIG_INET_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_ILA=m +CONFIG_INET6_XFRM_MODE_TRANSPORT=m +CONFIG_INET6_XFRM_MODE_TUNNEL=m +CONFIG_INET6_XFRM_MODE_BEET=m +CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=m +CONFIG_NF_TABLES_NETDEV=m +CONFIG_NFT_EXTHDR=m +CONFIG_NFT_META=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=y +CONFIG_NFT_SET_RBTREE=m +CONFIG_NFT_SET_HASH=m +CONFIG_NFT_COUNTER=y +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=y +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=y +CONFIG_NFT_HASH=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_LC=m +CONFIG_IP_VS_WLC=m +CONFIG_IP_VS_FO=m +CONFIG_IP_VS_OVF=m +CONFIG_IP_VS_LBLC=m +CONFIG_IP_VS_LBLCR=m +CONFIG_IP_VS_DH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_SED=m +CONFIG_IP_VS_NQ=m +CONFIG_IP_VS_FTP=m +CONFIG_IP_VS_PE_SIP=m +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_NF_SOCKET_IPV4=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_CHAIN_ROUTE_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=m +CONFIG_NF_LOG_ARP=m +CONFIG_NFT_CHAIN_NAT_IPV4=y +CONFIG_NFT_MASQ_IPV4=m +CONFIG_NFT_REDIR_IPV4=m +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_CONNTRACK_IPV6=m +CONFIG_NF_SOCKET_IPV6=m +CONFIG_NFT_CHAIN_ROUTE_IPV6=m +CONFIG_NFT_CHAIN_NAT_IPV6=m +CONFIG_NFT_MASQ_IPV6=m +CONFIG_NFT_REDIR_IPV6=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_IP_DCCP=m +CONFIG_NET_DCCPPROBE=m +CONFIG_NET_SCTPPROBE=m +CONFIG_SCTP_COOKIE_HMAC_SHA1=y +CONFIG_RDS=m +CONFIG_RDS_TCP=m +CONFIG_TIPC=m +CONFIG_ATM=m +CONFIG_ATM_CLIP=m +CONFIG_ATM_LANE=m +CONFIG_ATM_MPOA=m +CONFIG_ATM_BR2684=m +CONFIG_L2TP=m +CONFIG_L2TP_DEBUGFS=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=y +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m +CONFIG_ATALK=m +CONFIG_DEV_APPLETALK=m +CONFIG_IPDDP=m +CONFIG_IPDDP_ENCAP=y +CONFIG_PHONET=m +CONFIG_6LOWPAN=m +CONFIG_6LOWPAN_GHC_EXT_HDR_HOP=m +CONFIG_6LOWPAN_GHC_UDP=m +CONFIG_6LOWPAN_GHC_ICMPV6=m +CONFIG_6LOWPAN_GHC_EXT_HDR_DEST=m +CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m +CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m +CONFIG_IEEE802154=m +CONFIG_IEEE802154_6LOWPAN=m +CONFIG_MAC802154=m +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_ATM=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_CANID=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_NET_CLS_IND=y +CONFIG_DCB=y +CONFIG_BATMAN_ADV=m +CONFIG_BATMAN_ADV_DAT=y +CONFIG_BATMAN_ADV_NC=y +CONFIG_BATMAN_ADV_MCAST=y +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_MPLS_GSO=y +CONFIG_MPLS_ROUTING=m +CONFIG_MPLS_IPTUNNEL=m +CONFIG_NET_SWITCHDEV=y +CONFIG_NET_L3_MASTER_DEV=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +CONFIG_NET_PKTGEN=m +CONFIG_NET_DROP_MONITOR=m +CONFIG_HAMRADIO=y +CONFIG_AX25=m +CONFIG_NETROM=m +CONFIG_ROSE=m +CONFIG_MKISS=m +CONFIG_6PACK=m +CONFIG_BPQETHER=m +CONFIG_BAYCOM_SER_FDX=m +CONFIG_BAYCOM_SER_HDX=m +CONFIG_YAM=m +CONFIG_CAN=y +CONFIG_CAN_RAW=m +CONFIG_CAN_BCM=m +CONFIG_CAN_GW=m +CONFIG_CAN_ISOTP=m +CONFIG_CAN_VCAN=m +CONFIG_CAN_SLCAN=m +CONFIG_CAN_C_CAN=y +CONFIG_CAN_C_CAN_PLATFORM=y +CONFIG_CAN_HI311X=m +CONFIG_CAN_MCP251X=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB2=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CAN_8DEV_USB=m +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_6LOWPAN=m +CONFIG_BT_LEDS=y +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTSDIO=m +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_ATH3K=y +CONFIG_BT_HCIUART_LL=y +CONFIG_BT_HCIUART_3WIRE=y +CONFIG_BT_HCIUART_QCA=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_MRVL_SDIO=m +CONFIG_BT_ATH3K=m +CONFIG_BT_WILINK=m +CONFIG_AF_RXRPC_IPV6=y +CONFIG_RXKAD=y +CONFIG_CFG80211=m +# CONFIG_CFG80211_DEFAULT_PS is not set +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_MAC80211_RC_MINSTREL_VHT=y +CONFIG_MAC80211_MESH=y +CONFIG_WIMAX=m +CONFIG_RFKILL=y +CONFIG_RFKILL_INPUT=y +CONFIG_NET_9P=m +CONFIG_NET_9P_VIRTIO=m +CONFIG_RPMSG_PROTO=m +CONFIG_NFC=m +CONFIG_NFC_DIGITAL=m +CONFIG_NFC_NCI=m +CONFIG_NFC_NCI_SPI=m +CONFIG_NFC_HCI=m +CONFIG_NFC_SHDLC=y +CONFIG_NFC_TRF7970A=m +CONFIG_NFC_SIM=m +CONFIG_NFC_PORT100=m +CONFIG_NFC_PN544_I2C=m +CONFIG_NFC_PN533_USB=m +CONFIG_NFC_PN533_I2C=m +CONFIG_NFC_MICROREAD_I2C=m +CONFIG_NFC_ST21NFCA_I2C=m +CONFIG_NFC_ST_NCI_I2C=m +CONFIG_NFC_ST_NCI_SPI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m +CONFIG_NFC_ST95HF=m +CONFIG_NET_DEVLINK=m +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_EXTRA_FIRMWARE="am335x-pm-firmware.elf am335x-bone-scale-data.bin am335x-evm-scale-data.bin am43x-evm-scale-data.bin" +CONFIG_EXTRA_FIRMWARE_DIR="firmware" +CONFIG_CMA_SIZE_MBYTES=48 +CONFIG_OMAP_OCP2SCP=y +CONFIG_SIMPLE_PM_BUS=y +CONFIG_CONNECTOR=y +CONFIG_MTD=y +CONFIG_MTD_OF_PARTS=m +CONFIG_MTD_AR7_PARTS=m +CONFIG_MTD_BLOCK=m +CONFIG_MTD_BLOCK_RO=m +CONFIG_RFD_FTL=m +CONFIG_SSFDC=m +CONFIG_MTD_OOPS=m +CONFIG_MTD_SWAP=m +CONFIG_MTD_PHYSMAP=m +CONFIG_MTD_PLATRAM=m +CONFIG_MTD_DATAFLASH=m +CONFIG_MTD_M25P80=y +CONFIG_MTD_SST25L=m +CONFIG_MTD_NAND=y +CONFIG_MTD_NAND_ECC_BCH=y +CONFIG_MTD_NAND_OMAP2=m +CONFIG_MTD_NAND_NANDSIM=m +CONFIG_MTD_ONENAND=y +CONFIG_MTD_ONENAND_VERIFY_WRITE=y +CONFIG_MTD_ONENAND_2X_PROGRAM=y +CONFIG_MTD_LPDDR=m +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_BLOCK=y +CONFIG_OF_CONFIGFS=y +CONFIG_BLK_DEV_NULL_BLK=m +CONFIG_ZRAM=m +CONFIG_BLK_DEV_LOOP=m +CONFIG_BLK_DEV_DRBD=m +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=m +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_ATA_OVER_ETH=m +CONFIG_VIRTIO_BLK=m +CONFIG_BLK_DEV_RBD=m +CONFIG_AD525X_DPOT=m +CONFIG_AD525X_DPOT_I2C=m +CONFIG_AD525X_DPOT_SPI=m +CONFIG_ICS932S401=m +CONFIG_APDS9802ALS=m +CONFIG_ISL29003=m +CONFIG_ISL29020=m +CONFIG_SENSORS_TSL2550=m +CONFIG_SENSORS_BH1770=m +CONFIG_SENSORS_APDS990X=m +CONFIG_HMC6352=m +CONFIG_DS1682=m +CONFIG_TI_DAC7512=m +CONFIG_SRAM=y +CONFIG_BONE_CAPEMGR=y +CONFIG_TIEQEP=y +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_EEPROM_LEGACY=m +CONFIG_EEPROM_MAX6875=m +CONFIG_EEPROM_93CX6=y +CONFIG_EEPROM_93XX46=m +CONFIG_TI_ST=m +CONFIG_BEAGLEBONE_PINMUX_HELPER=y +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=m +CONFIG_CHR_DEV_OSST=m +CONFIG_BLK_DEV_SR=m +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=m +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=m +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=m +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=m +CONFIG_SCSI_UFSHCD=m +CONFIG_LIBFC=m +CONFIG_LIBFCOE=m +CONFIG_SCSI_VIRTIO=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=m +CONFIG_SCSI_DH_HP_SW=m +CONFIG_SCSI_DH_EMC=m +CONFIG_SCSI_DH_ALUA=m +CONFIG_SCSI_OSD_INITIATOR=m +CONFIG_SCSI_OSD_ULD=m +CONFIG_ATA=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=m +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BLK_DEV_DM=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_TCM_FC=m +CONFIG_ISCSI_TARGET=m +CONFIG_NETDEVICES=y +CONFIG_BONDING=m +CONFIG_DUMMY=m +CONFIG_EQUALIZER=m +CONFIG_IFB=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_IPVLAN=m +CONFIG_VXLAN=m +CONFIG_GENEVE=m +CONFIG_GTP=m +CONFIG_MACSEC=m +CONFIG_NETCONSOLE=m +CONFIG_NETCONSOLE_DYNAMIC=y +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_VIRTIO_NET=m +CONFIG_NLMON=m +CONFIG_NET_VRF=m +CONFIG_ATM_DUMMY=m +# CONFIG_NET_VENDOR_ALACRITECH is not set +# CONFIG_NET_VENDOR_AMAZON is not set +# CONFIG_NET_VENDOR_AQUANTIA is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_CADENCE is not set +# CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_HISILICON is not set +# CONFIG_NET_VENDOR_HUAWEI is not set +# CONFIG_NET_VENDOR_INTEL is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MELLANOX is not set +CONFIG_KS8851=m +CONFIG_ENC28J60=y +CONFIG_ENCX24J600=y +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_NETRONOME is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set +CONFIG_SMC91X=m +CONFIG_SMC911X=m +CONFIG_SMSC911X=m +# CONFIG_NET_VENDOR_STMICRO is not set +CONFIG_TI_CPSW=y +CONFIG_TI_CPTS=y +# CONFIG_NET_VENDOR_VIA is not set +CONFIG_WIZNET_W5100=y +CONFIG_WIZNET_W5100_SPI=y +# CONFIG_NET_VENDOR_SYNOPSYS is not set +CONFIG_AT803X_PHY=y +CONFIG_DP83848_PHY=y +CONFIG_DP83867_PHY=m +CONFIG_MICREL_PHY=y +CONFIG_MICROSEMI_PHY=y +CONFIG_SMSC_PHY=y +CONFIG_VITESSE_PHY=y +CONFIG_PPP=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOATM=m +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_SLIP=m +CONFIG_SLIP_COMPRESSED=y +CONFIG_SLIP_SMART=y +CONFIG_SLIP_MODE_SLIP6=y +CONFIG_USB_CATC=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_LAN78XX=m +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_NCM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_SR9800=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_NET1080=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_CDC_SUBSET=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_ZAURUS=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_HSO=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_CDC_PHONET=m +CONFIG_USB_IPHETH=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_USB_NET_CH9200=m +CONFIG_ATH9K=m +CONFIG_ATH9K_HTC=m +CONFIG_ATH9K_HWRNG=y +CONFIG_CARL9170=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_SDIO=m +CONFIG_ATH6KL_USB=m +CONFIG_AR5523=m +CONFIG_ATH10K=m +CONFIG_ATH10K_USB=m +CONFIG_AT76C50X_USB=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_HOSTAP=m +CONFIG_HOSTAP_FIRMWARE=y +CONFIG_P54_COMMON=m +CONFIG_P54_USB=m +CONFIG_P54_SPI=m +CONFIG_LIBERTAS=m +CONFIG_LIBERTAS_USB=m +CONFIG_LIBERTAS_SDIO=m +CONFIG_LIBERTAS_SPI=m +CONFIG_LIBERTAS_MESH=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_RT2X00=m +CONFIG_RT2500USB=m +CONFIG_RT73USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RTL8187=m +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +CONFIG_RTL8XXXU=m +CONFIG_RSI_91X=m +CONFIG_CW1200=m +CONFIG_CW1200_WLAN_SDIO=m +CONFIG_CW1200_WLAN_SPI=m +CONFIG_WL1251=m +CONFIG_WL1251_SPI=m +CONFIG_WL1251_SDIO=m +CONFIG_WL12XX=m +CONFIG_WL18XX=m +CONFIG_WLCORE_SPI=m +CONFIG_WLCORE_SDIO=m +CONFIG_USB_ZD1201=m +CONFIG_ZD1211RW=m +CONFIG_MAC80211_HWSIM=m +CONFIG_USB_NET_RNDIS_WLAN=m +CONFIG_WIMAX_I2400M_USB=m +CONFIG_IEEE802154_FAKELB=m +CONFIG_IEEE802154_AT86RF230=m +CONFIG_IEEE802154_MRF24J40=m +CONFIG_IEEE802154_CC2520=m +CONFIG_IEEE802154_ATUSB=m +CONFIG_IEEE802154_ADF7242=m +CONFIG_INPUT_SPARSEKMAP=m +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_ADP5588=m +CONFIG_KEYBOARD_ADP5589=m +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_QT1070=m +CONFIG_KEYBOARD_QT2160=m +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_GPIO_POLLED=m +CONFIG_KEYBOARD_TCA6416=m +CONFIG_KEYBOARD_TCA8418=m +CONFIG_KEYBOARD_MATRIX=m +CONFIG_KEYBOARD_LM8323=m +CONFIG_KEYBOARD_LM8333=m +CONFIG_KEYBOARD_MAX7359=m +CONFIG_KEYBOARD_MCS=m +CONFIG_KEYBOARD_NEWTON=m +CONFIG_KEYBOARD_OPENCORES=m +CONFIG_KEYBOARD_STOWAWAY=m +CONFIG_KEYBOARD_CAP11XX=m +# CONFIG_MOUSE_PS2 is not set +CONFIG_MOUSE_APPLETOUCH=m +CONFIG_MOUSE_ELAN_I2C=m +CONFIG_MOUSE_SYNAPTICS_I2C=m +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=y +CONFIG_JOYSTICK_IFORCE_232=y +CONFIG_JOYSTICK_WARRIOR=m +CONFIG_JOYSTICK_MAGELLAN=m +CONFIG_JOYSTICK_SPACEORB=m +CONFIG_JOYSTICK_SPACEBALL=m +CONFIG_JOYSTICK_STINGER=m +CONFIG_JOYSTICK_TWIDJOY=m +CONFIG_JOYSTICK_ZHENHUA=m +CONFIG_JOYSTICK_AS5011=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_JOYSTICK_PSXPAD_SPI=m +CONFIG_JOYSTICK_PSXPAD_SPI_FF=y +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_GTCO=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_TABLET_SERIAL_WACOM4=m +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_ADS7846=m +CONFIG_TOUCHSCREEN_AD7877=m +CONFIG_TOUCHSCREEN_AD7879=m +CONFIG_TOUCHSCREEN_AD7879_I2C=m +CONFIG_TOUCHSCREEN_AD7879_SPI=m +CONFIG_TOUCHSCREEN_AR1021_I2C=y +CONFIG_TOUCHSCREEN_ATMEL_MXT=m +CONFIG_TOUCHSCREEN_AUO_PIXCIR=m +CONFIG_TOUCHSCREEN_BU21013=m +CONFIG_TOUCHSCREEN_CHIPONE_ICN8318=m +CONFIG_TOUCHSCREEN_CY8CTMG110=m +CONFIG_TOUCHSCREEN_CYTTSP_CORE=m +CONFIG_TOUCHSCREEN_CYTTSP_I2C=m +CONFIG_TOUCHSCREEN_CYTTSP_SPI=m +CONFIG_TOUCHSCREEN_CYTTSP4_CORE=m +CONFIG_TOUCHSCREEN_CYTTSP4_I2C=m +CONFIG_TOUCHSCREEN_CYTTSP4_SPI=m +CONFIG_TOUCHSCREEN_DYNAPRO=m +CONFIG_TOUCHSCREEN_HAMPSHIRE=m +CONFIG_TOUCHSCREEN_EETI=m +CONFIG_TOUCHSCREEN_EGALAX=m +CONFIG_TOUCHSCREEN_FUJITSU=m +CONFIG_TOUCHSCREEN_GOODIX=m +CONFIG_TOUCHSCREEN_ILI210X=m +CONFIG_TOUCHSCREEN_GUNZE=m +CONFIG_TOUCHSCREEN_EKTF2127=m +CONFIG_TOUCHSCREEN_ELAN=m +CONFIG_TOUCHSCREEN_ELO=m +CONFIG_TOUCHSCREEN_WACOM_W8001=m +CONFIG_TOUCHSCREEN_WACOM_I2C=m +CONFIG_TOUCHSCREEN_MAX11801=m +CONFIG_TOUCHSCREEN_MCS5000=m +CONFIG_TOUCHSCREEN_MMS114=m +CONFIG_TOUCHSCREEN_MELFAS_MIP4=m +CONFIG_TOUCHSCREEN_MTOUCH=m +CONFIG_TOUCHSCREEN_IMX6UL_TSC=m +CONFIG_TOUCHSCREEN_INEXIO=m +CONFIG_TOUCHSCREEN_MK712=m +CONFIG_TOUCHSCREEN_PENMOUNT=m +CONFIG_TOUCHSCREEN_EDT_FT5X06=y +CONFIG_TOUCHSCREEN_TOUCHRIGHT=m +CONFIG_TOUCHSCREEN_TOUCHWIN=m +CONFIG_TOUCHSCREEN_TI_AM335X_TSC=y +CONFIG_TOUCHSCREEN_PIXCIR=m +CONFIG_TOUCHSCREEN_WDT87XX_I2C=m +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_TSC_SERIO=m +CONFIG_TOUCHSCREEN_TSC2004=m +CONFIG_TOUCHSCREEN_TSC2005=m +CONFIG_TOUCHSCREEN_TSC2007=m +CONFIG_TOUCHSCREEN_SILEAD=y +CONFIG_TOUCHSCREEN_ST1232=m +CONFIG_TOUCHSCREEN_STMPE=y +CONFIG_TOUCHSCREEN_SX8654=m +CONFIG_TOUCHSCREEN_TPS6507X=m +CONFIG_TOUCHSCREEN_ZFORCE=m +CONFIG_TOUCHSCREEN_ROHM_BU21023=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_AD714X=m +CONFIG_INPUT_BMA150=m +CONFIG_INPUT_E3X0_BUTTON=m +CONFIG_INPUT_MMA8450=m +CONFIG_INPUT_GP2A=m +CONFIG_INPUT_GPIO_BEEPER=m +CONFIG_INPUT_GPIO_TILT_POLLED=m +CONFIG_INPUT_GPIO_DECODER=m +CONFIG_INPUT_ATI_REMOTE2=m +CONFIG_INPUT_KEYSPAN_REMOTE=m +CONFIG_INPUT_KXTJ9=m +CONFIG_INPUT_KXTJ9_POLLED_MODE=y +CONFIG_INPUT_POWERMATE=m +CONFIG_INPUT_YEALINK=m +CONFIG_INPUT_CM109=m +CONFIG_INPUT_REGULATOR_HAPTIC=m +CONFIG_INPUT_TPS65218_PWRBUTTON=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_PALMAS_PWRBUTTON=y +CONFIG_INPUT_PCF8574=m +CONFIG_INPUT_PWM_BEEPER=m +CONFIG_INPUT_PWM_VIBRA=m +CONFIG_INPUT_GPIO_ROTARY_ENCODER=m +CONFIG_INPUT_IMS_PCU=m +CONFIG_INPUT_CMA3000=m +CONFIG_INPUT_CMA3000_I2C=m +CONFIG_INPUT_DRV260X_HAPTICS=m +CONFIG_INPUT_DRV2665_HAPTICS=m +CONFIG_INPUT_DRV2667_HAPTICS=m +# CONFIG_LEGACY_PTYS is not set +CONFIG_N_GSM=m +CONFIG_DEVKMEM=y +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_DMA is not set +CONFIG_SERIAL_8250_NR_UARTS=6 +CONFIG_SERIAL_8250_RUNTIME_UARTS=6 +CONFIG_SERIAL_8250_OMAP=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_MAX3100=m +CONFIG_SERIAL_MAX310X=m +CONFIG_SERIAL_SC16IS7XX=m +CONFIG_SERIAL_SC16IS7XX_SPI=y +CONFIG_SERIAL_DEV_BUS=y +CONFIG_TTY_PRINTK=m +CONFIG_VIRTIO_CONSOLE=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS_I2C_ATMEL=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_I2C_ARB_GPIO_CHALLENGE=m +CONFIG_I2C_MUX_GPIO=y +CONFIG_I2C_MUX_PCA954x=m +CONFIG_I2C_MUX_PINCTRL=y +CONFIG_I2C_GPIO=y +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_ROBOTFUZZ_OSIF=m +CONFIG_I2C_TAOS_EVM=m +CONFIG_I2C_TINY_USB=m +CONFIG_SPI=y +CONFIG_SPI_GPIO=m +CONFIG_SPI_OMAP24XX=y +CONFIG_SPI_TI_QSPI=y +CONFIG_SPI_SPIDEV=m +CONFIG_PPS_CLIENT_KTIMER=m +CONFIG_PPS_CLIENT_LDISC=m +CONFIG_PPS_CLIENT_GPIO=m +CONFIG_PINCTRL_MCP23S08=m +CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_PALMAS=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_OF_HELPER=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_SYSCON=y +CONFIG_GPIO_ADP5588=m +CONFIG_GPIO_ADNP=m +CONFIG_GPIO_MAX7300=m +CONFIG_GPIO_MAX732X=m +CONFIG_GPIO_PCA953X=y +CONFIG_GPIO_PCA953X_IRQ=y +CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_TPIC2810=m +CONFIG_GPIO_PALMAS=y +CONFIG_GPIO_STMPE=y +CONFIG_GPIO_TPS65218=y +CONFIG_GPIO_74X164=m +CONFIG_GPIO_MAX7301=m +CONFIG_GPIO_MC33880=m +CONFIG_GPIO_PISOSR=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_MASTER_DS2482=m +CONFIG_W1_MASTER_GPIO=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2760=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_POWER_AVS=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_GPIO=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_POWER_RESET_RESTART=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_POWER_RESET_SYSCON_POWEROFF=y +CONFIG_GENERIC_ADC_BATTERY=y +CONFIG_CHARGER_GPIO=y +CONFIG_SENSORS_AD7314=m +CONFIG_SENSORS_AD7414=m +CONFIG_SENSORS_AD7418=m +CONFIG_SENSORS_ADM1021=m +CONFIG_SENSORS_ADM1025=m +CONFIG_SENSORS_ADM1026=m +CONFIG_SENSORS_ADM1029=m +CONFIG_SENSORS_ADM1031=m +CONFIG_SENSORS_ADM9240=m +CONFIG_SENSORS_ADT7310=m +CONFIG_SENSORS_ADT7410=m +CONFIG_SENSORS_ADT7411=m +CONFIG_SENSORS_ADT7462=m +CONFIG_SENSORS_ADT7470=m +CONFIG_SENSORS_ADT7475=m +CONFIG_SENSORS_ASC7621=m +CONFIG_SENSORS_ATXP1=m +CONFIG_SENSORS_DS620=m +CONFIG_SENSORS_DS1621=m +CONFIG_SENSORS_F71805F=m +CONFIG_SENSORS_F71882FG=m +CONFIG_SENSORS_F75375S=m +CONFIG_SENSORS_FTSTEUTATES=m +CONFIG_SENSORS_GL518SM=m +CONFIG_SENSORS_GL520SM=m +CONFIG_SENSORS_G760A=m +CONFIG_SENSORS_G762=m +CONFIG_SENSORS_GPIO_FAN=y +CONFIG_SENSORS_HIH6130=m +CONFIG_SENSORS_IIO_HWMON=m +CONFIG_SENSORS_IT87=m +CONFIG_SENSORS_JC42=m +CONFIG_SENSORS_POWR1220=m +CONFIG_SENSORS_LINEAGE=m +CONFIG_SENSORS_LTC2945=m +CONFIG_SENSORS_LTC2990=m +CONFIG_SENSORS_LTC4151=m +CONFIG_SENSORS_LTC4215=m +CONFIG_SENSORS_LTC4222=m +CONFIG_SENSORS_LTC4245=m +CONFIG_SENSORS_LTC4260=m +CONFIG_SENSORS_LTC4261=m +CONFIG_SENSORS_MAX1111=m +CONFIG_SENSORS_MAX16065=m +CONFIG_SENSORS_MAX1619=m +CONFIG_SENSORS_MAX1668=m +CONFIG_SENSORS_MAX197=m +CONFIG_SENSORS_MAX31722=m +CONFIG_SENSORS_MAX6639=m +CONFIG_SENSORS_MAX6642=m +CONFIG_SENSORS_MAX6650=m +CONFIG_SENSORS_MAX6697=m +CONFIG_SENSORS_MAX31790=m +CONFIG_SENSORS_MCP3021=m +CONFIG_SENSORS_TC654=m +CONFIG_SENSORS_ADCXX=m +CONFIG_SENSORS_LM63=m +CONFIG_SENSORS_LM70=m +CONFIG_SENSORS_LM73=m +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM77=m +CONFIG_SENSORS_LM78=m +CONFIG_SENSORS_LM80=m +CONFIG_SENSORS_LM83=m +CONFIG_SENSORS_LM85=m +CONFIG_SENSORS_LM87=m +CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_LM92=m +CONFIG_SENSORS_LM93=m +CONFIG_SENSORS_LM95234=m +CONFIG_SENSORS_LM95241=m +CONFIG_SENSORS_LM95245=m +CONFIG_SENSORS_PC87360=m +CONFIG_SENSORS_PC87427=m +CONFIG_SENSORS_NTC_THERMISTOR=m +CONFIG_SENSORS_NCT6683=m +CONFIG_SENSORS_NCT6775=m +CONFIG_SENSORS_NCT7802=m +CONFIG_SENSORS_NCT7904=m +CONFIG_SENSORS_PCF8591=m +CONFIG_PMBUS=m +CONFIG_SENSORS_ADM1275=m +CONFIG_SENSORS_LM25066=m +CONFIG_SENSORS_LTC2978=m +CONFIG_SENSORS_LTC2978_REGULATOR=y +CONFIG_SENSORS_LTC3815=m +CONFIG_SENSORS_MAX16064=m +CONFIG_SENSORS_MAX20751=m +CONFIG_SENSORS_MAX34440=m +CONFIG_SENSORS_MAX8688=m +CONFIG_SENSORS_TPS40422=m +CONFIG_SENSORS_TPS53679=m +CONFIG_SENSORS_UCD9000=m +CONFIG_SENSORS_UCD9200=m +CONFIG_SENSORS_ZL6100=m +CONFIG_SENSORS_PWM_FAN=m +CONFIG_SENSORS_SHT15=m +CONFIG_SENSORS_SHT21=m +CONFIG_SENSORS_SHTC1=m +CONFIG_SENSORS_DME1737=m +CONFIG_SENSORS_EMC1403=m +CONFIG_SENSORS_EMC2103=m +CONFIG_SENSORS_EMC6W201=m +CONFIG_SENSORS_SMSC47M1=m +CONFIG_SENSORS_SMSC47M192=m +CONFIG_SENSORS_SMSC47B397=m +CONFIG_SENSORS_SCH5627=m +CONFIG_SENSORS_SCH5636=m +CONFIG_SENSORS_SMM665=m +CONFIG_SENSORS_ADC128D818=m +CONFIG_SENSORS_ADS1015=m +CONFIG_SENSORS_ADS7828=m +CONFIG_SENSORS_ADS7871=m +CONFIG_SENSORS_AMC6821=m +CONFIG_SENSORS_INA209=m +CONFIG_SENSORS_INA2XX=m +CONFIG_SENSORS_INA3221=m +CONFIG_SENSORS_TC74=m +CONFIG_SENSORS_THMC50=m +CONFIG_SENSORS_TMP102=y +CONFIG_SENSORS_TMP103=m +CONFIG_SENSORS_TMP108=m +CONFIG_SENSORS_TMP401=m +CONFIG_SENSORS_TMP421=m +CONFIG_SENSORS_VT1211=m +CONFIG_SENSORS_W83781D=m +CONFIG_SENSORS_W83791D=m +CONFIG_SENSORS_W83792D=m +CONFIG_SENSORS_W83793=m +CONFIG_SENSORS_W83795=m +CONFIG_SENSORS_W83L785TS=m +CONFIG_SENSORS_W83L786NG=m +CONFIG_SENSORS_W83627HF=m +CONFIG_SENSORS_W83627EHF=m +CONFIG_THERMAL_GOV_FAIR_SHARE=y +CONFIG_THERMAL_GOV_BANG_BANG=y +CONFIG_CPU_THERMAL=y +CONFIG_CLOCK_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y +CONFIG_TI_SOC_THERMAL=y +CONFIG_TI_THERMAL=y +CONFIG_OMAP5_THERMAL=y +CONFIG_DRA752_THERMAL=y +CONFIG_GENERIC_ADC_THERMAL=m +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_SOFT_WATCHDOG=y +CONFIG_OMAP_WATCHDOG=y +CONFIG_MFD_RTSX_USB=m +CONFIG_MFD_STMPE=y +CONFIG_MFD_TI_AM335X_TSCADC=y +CONFIG_MFD_PALMAS=y +CONFIG_MFD_TPS65217=y +CONFIG_MFD_TPS65218=y +CONFIG_MFD_WL1273_CORE=m +CONFIG_REGULATOR_USERSPACE_CONSUMER=y +CONFIG_REGULATOR_GPIO=y +CONFIG_REGULATOR_PALMAS=y +CONFIG_REGULATOR_PBIAS=y +CONFIG_REGULATOR_PWM=y +CONFIG_REGULATOR_TI_ABB=y +CONFIG_REGULATOR_TPS65217=y +CONFIG_REGULATOR_TPS65218=y +CONFIG_RC_CORE=m +CONFIG_LIRC=m +CONFIG_RC_DEVICES=y +CONFIG_RC_ATI_REMOTE=m +CONFIG_IR_IMON=m +CONFIG_IR_MCEUSB=m +CONFIG_IR_REDRAT3=m +CONFIG_IR_STREAMZAP=m +CONFIG_IR_IGORPLUGUSB=m +CONFIG_IR_IGUANA=m +CONFIG_IR_TTUSBIR=m +CONFIG_RC_LOOPBACK=m +CONFIG_IR_GPIO_CIR=m +CONFIG_MEDIA_SUPPORT=m +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y +CONFIG_MEDIA_RADIO_SUPPORT=y +CONFIG_MEDIA_SDR_SUPPORT=y +CONFIG_MEDIA_CEC_SUPPORT=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_V4L2_SUBDEV_API=y +CONFIG_DVB_MAX_ADAPTERS=8 +CONFIG_DVB_DYNAMIC_MINORS=y +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_VIDEO_CLASS=m +CONFIG_USB_M5602=m +CONFIG_USB_STV06XX=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_PWC=m +CONFIG_VIDEO_CPIA2=m +CONFIG_USB_ZR364XX=m +CONFIG_USB_STKWEBCAM=m +CONFIG_USB_S2255=m +CONFIG_VIDEO_USBTV=m +CONFIG_VIDEO_PVRUSB2=m +CONFIG_VIDEO_HDPVR=m +CONFIG_VIDEO_USBVISION=m +CONFIG_VIDEO_STK1160_COMMON=m +CONFIG_VIDEO_GO7007=m +CONFIG_VIDEO_GO7007_USB=m +CONFIG_VIDEO_GO7007_USB_S2250_BOARD=m +CONFIG_VIDEO_AU0828=m +CONFIG_VIDEO_AU0828_RC=y +CONFIG_VIDEO_CX231XX=m +CONFIG_VIDEO_CX231XX_ALSA=m +CONFIG_VIDEO_CX231XX_DVB=m +CONFIG_DVB_USB=m +CONFIG_DVB_USB_A800=m +CONFIG_DVB_USB_DIBUSB_MB=m +CONFIG_DVB_USB_DIBUSB_MB_FAULTY=y +CONFIG_DVB_USB_DIBUSB_MC=m +CONFIG_DVB_USB_DIB0700=m +CONFIG_DVB_USB_UMT_010=m +CONFIG_DVB_USB_CXUSB=m +CONFIG_DVB_USB_M920X=m +CONFIG_DVB_USB_DIGITV=m +CONFIG_DVB_USB_VP7045=m +CONFIG_DVB_USB_VP702X=m +CONFIG_DVB_USB_GP8PSK=m +CONFIG_DVB_USB_NOVA_T_USB2=m +CONFIG_DVB_USB_TTUSB2=m +CONFIG_DVB_USB_DTT200U=m +CONFIG_DVB_USB_OPERA1=m +CONFIG_DVB_USB_AF9005=m +CONFIG_DVB_USB_AF9005_REMOTE=m +CONFIG_DVB_USB_PCTV452E=m +CONFIG_DVB_USB_DW2102=m +CONFIG_DVB_USB_CINERGY_T2=m +CONFIG_DVB_USB_DTV5100=m +CONFIG_DVB_USB_FRIIO=m +CONFIG_DVB_USB_AZ6027=m +CONFIG_DVB_USB_TECHNISAT_USB2=m +CONFIG_DVB_USB_V2=m +CONFIG_DVB_USB_AF9015=m +CONFIG_DVB_USB_AF9035=m +CONFIG_DVB_USB_ANYSEE=m +CONFIG_DVB_USB_AU6610=m +CONFIG_DVB_USB_AZ6007=m +CONFIG_DVB_USB_CE6230=m +CONFIG_DVB_USB_EC168=m +CONFIG_DVB_USB_GL861=m +CONFIG_DVB_USB_LME2510=m +CONFIG_DVB_USB_MXL111SF=m +CONFIG_DVB_USB_RTL28XXU=m +CONFIG_DVB_USB_DVBSKY=m +CONFIG_SMS_USB_DRV=m +CONFIG_DVB_B2C2_FLEXCOP_USB=m +CONFIG_DVB_AS102=m +CONFIG_VIDEO_EM28XX=m +CONFIG_VIDEO_EM28XX_V4L2=m +CONFIG_VIDEO_EM28XX_ALSA=m +CONFIG_VIDEO_EM28XX_DVB=m +CONFIG_USB_AIRSPY=m +CONFIG_USB_HACKRF=m +CONFIG_USB_MSI2500=m +CONFIG_USB_PULSE8_CEC=m +CONFIG_USB_RAINSHADOW_CEC=m +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_SOC_CAMERA=m +CONFIG_SOC_CAMERA_PLATFORM=m +CONFIG_VIDEO_AM437X_VPFE=m +CONFIG_V4L_MEM2MEM_DRIVERS=y +CONFIG_VIDEO_MEM2MEM_DEINTERLACE=m +CONFIG_VIDEO_TI_VPE=m +CONFIG_SMS_SDIO_DRV=m +CONFIG_RADIO_SI470X=y +CONFIG_USB_SI470X=m +CONFIG_RADIO_SI4713=m +CONFIG_I2C_SI4713=m +CONFIG_USB_MR800=m +CONFIG_RADIO_SHARK=m +CONFIG_RADIO_SHARK2=m +CONFIG_USB_KEENE=m +CONFIG_USB_RAREMONO=m +CONFIG_USB_MA901=m +CONFIG_RADIO_WL128X=m +CONFIG_SOC_CAMERA_IMX074=m +CONFIG_SOC_CAMERA_MT9M001=m +CONFIG_SOC_CAMERA_MT9M111=m +CONFIG_SOC_CAMERA_MT9T031=m +CONFIG_SOC_CAMERA_MT9T112=m +CONFIG_SOC_CAMERA_MT9V022=m +CONFIG_SOC_CAMERA_OV5642=m +CONFIG_SOC_CAMERA_OV772X=m +CONFIG_SOC_CAMERA_OV9640=m +CONFIG_SOC_CAMERA_OV9740=m +CONFIG_SOC_CAMERA_RJ54N1=m +CONFIG_SOC_CAMERA_TW9910=m +CONFIG_DRM=y +CONFIG_DRM_LOAD_EDID_FIRMWARE=y +CONFIG_DRM_I2C_ADIHDMI=y +CONFIG_DRM_I2C_NXP_TDA998X=y +CONFIG_DRM_VGEM=y +CONFIG_DRM_UDL=y +CONFIG_DRM_OMAP=y +CONFIG_OMAP5_DSS_HDMI=y +CONFIG_DRM_OMAP_ENCODER_TPD12S015=y +CONFIG_DRM_OMAP_CONNECTOR_HDMI=y +CONFIG_DRM_OMAP_PANEL_DPI=y +CONFIG_DRM_TILCDC=y +CONFIG_DRM_PANEL_SIMPLE=y +CONFIG_DRM_SII902X=y +CONFIG_DRM_TI_TFP410=y +CONFIG_DRM_I2C_ADV7511=y +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_ETNAVIV=y +CONFIG_DRM_TINYDRM=m +CONFIG_TINYDRM_MI0283QT=m +CONFIG_TINYDRM_REPAPER=m +CONFIG_TINYDRM_ST7586=m +CONFIG_DRM_LEGACY=y +CONFIG_FIRMWARE_EDID=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m +CONFIG_FB_SIMPLE=y +CONFIG_FB_SSD1307=y +# CONFIG_LCD_CLASS_DEVICE is not set +# CONFIG_BACKLIGHT_GENERIC is not set +CONFIG_BACKLIGHT_PWM=y +CONFIG_BACKLIGHT_GPIO=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_HRTIMER=m +CONFIG_SND_DYNAMIC_MINORS=y +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_DUMMY=m +CONFIG_SND_ALOOP=m +CONFIG_SND_PWMSP=m +CONFIG_SND_HDA_PREALLOC_SIZE=2048 +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_BCD2000=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_SOC=y +CONFIG_SND_EDMA_SOC=y +CONFIG_SND_AM33XX_SOC_EVM=y +CONFIG_SND_OMAP_SOC=y +CONFIG_SND_OMAP_SOC_HDMI_AUDIO=y +CONFIG_SND_SOC_TLV320AIC23_I2C=y +CONFIG_SND_SOC_TLV320AIC31XX=y +CONFIG_SND_SOC_TS3A227E=m +CONFIG_SND_SOC_WM8753=m +CONFIG_SND_SOC_WM8903=m +CONFIG_SND_SOC_TPA6130A2=m +CONFIG_SND_SIMPLE_CARD=y +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_ASUS=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CP2112=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EZKEY=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_HOLTEK=m +CONFIG_HOLTEK_FF=y +CONFIG_HID_GT683R=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_WALTOP=m +CONFIG_HID_GYRATION=m +CONFIG_HID_ICADE=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PICOLCD_CIR=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_RETRODE=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SONY=m +CONFIG_SONY_FF=y +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_RMI=m +CONFIG_HID_GREENASIA=m +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_WACOM=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_ZEROPLUS_FF=y +CONFIG_HID_ZYDACRON=m +CONFIG_HID_SENSOR_CUSTOM_SENSOR=m +CONFIG_HID_ALPS=m +CONFIG_USB_HID=m +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_I2C_HID=m +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_DYNAMIC_MINORS=y +CONFIG_USB_OTG=y +CONFIG_USB_LEDS_TRIGGER_USBPORT=m +CONFIG_USB_MON=m +CONFIG_USB_WUSB_CBAF=m +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_PRINTER=m +CONFIG_USB_TMC=m +CONFIG_USB_STORAGE=y +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_UAS=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VUDC=m +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_DSPS=y +CONFIG_MUSB_PIO_ONLY=y +CONFIG_USB_DWC3=y +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_METRO=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_MXUPORT=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_XIRCOM=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SERIAL_WISHBONE=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_DEBUG=m +CONFIG_USB_EMI62=m +CONFIG_USB_EMI26=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_RIO500=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_LCD=m +CONFIG_USB_CYPRESS_CY7C63=m +CONFIG_USB_CYTHERM=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_FTDI_ELAN=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_SISUSBVGA_CON=y +CONFIG_USB_LD=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_TEST=m +CONFIG_USB_EHSET_TEST_FIXTURE=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_YUREX=m +CONFIG_USB_HSIC_USB3503=m +CONFIG_USB_CHAOSKEY=m +CONFIG_NOP_USB_XCEIV=y +CONFIG_AM335X_PHY_USB=y +CONFIG_USB_GPIO_VBUS=y +CONFIG_USB_ULPI=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_VBUS_DRAW=500 +CONFIG_USB_DUMMY_HCD=m +CONFIG_USB_CONFIGFS=m +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_OBEX=y +CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y +CONFIG_USB_CONFIGFS_ECM_SUBSET=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y +CONFIG_USB_CONFIGFS_PHONET=y +CONFIG_USB_CONFIGFS_MASS_STORAGE=y +CONFIG_USB_CONFIGFS_F_LB_SS=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_UAC1=y +CONFIG_USB_CONFIGFS_F_UAC2=y +CONFIG_USB_CONFIGFS_F_MIDI=y +CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y +CONFIG_USB_CONFIGFS_F_PRINTER=y +CONFIG_USB_ZERO=m +CONFIG_USB_AUDIO=m +CONFIG_USB_ETH=m +CONFIG_USB_G_NCM=m +CONFIG_USB_GADGETFS=m +CONFIG_USB_FUNCTIONFS=m +CONFIG_USB_FUNCTIONFS_ETH=y +CONFIG_USB_FUNCTIONFS_RNDIS=y +CONFIG_USB_FUNCTIONFS_GENERIC=y +CONFIG_USB_MASS_STORAGE=m +CONFIG_USB_G_SERIAL=m +CONFIG_USB_MIDI_GADGET=m +CONFIG_USB_G_PRINTER=m +CONFIG_USB_CDC_COMPOSITE=m +CONFIG_USB_G_NOKIA=m +CONFIG_USB_G_ACM_MS=m +CONFIG_USB_G_MULTI=m +CONFIG_USB_G_HID=m +CONFIG_USB_G_DBGP=m +CONFIG_USB_LED_TRIG=y +CONFIG_MMC=y +CONFIG_SDIO_UART=m +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_OMAP_HS=y +CONFIG_MMC_SPI=y +CONFIG_MMC_VUB300=m +CONFIG_MMC_USHC=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_SDHCI_OMAP=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_LM3530=m +CONFIG_LEDS_LM3642=m +CONFIG_LEDS_PCA9532=m +CONFIG_LEDS_PCA9532_GPIO=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_LP3944=m +CONFIG_LEDS_LP5521=m +CONFIG_LEDS_LP5523=m +CONFIG_LEDS_LP5562=m +CONFIG_LEDS_LP8501=m +CONFIG_LEDS_LP8860=m +CONFIG_LEDS_PCA955X=m +CONFIG_LEDS_PCA963X=m +CONFIG_LEDS_DAC124S085=m +CONFIG_LEDS_PWM=m +CONFIG_LEDS_REGULATOR=m +CONFIG_LEDS_BD2802=m +CONFIG_LEDS_LT3593=m +CONFIG_LEDS_TCA6507=m +CONFIG_LEDS_TLC591XX=m +CONFIG_LEDS_LM355x=m +CONFIG_LEDS_IS31FL319X=m +CONFIG_LEDS_IS31FL32XX=m +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_DISK=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_ACTIVITY=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_TRANSIENT=m +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_LEDS_TRIGGER_NETDEV=y +CONFIG_ACCESSIBILITY=y +CONFIG_A11Y_BRAILLE_CONSOLE=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_ABB5ZES3=y +CONFIG_RTC_DRV_ABX80X=y +CONFIG_RTC_DRV_DS1307=y +CONFIG_RTC_DRV_DS1374=y +CONFIG_RTC_DRV_DS1374_WDT=y +CONFIG_RTC_DRV_DS1672=y +CONFIG_RTC_DRV_HYM8563=y +CONFIG_RTC_DRV_MAX6900=y +CONFIG_RTC_DRV_RS5C372=y +CONFIG_RTC_DRV_ISL1208=y +CONFIG_RTC_DRV_ISL12022=y +CONFIG_RTC_DRV_X1205=y +CONFIG_RTC_DRV_PCF8523=y +CONFIG_RTC_DRV_PCF85063=y +CONFIG_RTC_DRV_PCF8563=y +CONFIG_RTC_DRV_PCF8583=y +CONFIG_RTC_DRV_M41T80=y +CONFIG_RTC_DRV_M41T80_WDT=y +CONFIG_RTC_DRV_BQ32K=y +CONFIG_RTC_DRV_PALMAS=y +CONFIG_RTC_DRV_S35390A=y +CONFIG_RTC_DRV_FM3130=y +CONFIG_RTC_DRV_RX8010=y +CONFIG_RTC_DRV_RX8581=y +CONFIG_RTC_DRV_RX8025=y +CONFIG_RTC_DRV_EM3027=y +CONFIG_RTC_DRV_RV8803=y +CONFIG_RTC_DRV_M41T93=y +CONFIG_RTC_DRV_M41T94=y +CONFIG_RTC_DRV_DS1302=y +CONFIG_RTC_DRV_DS1305=y +CONFIG_RTC_DRV_DS1343=y +CONFIG_RTC_DRV_DS1347=y +CONFIG_RTC_DRV_DS1390=y +CONFIG_RTC_DRV_MAX6916=y +CONFIG_RTC_DRV_R9701=y +CONFIG_RTC_DRV_RX4581=y +CONFIG_RTC_DRV_RX6110=y +CONFIG_RTC_DRV_RS5C348=y +CONFIG_RTC_DRV_MAX6902=y +CONFIG_RTC_DRV_PCF2123=y +CONFIG_RTC_DRV_MCP795=y +CONFIG_RTC_DRV_DS3232=y +CONFIG_RTC_DRV_PCF2127=y +CONFIG_RTC_DRV_RV3029C2=y +CONFIG_RTC_DRV_CMOS=m +CONFIG_RTC_DRV_DS1286=m +CONFIG_RTC_DRV_DS1511=m +CONFIG_RTC_DRV_DS1553=m +CONFIG_RTC_DRV_DS1685_FAMILY=m +CONFIG_RTC_DRV_DS1742=m +CONFIG_RTC_DRV_DS2404=m +CONFIG_RTC_DRV_STK17TA8=m +CONFIG_RTC_DRV_M48T86=m +CONFIG_RTC_DRV_M48T35=m +CONFIG_RTC_DRV_M48T59=m +CONFIG_RTC_DRV_MSM6242=m +CONFIG_RTC_DRV_BQ4802=m +CONFIG_RTC_DRV_RP5C01=m +CONFIG_RTC_DRV_V3020=m +CONFIG_RTC_DRV_OMAP=y +CONFIG_RTC_DRV_HID_SENSOR_TIME=m +CONFIG_DMADEVICES=y +CONFIG_TI_CPPI41=y +CONFIG_ASYNC_TX_DMA=y +CONFIG_AUXDISPLAY=y +CONFIG_HD44780=m +CONFIG_IMG_ASCII_LCD=m +CONFIG_HT16K33=m +CONFIG_UIO=m +CONFIG_UIO_PDRV_GENIRQ=m +CONFIG_UIO_DMEM_GENIRQ=m +CONFIG_UIO_PRUSS=m +CONFIG_UIO_PRUSS_SHMEM=m +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_RTLLIB=m +CONFIG_RTL8723BS=m +CONFIG_R8712U=m +CONFIG_R8188EU=m +CONFIG_ADIS16201=m +CONFIG_ADIS16203=m +CONFIG_ADIS16209=m +CONFIG_ADIS16240=m +CONFIG_AD7606=m +CONFIG_AD7606_IFACE_SPI=m +CONFIG_AD7780=m +CONFIG_AD7816=m +CONFIG_AD7192=m +CONFIG_AD7280=m +CONFIG_ADT7316=m +CONFIG_ADT7316_I2C=m +CONFIG_AD7150=m +CONFIG_AD7152=m +CONFIG_AD7746=m +CONFIG_AD9832=m +CONFIG_AD9834=m +CONFIG_ADIS16060=m +CONFIG_AD5933=m +CONFIG_TSL2x7x=m +CONFIG_ADE7753=m +CONFIG_ADE7754=m +CONFIG_ADE7758=m +CONFIG_ADE7759=m +CONFIG_ADE7854=m +CONFIG_AD2S90=m +CONFIG_AD2S1200=m +CONFIG_AD2S1210=m +CONFIG_FB_TFT=m +CONFIG_FB_TFT_AGM1264K_FL=m +CONFIG_FB_TFT_BD663474=m +CONFIG_FB_TFT_HX8340BN=m +CONFIG_FB_TFT_HX8347D=m +CONFIG_FB_TFT_HX8353D=m +CONFIG_FB_TFT_HX8357D=m +CONFIG_FB_TFT_ILI9163=m +CONFIG_FB_TFT_ILI9320=m +CONFIG_FB_TFT_ILI9325=m +CONFIG_FB_TFT_ILI9340=m +CONFIG_FB_TFT_ILI9341=m +CONFIG_FB_TFT_ILI9481=m +CONFIG_FB_TFT_ILI9486=m +CONFIG_FB_TFT_PCD8544=m +CONFIG_FB_TFT_RA8875=m +CONFIG_FB_TFT_S6D02A1=m +CONFIG_FB_TFT_S6D1121=m +CONFIG_FB_TFT_SH1106=m +CONFIG_FB_TFT_SSD1289=m +CONFIG_FB_TFT_SSD1305=m +CONFIG_FB_TFT_SSD1306=m +CONFIG_FB_TFT_SSD1325=m +CONFIG_FB_TFT_SSD1331=m +CONFIG_FB_TFT_SSD1351=m +CONFIG_FB_TFT_ST7735R=m +CONFIG_FB_TFT_ST7789V=m +CONFIG_FB_TFT_TINYLCD=m +CONFIG_FB_TFT_TLS8204=m +CONFIG_FB_TFT_UC1611=m +CONFIG_FB_TFT_UC1701=m +CONFIG_FB_TFT_UPD161704=m +CONFIG_FB_TFT_WATTEROTT=m +CONFIG_FB_FLEX=m +CONFIG_FB_TFT_FBTFT_DEVICE=m +CONFIG_GREYBUS=m +CONFIG_GREYBUS_ES2=m +CONFIG_GREYBUS_AUDIO=m +CONFIG_GREYBUS_BOOTROM=m +CONFIG_GREYBUS_FIRMWARE=m +CONFIG_GREYBUS_HID=m +CONFIG_GREYBUS_LIGHT=m +CONFIG_GREYBUS_LOG=m +CONFIG_GREYBUS_LOOPBACK=m +CONFIG_GREYBUS_POWER=m +CONFIG_GREYBUS_RAW=m +CONFIG_GREYBUS_VIBRATOR=m +CONFIG_GREYBUS_BRIDGED_PHY=m +CONFIG_GREYBUS_GPIO=m +CONFIG_GREYBUS_I2C=m +CONFIG_GREYBUS_PWM=m +CONFIG_GREYBUS_SDIO=m +CONFIG_GREYBUS_SPI=m +CONFIG_GREYBUS_UART=m +CONFIG_GREYBUS_USB=m +CONFIG_COMMON_CLK_PALMAS=y +CONFIG_HWSPINLOCK=y +CONFIG_HWSPINLOCK_OMAP=y +CONFIG_OMAP2PLUS_MBOX=y +CONFIG_IOMMU_IO_PGTABLE_LPAE=y +CONFIG_OMAP_IOMMU=y +CONFIG_REMOTEPROC=y +CONFIG_OMAP_REMOTEPROC=m +CONFIG_WKUP_M3_RPROC=y +CONFIG_PRUSS_REMOTEPROC=m +CONFIG_RPMSG_CHAR=m +CONFIG_RPMSG_VIRTIO=m +CONFIG_RPMSG_RPC=m +CONFIG_RPMSG_PRU=m +CONFIG_SOC_TI=y +CONFIG_AMX3_PM=y +CONFIG_WKUP_M3_IPC=y +CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y +CONFIG_DEVFREQ_GOV_PERFORMANCE=y +CONFIG_DEVFREQ_GOV_POWERSAVE=y +CONFIG_DEVFREQ_GOV_USERSPACE=y +CONFIG_DEVFREQ_GOV_PASSIVE=y +CONFIG_PM_DEVFREQ_EVENT=y +CONFIG_EXTCON_GPIO=y +CONFIG_EXTCON_PALMAS=y +CONFIG_EXTCON_USB_GPIO=y +CONFIG_TI_EMIF=y +CONFIG_TI_EMIF_SRAM=y +CONFIG_IIO=y +CONFIG_IIO_SW_DEVICE=m +CONFIG_IIO_SW_TRIGGER=m +CONFIG_ADXL345_I2C=m +CONFIG_ADXL345_SPI=m +CONFIG_BMA180=m +CONFIG_BMA220=m +CONFIG_BMC150_ACCEL=m +CONFIG_DA280=m +CONFIG_DA311=m +CONFIG_DMARD06=m +CONFIG_DMARD09=m +CONFIG_DMARD10=m +CONFIG_HID_SENSOR_ACCEL_3D=m +CONFIG_IIO_ST_ACCEL_3AXIS=m +CONFIG_KXSD9=m +CONFIG_KXCJK1013=m +CONFIG_MC3230=m +CONFIG_MMA7455_I2C=m +CONFIG_MMA7455_SPI=m +CONFIG_MMA7660=m +CONFIG_MMA8452=m +CONFIG_MMA9551=m +CONFIG_MMA9553=m +CONFIG_MXC4005=m +CONFIG_MXC6255=m +CONFIG_SCA3000=m +CONFIG_STK8312=m +CONFIG_STK8BA50=m +CONFIG_AD7266=m +CONFIG_AD7291=m +CONFIG_AD7298=m +CONFIG_AD7476=m +CONFIG_AD7766=m +CONFIG_AD7791=m +CONFIG_AD7793=m +CONFIG_AD7887=m +CONFIG_AD7923=m +CONFIG_AD799X=m +CONFIG_CC10001_ADC=m +CONFIG_ENVELOPE_DETECTOR=m +CONFIG_HI8435=m +CONFIG_HX711=m +CONFIG_INA2XX_ADC=m +CONFIG_LTC2471=m +CONFIG_LTC2485=m +CONFIG_LTC2497=m +CONFIG_MAX1027=m +CONFIG_MAX11100=m +CONFIG_MAX1118=m +CONFIG_MAX1363=m +CONFIG_MAX9611=m +CONFIG_MCP320X=m +CONFIG_MCP3422=m +CONFIG_NAU7802=m +CONFIG_PALMAS_GPADC=m +CONFIG_STMPE_ADC=m +CONFIG_TI_ADC081C=m +CONFIG_TI_ADC0832=m +CONFIG_TI_ADC084S021=m +CONFIG_TI_ADC12138=m +CONFIG_TI_ADC108S102=m +CONFIG_TI_ADC128S052=m +CONFIG_TI_ADC161S626=m +CONFIG_TI_ADS1015=m +CONFIG_TI_ADS7950=m +CONFIG_TI_ADS8688=m +CONFIG_TI_AM335X_ADC=y +CONFIG_TI_TLC4541=m +CONFIG_AD8366=m +CONFIG_ATLAS_PH_SENSOR=m +CONFIG_CCS811=m +CONFIG_IAQCORE=m +CONFIG_VZ89X=m +CONFIG_AD5064=m +CONFIG_AD5360=m +CONFIG_AD5380=m +CONFIG_AD5421=m +CONFIG_AD5446=m +CONFIG_AD5449=m +CONFIG_AD5592R=m +CONFIG_AD5593R=m +CONFIG_AD5504=m +CONFIG_AD5624R_SPI=m +CONFIG_LTC2632=m +CONFIG_AD5686=m +CONFIG_AD5755=m +CONFIG_AD5761=m +CONFIG_AD5764=m +CONFIG_AD5791=m +CONFIG_AD7303=m +CONFIG_AD8801=m +CONFIG_DPOT_DAC=m +CONFIG_M62332=m +CONFIG_MAX517=m +CONFIG_MAX5821=m +CONFIG_MCP4725=m +CONFIG_MCP4922=m +CONFIG_AD9523=m +CONFIG_ADF4350=m +CONFIG_ADIS16080=m +CONFIG_ADIS16130=m +CONFIG_ADIS16136=m +CONFIG_ADIS16260=m +CONFIG_ADXRS450=m +CONFIG_BMG160=m +CONFIG_HID_SENSOR_GYRO_3D=m +CONFIG_MPU3050_I2C=m +CONFIG_IIO_ST_GYRO_3AXIS=m +CONFIG_ITG3200=m +CONFIG_AFE4403=m +CONFIG_AFE4404=m +CONFIG_MAX30100=m +CONFIG_MAX30102=m +CONFIG_AM2315=m +CONFIG_DHT11=m +CONFIG_HDC100X=m +CONFIG_HTS221=m +CONFIG_HTU21=m +CONFIG_SI7005=m +CONFIG_SI7020=m +CONFIG_ADIS16400=m +CONFIG_ADIS16480=m +CONFIG_BMI160_I2C=m +CONFIG_BMI160_SPI=m +CONFIG_KMX61=m +CONFIG_INV_MPU6050_I2C=m +CONFIG_INV_MPU6050_SPI=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_ADJD_S311=m +CONFIG_AL3320A=m +CONFIG_APDS9300=m +CONFIG_APDS9960=m +CONFIG_BH1750=m +CONFIG_BH1780=m +CONFIG_CM32181=m +CONFIG_CM3232=m +CONFIG_CM3323=m +CONFIG_CM3605=m +CONFIG_CM36651=m +CONFIG_GP2AP020A00F=m +CONFIG_SENSORS_ISL29018=m +CONFIG_SENSORS_ISL29028=m +CONFIG_ISL29125=m +CONFIG_HID_SENSOR_ALS=m +CONFIG_HID_SENSOR_PROX=m +CONFIG_JSA1212=m +CONFIG_RPR0521=m +CONFIG_LTR501=m +CONFIG_MAX44000=m +CONFIG_OPT3001=m +CONFIG_PA12203001=m +CONFIG_SI1145=m +CONFIG_STK3310=m +CONFIG_TCS3414=m +CONFIG_TCS3472=m +CONFIG_SENSORS_TSL2563=m +CONFIG_TSL2583=m +CONFIG_TSL4531=m +CONFIG_US5182D=m +CONFIG_VCNL4000=m +CONFIG_VEML6070=m +CONFIG_VL6180=m +CONFIG_AK8974=m +CONFIG_AK09911=m +CONFIG_BMC150_MAGN_I2C=m +CONFIG_BMC150_MAGN_SPI=m +CONFIG_MAG3110=m +CONFIG_HID_SENSOR_MAGNETOMETER_3D=m +CONFIG_MMC35240=m +CONFIG_IIO_ST_MAGN_3AXIS=m +CONFIG_SENSORS_HMC5843_I2C=m +CONFIG_SENSORS_HMC5843_SPI=m +CONFIG_HID_SENSOR_INCLINOMETER_3D=m +CONFIG_HID_SENSOR_DEVICE_ROTATION=m +CONFIG_IIO_INTERRUPT_TRIGGER=m +CONFIG_IIO_SYSFS_TRIGGER=m +CONFIG_DS1803=m +CONFIG_MAX5481=m +CONFIG_MAX5487=m +CONFIG_MCP4131=m +CONFIG_MCP4531=m +CONFIG_TPL0102=m +CONFIG_LMP91000=m +CONFIG_ABP060MG=m +CONFIG_BMP280=m +CONFIG_HID_SENSOR_PRESS=m +CONFIG_HP03=m +CONFIG_MPL115_I2C=m +CONFIG_MPL115_SPI=m +CONFIG_MPL3115=m +CONFIG_MS5611=m +CONFIG_MS5611_I2C=m +CONFIG_MS5611_SPI=m +CONFIG_MS5637=m +CONFIG_IIO_ST_PRESS=m +CONFIG_T5403=m +CONFIG_HP206C=m +CONFIG_ZPA2326=m +CONFIG_AS3935=m +CONFIG_LIDAR_LITE_V2=m +CONFIG_SRF04=m +CONFIG_SX9500=m +CONFIG_SRF08=m +CONFIG_VL53L0X_I2C=m +CONFIG_MAXIM_THERMOCOUPLE=m +CONFIG_MLX90614=m +CONFIG_TMP006=m +CONFIG_TMP007=m +CONFIG_TSYS01=m +CONFIG_TSYS02D=m +CONFIG_PWM_OMAP_DMTIMER=y +CONFIG_PWM_PCA9685=y +CONFIG_PWM_TIECAP=y +CONFIG_PWM_TIEHRPWM=y +CONFIG_RESET_CONTROLLER=y +CONFIG_RESET_TI_SYSCON=y +CONFIG_OMAP_USB2=y +CONFIG_TI_PIPE3=y +CONFIG_RAS=y +CONFIG_FPGA=m +CONFIG_FPGA_REGION=m +CONFIG_FPGA_MGR_ICE40_SPI=m +CONFIG_FPGA_MGR_ALTERA_PS_SPI=m +CONFIG_FPGA_MGR_XILINX_SPI=m +CONFIG_FPGA_BRIDGE=m +CONFIG_ALTERA_PR_IP_CORE=m +CONFIG_ALTERA_PR_IP_CORE_PLAT=m +CONFIG_XILINX_PR_DECOUPLER=m +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_ENCRYPTION=y +CONFIG_JFS_FS=m +CONFIG_JFS_POSIX_ACL=y +CONFIG_JFS_SECURITY=y +CONFIG_GFS2_FS=m +CONFIG_GFS2_FS_LOCKING_DLM=y +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_F2FS_FS=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_ENCRYPTION=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_QFMT_V2=m +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=m +CONFIG_OVERLAY_FS=y +CONFIG_FSCACHE=m +CONFIG_FSCACHE_STATS=y +CONFIG_CACHEFILES=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=m +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_IOCHARSET="ascii" +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=m +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_ECRYPT_FS=m +CONFIG_ECRYPT_FS_MESSAGING=y +CONFIG_JFFS2_FS=m +CONFIG_JFFS2_SUMMARY=y +CONFIG_JFFS2_FS_XATTR=y +CONFIG_JFFS2_COMPRESSION_OPTIONS=y +CONFIG_JFFS2_LZO=y +CONFIG_UBIFS_FS=y +CONFIG_UBIFS_FS_ADVANCED_COMPR=y +CONFIG_SQUASHFS=m +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_ROMFS_FS=m +CONFIG_ROMFS_BACKED_BY_BOTH=y +CONFIG_AUFS_FS=m +CONFIG_AUFS_EXPORT=y +CONFIG_AUFS_XATTR=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_RPCSEC_GSS_KRB5=m +CONFIG_SUNRPC_DEBUG=y +CONFIG_CEPH_FS=m +CONFIG_CEPH_FSCACHE=y +CONFIG_CEPH_FS_POSIX_ACL=y +CONFIG_CIFS=m +CONFIG_CIFS_WEAK_PW_HASH=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_ACL=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_FSCACHE=y +CONFIG_CODA_FS=m +CONFIG_AFS_FS=m +CONFIG_AFS_FSCACHE=y +CONFIG_9P_FS=m +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_DLM=m +CONFIG_DLM_DEBUG=y +CONFIG_PRINTK_TIME=y +CONFIG_BOOT_PRINTK_DELAY=y +CONFIG_DYNAMIC_DEBUG=y +CONFIG_STRIP_ASM_SYMS=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6 +CONFIG_DETECT_HUNG_TASK=y +CONFIG_SCHEDSTATS=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_DEBUG_LIST=y +CONFIG_NOTIFIER_ERROR_INJECTION=m +CONFIG_FTRACE_SYSCALLS=y +CONFIG_TRACER_SNAPSHOT=y +CONFIG_STACK_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_TEST_USER_COPY=m +CONFIG_TEST_BPF=m +CONFIG_TEST_FIRMWARE=m +CONFIG_TEST_STATIC_KEYS=m +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_TOMOYO=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_SECURITY_YAMA=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y +CONFIG_DEFAULT_SECURITY_APPARMOR=y +CONFIG_CRYPTO_USER=m +CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_RMD128=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_RMD256=m +CONFIG_CRYPTO_RMD320=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_TGR192=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_ARC4=y +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_SALSA20=m +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_LZ4=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ANSI_CPRNG=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_DEV_OMAP=y +CONFIG_CRYPTO_DEV_OMAP_SHAM=y +CONFIG_CRYPTO_DEV_OMAP_AES=y +CONFIG_CRYPTO_DEV_OMAP_DES=y +CONFIG_ARM_CRYPTO=y +CONFIG_CRYPTO_SHA1_ARM_NEON=m +CONFIG_CRYPTO_SHA256_ARM=m +CONFIG_CRYPTO_SHA512_ARM=m +CONFIG_CRYPTO_AES_ARM=m +CONFIG_CRYPTO_AES_ARM_BS=m +# CONFIG_RAID6_PQ_BENCHMARK is not set +# CONFIG_XZ_DEC_X86 is not set +# CONFIG_XZ_DEC_POWERPC is not set +# CONFIG_XZ_DEC_IA64 is not set +# CONFIG_XZ_DEC_SPARC is not set +CONFIG_CORDIC=m +CONFIG_IRQ_POLL=y diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h index 4ffb26d4cad8d..cea3e0c6469bb 100644 --- a/arch/arm/include/asm/xor.h +++ b/arch/arm/include/asm/xor.h @@ -212,3 +212,9 @@ static struct xor_block_template xor_block_neon = { #else #define NEON_TEMPLATES #endif + +#ifdef CONFIG_KERNEL_MODE_NEON +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_neon) +#else +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_arm4regs) +#endif diff --git a/arch/arm/mach-omap2/omap4-common.c b/arch/arm/mach-omap2/omap4-common.c index cf65ab8bb0046..e5dcbda20129d 100644 --- a/arch/arm/mach-omap2/omap4-common.c +++ b/arch/arm/mach-omap2/omap4-common.c @@ -131,6 +131,9 @@ static int __init omap4_sram_init(void) struct device_node *np; struct gen_pool *sram_pool; + if (!soc_is_omap44xx() && !soc_is_omap54xx()) + return 0; + np = of_find_compatible_node(NULL, NULL, "ti,omap4-mpu"); if (!np) pr_warn("%s:Unable to allocate sram needed to handle errata I688\n", diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c index 5513dc26eb84a..38444e5917a50 100644 --- a/arch/arm/mach-omap2/omap_device.c +++ b/arch/arm/mach-omap2/omap_device.c @@ -138,8 +138,8 @@ static int omap_device_build_from_dt(struct platform_device *pdev) struct omap_device *od; struct omap_hwmod *oh; struct device_node *node = pdev->dev.of_node; - const char *oh_name; - int oh_cnt, i, ret = 0; + const char *oh_name, *rst_name; + int oh_cnt, dstr_cnt, i, ret = 0; bool device_active = false; oh_cnt = of_property_count_strings(node, "ti,hwmods"); @@ -190,6 +190,26 @@ static int omap_device_build_from_dt(struct platform_device *pdev) omap_device_enable(pdev); pm_runtime_set_active(&pdev->dev); } + dstr_cnt = + of_property_count_strings(node, "ti,deassert-hard-reset"); + if (dstr_cnt > 0) { + for (i = 0; i < dstr_cnt; i += 2) { + of_property_read_string_index( + node, "ti,deassert-hard-reset", i, + &oh_name); + of_property_read_string_index( + node, "ti,deassert-hard-reset", i+1, + &rst_name); + oh = omap_hwmod_lookup(oh_name); + if (!oh) { + dev_warn(&pdev->dev, + "Cannot parse deassert property for '%s'\n", + oh_name); + break; + } + omap_hwmod_deassert_hardreset(oh, rst_name); + } + } odbfd_exit1: kfree(hwmods); @@ -227,12 +247,21 @@ static int _omap_device_notifier_call(struct notifier_block *nb, { struct platform_device *pdev = to_platform_device(dev); struct omap_device *od; - int err; + int i, err; switch (event) { case BUS_NOTIFY_REMOVED_DEVICE: - if (pdev->archdata.od) - omap_device_delete(pdev->archdata.od); + od = to_omap_device(pdev); + if (!od) + break; + + for (i = 0; i < od->hwmods_cnt; i++) { + /* shutdown hwmods */ + omap_hwmod_shutdown(od->hwmods[i]); + /* we don't remove clocks cause there's no API to do so */ + /* no harm done, since they will not be created next time */ + } + omap_device_delete(od); break; case BUS_NOTIFY_UNBOUND_DRIVER: od = to_omap_device(pdev); @@ -832,6 +861,8 @@ int omap_device_idle(struct platform_device *pdev) struct omap_device *od; od = to_omap_device(pdev); + if (!od) + return 0; if (od->_state != OMAP_DEVICE_STATE_ENABLED) { dev_warn(&pdev->dev, diff --git a/drivers/Makefile b/drivers/Makefile index 5f5ccdbad21ae..700403447d2da 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -58,6 +58,9 @@ obj-$(CONFIG_RESET_CONTROLLER) += reset/ obj-y += tty/ obj-y += char/ +# put mmc early as many morden devices use emm/sd card as rootfs storage +obj-y += mmc/ + # iommu/ comes before gpu as gpu are using iommu controllers obj-$(CONFIG_IOMMU_SUPPORT) += iommu/ @@ -129,7 +132,6 @@ obj-$(CONFIG_EDAC) += edac/ obj-$(CONFIG_EISA) += eisa/ obj-$(CONFIG_CPU_FREQ) += cpufreq/ obj-$(CONFIG_CPU_IDLE) += cpuidle/ -obj-y += mmc/ obj-$(CONFIG_MEMSTICK) += memstick/ obj-$(CONFIG_NEW_LEDS) += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ diff --git a/drivers/base/power/opp/of.c b/drivers/base/power/opp/of.c index 87509cb69f792..cd02b51dc2b2f 100644 --- a/drivers/base/power/opp/of.c +++ b/drivers/base/power/opp/of.c @@ -366,7 +366,7 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev, free_opp: _opp_free(new_opp); - return ret; + return 0; } /* Initializes OPP tables based on new bindings */ diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 24a3fb35614f2..9499210a33557 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -711,6 +711,24 @@ static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, return error; } +/* + * for AUFS + * no get/put for file. + */ +struct file *loop_backing_file(struct super_block *sb) +{ + struct file *ret; + struct loop_device *l; + + ret = NULL; + if (MAJOR(sb->s_dev) == LOOP_MAJOR) { + l = sb->s_bdev->bd_disk->private_data; + ret = l->lo_backing_file; + } + return ret; +} +EXPORT_SYMBOL_GPL(loop_backing_file); + /* loop sysfs attributes */ static ssize_t loop_attr_show(struct device *dev, char *page, diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 1918007fa2c2c..5db48e6a37dcc 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -61,6 +61,20 @@ config GPIO_SYSFS Kernel drivers may also request that a particular GPIO be exported to userspace; this can be useful when debugging. +config GPIO_OF_HELPER + bool "GPIO OF helper device (EXPERIMENTAL)" + depends on OF_GPIO + help + Say Y here to add an GPIO OF helper driver + + Allows you specify a GPIO helper based on OF + which allows simple export of GPIO functionality + in user-space. + + Features include, value set/get, direction control, + interrupt/value change poll support, event counting + and others. + config GPIO_GENERIC depends on HAS_IOMEM # Only for IOMEM drivers tristate diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 8a2dfba3b2310..78bd52d7b30de 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_GPIOLIB) += gpiolib-devprop.o obj-$(CONFIG_OF_GPIO) += gpiolib-of.o obj-$(CONFIG_GPIO_SYSFS) += gpiolib-sysfs.o obj-$(CONFIG_GPIO_ACPI) += gpiolib-acpi.o +obj-$(CONFIG_GPIO_OF_HELPER) += gpio-of-helper.o # Device drivers. Generally keep list sorted alphabetically obj-$(CONFIG_GPIO_GENERIC) += gpio-generic.o diff --git a/drivers/gpio/gpio-of-helper.c b/drivers/gpio/gpio-of-helper.c new file mode 100644 index 0000000000000..83f362f7da5ff --- /dev/null +++ b/drivers/gpio/gpio-of-helper.c @@ -0,0 +1,435 @@ +/* + * GPIO OF based helper + * + * A simple DT based driver to provide access to GPIO functionality + * to user-space via sysfs. + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* fwd decl. */ +struct gpio_of_helper_info; + +enum gpio_type { + GPIO_TYPE_INPUT = 0, + GPIO_TYPE_OUTPUT = 1, +}; + +struct gpio_of_entry { + int id; + struct gpio_of_helper_info *info; + struct device_node *node; + enum gpio_type type; + int gpio; + int irq; + const char *name; + atomic64_t counter; + unsigned int count_flags; +#define COUNT_RISING_EDGE (1 << 0) +#define COUNT_FALLING_EDGE (1 << 1) +}; + +struct gpio_of_helper_info { + struct platform_device *pdev; + struct idr idr; +}; + +static const struct of_device_id gpio_of_helper_of_match[] = { + { + .compatible = "gpio-of-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, gpio_of_helper_of_match); + +static ssize_t gpio_of_helper_show_status(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + char *p, *e; + int id, n; + + p = buf; + e = p + PAGE_SIZE; + n = 0; + idr_for_each_entry(&info->idr, entry, id) { + switch (entry->type) { + case GPIO_TYPE_INPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s %llu\n", + entry->id, entry->name, entry->gpio, "IN", + (unsigned long long) + atomic64_read(&entry->counter)); + break; + case GPIO_TYPE_OUTPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s\n", + entry->id, entry->name, entry->gpio, "OUT"); + break; + } + p += n; + } + + return p - buf; +} + +static DEVICE_ATTR(status, S_IRUGO, + gpio_of_helper_show_status, NULL); + +static irqreturn_t gpio_of_helper_handler(int irq, void *ptr) +{ + struct gpio_of_entry *entry = ptr; + + /* caution - low speed interfaces only! */ + atomic64_inc(&entry->counter); + + return IRQ_HANDLED; +} + +static struct gpio_of_entry * +gpio_of_entry_create(struct gpio_of_helper_info *info, + struct device_node *node) +{ + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + struct gpio_of_entry *entry; + int err, gpio, irq; + unsigned int req_flags, count_flags, irq_flags; + enum gpio_type type; + enum of_gpio_flags gpio_flags; + const char *name; + + /* get the type of the node first */ + if (of_property_read_bool(node, "input")) + type = GPIO_TYPE_INPUT; + else if (of_property_read_bool(node, "output") + || of_property_read_bool(node, "init-low") + || of_property_read_bool(node, "init-high")) + type = GPIO_TYPE_OUTPUT; + else { + dev_err(dev, "Not valid gpio node type\n"); + err = -EINVAL; + goto err_bad_node; + } + + /* get the name */ + if (of_property_read_string(node, "line-name", &name)) + if (of_property_read_string(node, "gpio-name", &name)) + name = node->name; + + err = of_get_named_gpio_flags(node, "gpio", 0, &gpio_flags); + if (IS_ERR_VALUE(err)) { + dev_err(dev, "Failed to get gpio property of '%s'\n", name); + goto err_bad_node; + } + gpio = err; + + req_flags = 0; + count_flags = 0; + + /* set the request flags */ + switch (type) { + case GPIO_TYPE_INPUT: + req_flags = GPIOF_DIR_IN | GPIOF_EXPORT; + if (of_property_read_bool(node, "count-falling-edge")) + count_flags |= COUNT_FALLING_EDGE; + if (of_property_read_bool(node, "count-rising-edge")) + count_flags |= COUNT_RISING_EDGE; + break; + case GPIO_TYPE_OUTPUT: + req_flags = GPIOF_DIR_OUT | GPIOF_EXPORT; + if (of_property_read_bool(node, "init-high")) + req_flags |= GPIOF_OUT_INIT_HIGH; + else if (of_property_read_bool(node, "init-low")) + req_flags |= GPIOF_OUT_INIT_LOW; + break; + } + if (of_property_read_bool(node, "dir-changeable")) + req_flags |= GPIOF_EXPORT_CHANGEABLE; + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_ACTIVE_LOW; + if (gpio_flags & OF_GPIO_SINGLE_ENDED) { + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_OPEN_DRAIN; + else + req_flags |= GPIOF_OPEN_SOURCE; + } + + /* request the gpio */ + err = devm_gpio_request_one(dev, gpio, req_flags, name); + if (err != 0) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + irq = -1; + irq_flags = 0; + + /* counter mode requested - need an interrupt */ + if (count_flags != 0) { + irq = gpio_to_irq(gpio); + if (IS_ERR_VALUE(irq)) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + if (count_flags & COUNT_RISING_EDGE) + irq_flags |= IRQF_TRIGGER_RISING; + if (count_flags & COUNT_FALLING_EDGE) + irq_flags |= IRQF_TRIGGER_FALLING; + } + +// if (!idr_pre_get(&info->idr, GFP_KERNEL)) { +// dev_err(dev, "Failed on idr_pre_get of '%s'\n", name); +// err = -ENOMEM; +// goto err_no_mem; +// } + + idr_preload(GFP_KERNEL); + + entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL); + if (entry == NULL) { + dev_err(dev, "Failed to allocate gpio entry of '%s'\n", name); + err = -ENOMEM; + goto err_no_mem; + } + + entry->id = -1; + entry->info = info; + entry->node = of_node_get(node); /* get node reference */ + entry->type = type; + entry->gpio = gpio; + entry->irq = irq; + entry->name = name; + + /* interrupt enable is last thing done */ + if (irq >= 0) { + atomic64_set(&entry->counter, 0); + entry->count_flags = count_flags; + err = devm_request_irq(dev, irq, gpio_of_helper_handler, + irq_flags, name, entry); + if (err != 0) { + dev_err(dev, "Failed to request irq of '%s'\n", name); + goto err_no_irq; + } + } + + /* all done; insert */ +// err = idr_get_new(&info->idr, entry, &entry->id); +// if (IS_ERR_VALUE(err)) { +// dev_err(dev, "Failed to idr_get_new of '%s'\n", name); +// goto err_fail_idr; +// } + + err = idr_alloc(&info->idr, entry, 0, 0, GFP_NOWAIT); + if (err >= 0) + entry->id = err; + + idr_preload_end(); + + if (err < 0) { + dev_err(dev, "Failed to idr_get_new of '%s'\n", name); + goto err_fail_idr; + } + + dev_dbg(dev, "Allocated GPIO id=%d name='%s'\n", entry->id, name); + + return entry; + +err_fail_idr: + /* nothing to do */ +err_no_irq: + /* release node ref */ + of_node_put(node); + /* nothing else needs to be done, devres handles it */ +err_no_mem: +err_bad_node: + return ERR_PTR(err); +} + +static int gpio_of_entry_destroy(struct gpio_of_entry *entry) +{ + struct gpio_of_helper_info *info = entry->info; + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + + dev_dbg(dev, "Destroying GPIO id=%d\n", entry->id); + + /* remove from the IDR */ + idr_remove(&info->idr, entry->id); + + /* remove node ref */ + of_node_put(entry->node); + + /* free gpio */ + devm_gpio_free(dev, entry->gpio); + + /* gree irq */ + if (entry->irq >= 0) + devm_free_irq(dev, entry->irq, entry); + + /* and free */ + devm_kfree(dev, entry); + + return 0; +} + +static int gpio_of_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct gpio_of_helper_info *info; + struct gpio_of_entry *entry; + struct device_node *pnode = pdev->dev.of_node; + struct device_node *cnode; + struct pinctrl *pinctrl; + int err; + + /* we only support OF */ + if (pnode == NULL) { + dev_err(&pdev->dev, "No platform of_node!\n"); + return -ENODEV; + } + + pinctrl = devm_pinctrl_get_select_default(&pdev->dev); + if (IS_ERR(pinctrl)) { + /* special handling for probe defer */ + if (PTR_ERR(pinctrl) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + dev_warn(&pdev->dev, + "pins are not configured from the driver\n"); + } + + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); + if (info == NULL) { + dev_err(&pdev->dev, "Failed to allocate info\n"); + err = -ENOMEM; + goto err_no_mem; + } + platform_set_drvdata(pdev, info); + info->pdev = pdev; + + idr_init(&info->idr); + + err = device_create_file(dev, &dev_attr_status); + if (err != 0) { + dev_err(dev, "Failed to create status sysfs attribute\n"); + goto err_no_sysfs; + } + + for_each_child_of_node(pnode, cnode) { + + entry = gpio_of_entry_create(info, cnode); + if (IS_ERR_OR_NULL(entry)) { + dev_err(dev, "Failed to create gpio entry\n"); + err = PTR_ERR(entry); + goto err_fail_entry; + } + } + + dev_info(&pdev->dev, "ready\n"); + + return 0; +err_fail_entry: + device_remove_file(&pdev->dev, &dev_attr_status); +err_no_sysfs: +err_no_mem: + return err; +} + +static int gpio_of_helper_remove(struct platform_device *pdev) +{ + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + int id; + + dev_info(&pdev->dev, "removing\n"); + + device_remove_file(&pdev->dev, &dev_attr_status); + + id = 0; + idr_for_each_entry(&info->idr, entry, id) { + /* destroy each and every one */ + gpio_of_entry_destroy(entry); + } + + return 0; +} + +#ifdef CONFIG_PM +//#ifdef CONFIG_PM_RUNTIME +static int gpio_of_helper_runtime_suspend(struct device *dev) +{ + /* place holder */ + return 0; +} + +static int gpio_of_helper_runtime_resume(struct device *dev) +{ + /* place holder */ + return 0; +} +//#endif /* CONFIG_PM_RUNTIME */ + +static struct dev_pm_ops gpio_of_helper_pm_ops = { + SET_RUNTIME_PM_OPS(gpio_of_helper_runtime_suspend, + gpio_of_helper_runtime_resume, NULL) +}; +#define GPIO_OF_HELPER_PM_OPS (&gpio_of_helper_pm_ops) +#else +#define GPIO_OF_HELPER_PM_OPS NULL +#endif /* CONFIG_PM */ + +struct platform_driver gpio_of_helper_driver = { + .probe = gpio_of_helper_probe, + .remove = gpio_of_helper_remove, + .driver = { + .name = "gpio-of-helper", + .owner = THIS_MODULE, + .pm = GPIO_OF_HELPER_PM_OPS, + .of_match_table = gpio_of_helper_of_match, + }, +}; + +module_platform_driver(gpio_of_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou "); +MODULE_DESCRIPTION("GPIO OF Helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:gpio-of-helper"); diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 3f454eaf2101f..e5fa99e575484 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -36,10 +36,10 @@ static DEFINE_MUTEX(sysfs_lock); /* * /sys/class/gpio/gpioN... only for GPIOs that are exported * /direction - * * MAY BE OMITTED if kernel won't allow direction changes * * is read/write as "in" or "out" * * may also be written as "high" or "low", initializing * output value as specified ("out" implies "low") + * * read-only if kernel won't allow direction changes * /value * * always readable, subject to hardware behavior * * may be writable, as zero/nonzero @@ -52,6 +52,8 @@ static DEFINE_MUTEX(sysfs_lock); * * is read/write as zero/nonzero * * also affects existing and subsequent "falling" and "rising" * /edge configuration + * /label + * * descriptor label */ static ssize_t direction_show(struct device *dev, @@ -82,7 +84,9 @@ static ssize_t direction_store(struct device *dev, mutex_lock(&data->mutex); - if (sysfs_streq(buf, "high")) + if (!data->direction_can_change) + status = -EPERM; + else if (sysfs_streq(buf, "high")) status = gpiod_direction_output_raw(desc, 1); else if (sysfs_streq(buf, "out") || sysfs_streq(buf, "low")) status = gpiod_direction_output_raw(desc, 0); @@ -351,6 +355,23 @@ static ssize_t active_low_store(struct device *dev, } static DEVICE_ATTR_RW(active_low); +static ssize_t label_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gpiod_data *data = dev_get_drvdata(dev); + struct gpio_desc *desc = data->desc; + ssize_t status; + + mutex_lock(&data->mutex); + + status = sprintf(buf, "%s\n", desc->label); + + mutex_unlock(&data->mutex); + + return status; +} +static DEVICE_ATTR_RO(label); + static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -362,12 +383,15 @@ static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, if (attr == &dev_attr_direction.attr) { if (!show_direction) - mode = 0; + mode &= 0444; } else if (attr == &dev_attr_edge.attr) { if (gpiod_to_irq(desc) < 0) mode = 0; if (!show_direction && test_bit(FLAG_IS_OUT, &desc->flags)) mode = 0; + } else if (attr == &dev_attr_value.attr) { + if (!show_direction && !test_bit(FLAG_IS_OUT, &desc->flags)) + mode &= 0444; } return mode; @@ -378,6 +402,7 @@ static struct attribute *gpio_attrs[] = { &dev_attr_edge.attr, &dev_attr_value.attr, &dev_attr_active_low.attr, + &dev_attr_label.attr, NULL, }; @@ -391,6 +416,10 @@ static const struct attribute_group *gpio_groups[] = { NULL }; +/* bwlegh, a second device in the same file... get out of my namespace! */ +#define dev_attr_label dev_attr_chip_label +#define label_show chip_label_show + /* * /sys/class/gpio/gpiochipN/ * /base ... matching gpio_chip.base (N) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 21062cb6b85f4..90c5152d82d3e 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2033,9 +2033,9 @@ static int __gpiod_request(struct gpio_desc *desc, const char *label) if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) { desc_set_label(desc, label ? : "?"); status = 0; - } else { - status = -EBUSY; - goto done; +// } else { +// status = -EBUSY; +// goto done; } if (chip->request) { diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 4df3c48adcecd..1c7264b86ba28 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -2,7 +2,7 @@ config DRM_ETNAVIV tristate "ETNAVIV (DRM support for Vivante GPU IP cores)" depends on DRM - depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST) + depends on ARCH_MXC || ARCH_DOVE || ARCH_OMAP2PLUS || (ARM && COMPILE_TEST) depends on MMU select SHMEM select SYNC_FILE diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 2cb4773823c26..7634f5baa9e32 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -651,6 +651,7 @@ static int etnaviv_pdev_remove(struct platform_device *pdev) static const struct of_device_id dt_match[] = { { .compatible = "fsl,imx-gpu-subsystem" }, { .compatible = "marvell,dove-gpu-subsystem" }, + { .compatible = "vivante,gc-gpu-subsystem"}, {} }; MODULE_DEVICE_TABLE(of, dt_match); diff --git a/drivers/gpu/drm/i2c/Kconfig b/drivers/gpu/drm/i2c/Kconfig index a6c92beb410a7..66ff8e3f7c2e1 100644 --- a/drivers/gpu/drm/i2c/Kconfig +++ b/drivers/gpu/drm/i2c/Kconfig @@ -1,6 +1,12 @@ menu "I2C encoder or helper chips" depends on DRM && DRM_KMS_HELPER && I2C +config DRM_I2C_ADIHDMI + tristate "ADI HDMI encoder" + default m if DRM_TILCDC + help + Support for ADI HDMI encoder. + config DRM_I2C_CH7006 tristate "Chrontel ch7006 TV encoder" default m if DRM_NOUVEAU diff --git a/drivers/gpu/drm/i2c/Makefile b/drivers/gpu/drm/i2c/Makefile index b20100c18ffb6..0986f685717b9 100644 --- a/drivers/gpu/drm/i2c/Makefile +++ b/drivers/gpu/drm/i2c/Makefile @@ -7,3 +7,6 @@ obj-$(CONFIG_DRM_I2C_SIL164) += sil164.o tda998x-y := tda998x_drv.o obj-$(CONFIG_DRM_I2C_NXP_TDA998X) += tda998x.o + +adihdmi-y := adihdmi_drv.o +obj-$(CONFIG_DRM_I2C_ADIHDMI) += adihdmi.o diff --git a/drivers/gpu/drm/i2c/adihdmi.h b/drivers/gpu/drm/i2c/adihdmi.h new file mode 100644 index 0000000000000..58d9a2b274ea0 --- /dev/null +++ b/drivers/gpu/drm/i2c/adihdmi.h @@ -0,0 +1,289 @@ +/* + * Analog Devices ADIHDMI HDMI transmitter driver + * + * Copyright 2012 Analog Devices Inc. + * + * Licensed under the GPL-2. + */ + +#ifndef __DRM_I2C_ADIHDMI_H__ +#define __DRM_I2C_ADIHDMI_H__ + +#include + +#define ADIHDMI_REG_CHIP_REVISION 0x00 +#define ADIHDMI_REG_N0 0x01 +#define ADIHDMI_REG_N1 0x02 +#define ADIHDMI_REG_N2 0x03 +#define ADIHDMI_REG_SPDIF_FREQ 0x04 +#define ADIHDMI_REG_CTS_AUTOMATIC1 0x05 +#define ADIHDMI_REG_CTS_AUTOMATIC2 0x06 +#define ADIHDMI_REG_CTS_MANUAL0 0x07 +#define ADIHDMI_REG_CTS_MANUAL1 0x08 +#define ADIHDMI_REG_CTS_MANUAL2 0x09 +#define ADIHDMI_REG_AUDIO_SOURCE 0x0a +#define ADIHDMI_REG_AUDIO_CONFIG 0x0b +#define ADIHDMI_REG_I2S_CONFIG 0x0c +#define ADIHDMI_REG_I2S_WIDTH 0x0d +#define ADIHDMI_REG_AUDIO_SUB_SRC0 0x0e +#define ADIHDMI_REG_AUDIO_SUB_SRC1 0x0f +#define ADIHDMI_REG_AUDIO_SUB_SRC2 0x10 +#define ADIHDMI_REG_AUDIO_SUB_SRC3 0x11 +#define ADIHDMI_REG_AUDIO_CFG1 0x12 +#define ADIHDMI_REG_AUDIO_CFG2 0x13 +#define ADIHDMI_REG_AUDIO_CFG3 0x14 +#define ADIHDMI_REG_I2C_FREQ_ID_CFG 0x15 +#define ADIHDMI_REG_VIDEO_INPUT_CFG1 0x16 +#define ADIHDMI_REG_CSC_UPPER(x) (0x18 + (x) * 2) +#define ADIHDMI_REG_CSC_LOWER(x) (0x19 + (x) * 2) +#define ADIHDMI_REG_SYNC_DECODER(x) (0x30 + (x)) +#define ADIHDMI_REG_DE_GENERATOR (0x35 + (x)) +#define ADIHDMI_REG_PIXEL_REPETITION 0x3b +#define ADIHDMI_REG_VIC_MANUAL 0x3c +#define ADIHDMI_REG_VIC_SEND 0x3d +#define ADIHDMI_REG_VIC_DETECTED 0x3e +#define ADIHDMI_REG_AUX_VIC_DETECTED 0x3f +#define ADIHDMI_REG_PACKET_ENABLE0 0x40 +#define ADIHDMI_REG_POWER 0x41 +#define ADIHDMI_REG_STATUS 0x42 +#define ADIHDMI_REG_EDID_I2C_ADDR 0x43 +#define ADIHDMI_REG_PACKET_ENABLE1 0x44 +#define ADIHDMI_REG_PACKET_I2C_ADDR 0x45 +#define ADIHDMI_REG_DSD_ENABLE 0x46 +#define ADIHDMI_REG_VIDEO_INPUT_CFG2 0x48 +#define ADIHDMI_REG_INFOFRAME_UPDATE 0x4a +#define ADIHDMI_REG_GC(x) (0x4b + (x)) /* 0x4b - 0x51 */ +#define ADIHDMI_REG_AVI_INFOFRAME_VERSION 0x52 +#define ADIHDMI_REG_AVI_INFOFRAME_LENGTH 0x53 +#define ADIHDMI_REG_AVI_INFOFRAME_CHECKSUM 0x54 +#define ADIHDMI_REG_AVI_INFOFRAME(x) (0x55 + (x)) /* 0x55 - 0x6f */ +#define ADIHDMI_REG_AUDIO_INFOFRAME_VERSION 0x70 +#define ADIHDMI_REG_AUDIO_INFOFRAME_LENGTH 0x71 +#define ADIHDMI_REG_AUDIO_INFOFRAME_CHECKSUM 0x72 +#define ADIHDMI_REG_AUDIO_INFOFRAME(x) (0x73 + (x)) /* 0x73 - 0x7c */ +#define ADIHDMI_REG_INT_ENABLE(x) (0x94 + (x)) +#define ADIHDMI_REG_INT(x) (0x96 + (x)) +#define ADIHDMI_REG_INPUT_CLK_DIV 0x9d +#define ADIHDMI_REG_PLL_STATUS 0x9e +#define ADIHDMI_REG_HDMI_POWER 0xa1 +#define ADIHDMI_REG_HDCP_HDMI_CFG 0xaf +#define ADIHDMI_REG_AN(x) (0xb0 + (x)) /* 0xb0 - 0xb7 */ +#define ADIHDMI_REG_HDCP_STATUS 0xb8 +#define ADIHDMI_REG_BCAPS 0xbe +#define ADIHDMI_REG_BKSV(x) (0xc0 + (x)) /* 0xc0 - 0xc3 */ +#define ADIHDMI_REG_EDID_SEGMENT 0xc4 +#define ADIHDMI_REG_DDC_STATUS 0xc8 +#define ADIHDMI_REG_EDID_READ_CTRL 0xc9 +#define ADIHDMI_REG_BSTATUS(x) (0xca + (x)) /* 0xca - 0xcb */ +#define ADIHDMI_REG_TIMING_GEN_SEQ 0xd0 +#define ADIHDMI_REG_POWER2 0xd6 +#define ADIHDMI_REG_HSYNC_PLACEMENT_MSB 0xfa + +#define ADIHDMI_REG_SYNC_ADJUSTMENT(x) (0xd7 + (x)) /* 0xd7 - 0xdc */ +#define ADIHDMI_REG_TMDS_CLOCK_INV 0xde +#define ADIHDMI_REG_ARC_CTRL 0xdf +#define ADIHDMI_REG_CEC_I2C_ADDR 0xe1 +#define ADIHDMI_REG_CEC_CTRL 0xe2 +#define ADIHDMI_REG_CHIP_ID_HIGH 0xf5 +#define ADIHDMI_REG_CHIP_ID_LOW 0xf6 + +#define ADIHDMI_CSC_ENABLE BIT(7) +#define ADIHDMI_CSC_UPDATE_MODE BIT(5) + +#define ADIHDMI_INT0_HDP BIT(7) +#define ADIHDMI_INT0_VSYNC BIT(5) +#define ADIHDMI_INT0_AUDIO_FIFO_FULL BIT(4) +#define ADIHDMI_INT0_EDID_READY BIT(2) +#define ADIHDMI_INT0_HDCP_AUTHENTICATED BIT(1) + +#define ADIHDMI_INT1_DDC_ERROR BIT(7) +#define ADIHDMI_INT1_BKSV BIT(6) +#define ADIHDMI_INT1_CEC_TX_READY BIT(5) +#define ADIHDMI_INT1_CEC_TX_ARBIT_LOST BIT(4) +#define ADIHDMI_INT1_CEC_TX_RETRY_TIMEOUT BIT(3) +#define ADIHDMI_INT1_CEC_RX_READY3 BIT(2) +#define ADIHDMI_INT1_CEC_RX_READY2 BIT(1) +#define ADIHDMI_INT1_CEC_RX_READY1 BIT(0) + +#define ADIHDMI_ARC_CTRL_POWER_DOWN BIT(0) + +#define ADIHDMI_CEC_CTRL_POWER_DOWN BIT(0) + +#define ADIHDMI_POWER_POWER_DOWN BIT(6) + +#define ADIHDMI_HDMI_CFG_MODE_MASK 0x2 +#define ADIHDMI_HDMI_CFG_MODE_DVI 0x0 +#define ADIHDMI_HDMI_CFG_MODE_HDMI 0x2 + +#define ADIHDMI_AUDIO_SELECT_I2C 0x0 +#define ADIHDMI_AUDIO_SELECT_SPDIF 0x1 +#define ADIHDMI_AUDIO_SELECT_DSD 0x2 +#define ADIHDMI_AUDIO_SELECT_HBR 0x3 +#define ADIHDMI_AUDIO_SELECT_DST 0x4 + +#define ADIHDMI_I2S_SAMPLE_LEN_16 0x2 +#define ADIHDMI_I2S_SAMPLE_LEN_20 0x3 +#define ADIHDMI_I2S_SAMPLE_LEN_18 0x4 +#define ADIHDMI_I2S_SAMPLE_LEN_22 0x5 +#define ADIHDMI_I2S_SAMPLE_LEN_19 0x8 +#define ADIHDMI_I2S_SAMPLE_LEN_23 0x9 +#define ADIHDMI_I2S_SAMPLE_LEN_24 0xb +#define ADIHDMI_I2S_SAMPLE_LEN_17 0xc +#define ADIHDMI_I2S_SAMPLE_LEN_21 0xd + +#define ADIHDMI_SAMPLE_FREQ_44100 0x0 +#define ADIHDMI_SAMPLE_FREQ_48000 0x2 +#define ADIHDMI_SAMPLE_FREQ_32000 0x3 +#define ADIHDMI_SAMPLE_FREQ_88200 0x8 +#define ADIHDMI_SAMPLE_FREQ_96000 0xa +#define ADIHDMI_SAMPLE_FREQ_176400 0xc +#define ADIHDMI_SAMPLE_FREQ_192000 0xe + +#define ADIHDMI_STATUS_POWER_DOWN_POLARITY BIT(7) +#define ADIHDMI_STATUS_HPD BIT(6) +#define ADIHDMI_STATUS_MONITOR_SENSE BIT(5) +#define ADIHDMI_STATUS_I2S_32BIT_MODE BIT(3) + +#define ADIHDMI_PACKET_ENABLE_N_CTS BIT(8+6) +#define ADIHDMI_PACKET_ENABLE_AUDIO_SAMPLE BIT(8+5) +#define ADIHDMI_PACKET_ENABLE_AVI_INFOFRAME BIT(8+4) +#define ADIHDMI_PACKET_ENABLE_AUDIO_INFOFRAME BIT(8+3) +#define ADIHDMI_PACKET_ENABLE_GC BIT(7) +#define ADIHDMI_PACKET_ENABLE_SPD BIT(6) +#define ADIHDMI_PACKET_ENABLE_MPEG BIT(5) +#define ADIHDMI_PACKET_ENABLE_ACP BIT(4) +#define ADIHDMI_PACKET_ENABLE_ISRC BIT(3) +#define ADIHDMI_PACKET_ENABLE_GM BIT(2) +#define ADIHDMI_PACKET_ENABLE_SPARE2 BIT(1) +#define ADIHDMI_PACKET_ENABLE_SPARE1 BIT(0) + +#define ADIHDMI_REG_POWER2_HDP_SRC_MASK 0xc0 +#define ADIHDMI_REG_POWER2_HDP_SRC_BOTH 0x00 +#define ADIHDMI_REG_POWER2_HDP_SRC_HDP 0x40 +#define ADIHDMI_REG_POWER2_HDP_SRC_CEC 0x80 +#define ADIHDMI_REG_POWER2_HDP_SRC_NONE 0xc0 +#define ADIHDMI_REG_POWER2_TDMS_ENABLE BIT(4) +#define ADIHDMI_REG_POWER2_GATE_INPUT_CLK BIT(0) + +#define ADIHDMI_LOW_REFRESH_RATE_NONE 0x0 +#define ADIHDMI_LOW_REFRESH_RATE_24HZ 0x1 +#define ADIHDMI_LOW_REFRESH_RATE_25HZ 0x2 +#define ADIHDMI_LOW_REFRESH_RATE_30HZ 0x3 + +#define ADIHDMI_AUDIO_CFG3_LEN_MASK 0x0f +#define ADIHDMI_I2C_FREQ_ID_CFG_RATE_MASK 0xf0 + +#define ADIHDMI_AUDIO_SOURCE_I2S 0 +#define ADIHDMI_AUDIO_SOURCE_SPDIF 1 + +#define ADIHDMI_I2S_FORMAT_I2S 0 +#define ADIHDMI_I2S_FORMAT_RIGHT_J 1 +#define ADIHDMI_I2S_FORMAT_LEFT_J 2 + +#define ADIHDMI_PACKET(p, x) ((p) * 0x20 + (x)) +#define ADIHDMI_PACKET_SDP(x) ADIHDMI_PACKET(0, x) +#define ADIHDMI_PACKET_MPEG(x) ADIHDMI_PACKET(1, x) +#define ADIHDMI_PACKET_ACP(x) ADIHDMI_PACKET(2, x) +#define ADIHDMI_PACKET_ISRC1(x) ADIHDMI_PACKET(3, x) +#define ADIHDMI_PACKET_ISRC2(x) ADIHDMI_PACKET(4, x) +#define ADIHDMI_PACKET_GM(x) ADIHDMI_PACKET(5, x) +#define ADIHDMI_PACKET_SPARE(x) ADIHDMI_PACKET(6, x) + +enum adihdmi_input_clock { + ADIHDMI_INPUT_CLOCK_1X, + ADIHDMI_INPUT_CLOCK_2X, + ADIHDMI_INPUT_CLOCK_DDR, +}; + +enum adihdmi_input_justification { + ADIHDMI_INPUT_JUSTIFICATION_EVENLY = 0, + ADIHDMI_INPUT_JUSTIFICATION_RIGHT = 1, + ADIHDMI_INPUT_JUSTIFICATION_LEFT = 2, +}; + +enum adihdmi_input_sync_pulse { + ADIHDMI_INPUT_SYNC_PULSE_DE = 0, + ADIHDMI_INPUT_SYNC_PULSE_HSYNC = 1, + ADIHDMI_INPUT_SYNC_PULSE_VSYNC = 2, + ADIHDMI_INPUT_SYNC_PULSE_NONE = 3, +}; + +/** + * enum adihdmi_sync_polarity - Polarity for the input sync signals + * @ADIHDMI_SYNC_POLARITY_PASSTHROUGH: Sync polarity matches that of + * the currently configured mode. + * @ADIHDMI_SYNC_POLARITY_LOW: Sync polarity is low + * @ADIHDMI_SYNC_POLARITY_HIGH: Sync polarity is high + * + * If the polarity is set to either LOW or HIGH the driver will configure the + * ADIHDMI to internally invert the sync signal if required to match the sync + * polarity setting for the currently selected output mode. + * + * If the polarity is set to PASSTHROUGH, the ADIHDMI will route the signal + * unchanged. This is used when the upstream graphics core already generates + * the sync signals with the correct polarity. + */ +enum adihdmi_sync_polarity { + ADIHDMI_SYNC_POLARITY_PASSTHROUGH, + ADIHDMI_SYNC_POLARITY_LOW, + ADIHDMI_SYNC_POLARITY_HIGH, +}; + +/** + * struct adihdmi_link_config - Describes adihdmi hardware configuration + * @input_color_depth: Number of bits per color component (8, 10 or 12) + * @input_colorspace: The input colorspace (RGB, YUV444, YUV422) + * @input_clock: The input video clock style (1x, 2x, DDR) + * @input_style: The input component arrangement variant + * @input_justification: Video input format bit justification + * @clock_delay: Clock delay for the input clock (in ps) + * @embedded_sync: Video input uses BT.656-style embedded sync + * @sync_pulse: Select the sync pulse + * @vsync_polarity: vsync input signal configuration + * @hsync_polarity: hsync input signal configuration + */ +struct adihdmi_link_config { + unsigned int input_color_depth; + enum hdmi_colorspace input_colorspace; + enum adihdmi_input_clock input_clock; + unsigned int input_style; + enum adihdmi_input_justification input_justification; + + int clock_delay; + + bool embedded_sync; + enum adihdmi_input_sync_pulse sync_pulse; + enum adihdmi_sync_polarity vsync_polarity; + enum adihdmi_sync_polarity hsync_polarity; +}; + +/** + * enum adihdmi_csc_scaling - Scaling factor for the ADIHDMI CSC + * @ADIHDMI_CSC_SCALING_1: CSC results are not scaled + * @ADIHDMI_CSC_SCALING_2: CSC results are scaled by a factor of two + * @ADIHDMI_CSC_SCALING_4: CSC results are scalled by a factor of four + */ +enum adihdmi_csc_scaling { + ADIHDMI_CSC_SCALING_1 = 0, + ADIHDMI_CSC_SCALING_2 = 1, + ADIHDMI_CSC_SCALING_4 = 2, +}; + +/** + * struct adihdmi_video_config - Describes adihdmi hardware configuration + * @csc_enable: Whether to enable color space conversion + * @csc_scaling_factor: Color space conversion scaling factor + * @csc_coefficents: Color space conversion coefficents + * @hdmi_mode: Whether to use HDMI or DVI output mode + * @avi_infoframe: HDMI infoframe + */ +struct adihdmi_video_config { + bool csc_enable; + enum adihdmi_csc_scaling csc_scaling_factor; + const uint16_t *csc_coefficents; + + bool hdmi_mode; + struct hdmi_avi_infoframe avi_infoframe; +}; + +#endif /* __DRM_I2C_ADIHDMI_H__ */ diff --git a/drivers/gpu/drm/i2c/adihdmi_drv.c b/drivers/gpu/drm/i2c/adihdmi_drv.c new file mode 100644 index 0000000000000..6ab4682be4b9d --- /dev/null +++ b/drivers/gpu/drm/i2c/adihdmi_drv.c @@ -0,0 +1,1268 @@ +/* + * Analog Devices ADIHDMI HDMI transmitter driver + * + * Copyright 2012 Analog Devices Inc. + * Copyright 2015 Konsulko Group + * + * Licensed under the GPL-2. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "adihdmi.h" + +#define ADIHDMI_INFOFRAME_PACKETS (0x7900) + +struct adihdmi { + struct i2c_client *i2c_main; + struct i2c_client *i2c_edid; + + struct regmap *regmap; + struct regmap *packet_memory_regmap; + enum drm_connector_status status; + bool powered; + + unsigned int f_tmds; + + unsigned int current_edid_segment; + uint8_t edid_buf[256]; + bool edid_read; + + wait_queue_head_t wq; + struct drm_encoder *encoder; + + bool embedded_sync; + enum adihdmi_sync_polarity vsync_polarity; + enum adihdmi_sync_polarity hsync_polarity; + bool rgb; + + struct edid *edid; + + struct gpio_desc *gpio_pd; +}; + +struct adihdmi2 { + struct adihdmi base; + struct drm_encoder encoder; + struct drm_connector connector; +}; + +/* ADI recommended values for proper operation. */ +static const struct reg_sequence adihdmi_fixed_registers[] = { + { 0x98, 0x03 }, + { 0x9a, 0xe0 }, + { 0x9c, 0x30 }, + { 0x9d, 0x61 }, + { 0xa2, 0xa4 }, + { 0xa3, 0xa4 }, + { 0xe0, 0xd0 }, + { 0xf9, 0x00 }, + { 0x55, 0x02 }, +}; + +/* ----------------------------------------------------------------------------- + * Register access + */ + +static const uint8_t adihdmi_register_defaults[] = { + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00 */ + 0x00, 0x00, 0x01, 0x0e, 0xbc, 0x18, 0x01, 0x13, + 0x25, 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10 */ + 0x46, 0x62, 0x04, 0xa8, 0x00, 0x00, 0x1c, 0x84, + 0x1c, 0xbf, 0x04, 0xa8, 0x1e, 0x70, 0x02, 0x1e, /* 20 */ + 0x00, 0x00, 0x04, 0xa8, 0x08, 0x12, 0x1b, 0xac, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 30 */ + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0xb0, + 0x00, 0x50, 0x90, 0x7e, 0x79, 0x70, 0x00, 0x00, /* 40 */ + 0x00, 0xa8, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x0d, 0x00, 0x00, 0x00, 0x00, /* 50 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 60 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 70 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 90 */ + 0x0b, 0x02, 0x00, 0x18, 0x5a, 0x60, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x80, 0x08, 0x04, 0x00, 0x00, /* a0 */ + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x40, 0x14, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0 */ + 0x00, 0x03, 0x00, 0x00, 0x02, 0x00, 0x01, 0x04, + 0x30, 0xff, 0x80, 0x80, 0x80, 0x00, 0x00, 0x00, /* d0 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x01, + 0x80, 0x75, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, /* e0 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x11, 0x00, /* f0 */ + 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static bool adihdmi_register_volatile(struct device *dev, unsigned int reg) +{ + switch (reg) { + case ADIHDMI_REG_CHIP_REVISION: + case ADIHDMI_REG_SPDIF_FREQ: + case ADIHDMI_REG_CTS_AUTOMATIC1: + case ADIHDMI_REG_CTS_AUTOMATIC2: + case ADIHDMI_REG_VIC_DETECTED: + case ADIHDMI_REG_VIC_SEND: + case ADIHDMI_REG_AUX_VIC_DETECTED: + case ADIHDMI_REG_STATUS: + case ADIHDMI_REG_GC(1): + case ADIHDMI_REG_INT(0): + case ADIHDMI_REG_INT(1): + case ADIHDMI_REG_PLL_STATUS: + case ADIHDMI_REG_AN(0): + case ADIHDMI_REG_AN(1): + case ADIHDMI_REG_AN(2): + case ADIHDMI_REG_AN(3): + case ADIHDMI_REG_AN(4): + case ADIHDMI_REG_AN(5): + case ADIHDMI_REG_AN(6): + case ADIHDMI_REG_AN(7): + case ADIHDMI_REG_HDCP_STATUS: + case ADIHDMI_REG_BCAPS: + case ADIHDMI_REG_BKSV(0): + case ADIHDMI_REG_BKSV(1): + case ADIHDMI_REG_BKSV(2): + case ADIHDMI_REG_BKSV(3): + case ADIHDMI_REG_BKSV(4): + case ADIHDMI_REG_DDC_STATUS: + case ADIHDMI_REG_BSTATUS(0): + case ADIHDMI_REG_BSTATUS(1): + case ADIHDMI_REG_CHIP_ID_HIGH: + case ADIHDMI_REG_CHIP_ID_LOW: + return true; + } + + return false; +} + +static const struct regmap_config adihdmi_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = 0xff, + .cache_type = REGCACHE_RBTREE, + .reg_defaults_raw = adihdmi_register_defaults, + .num_reg_defaults_raw = ARRAY_SIZE(adihdmi_register_defaults), + + .volatile_reg = adihdmi_register_volatile, +}; + +/* ----------------------------------------------------------------------------- + * Hardware configuration + */ + + static void adihdmi_audio_setup(struct adihdmi * adihdmi) +{ + /* Select I2S. */ + regmap_write(adihdmi->regmap, ADIHDMI_REG_AUDIO_SOURCE, 0x01); + regmap_write(adihdmi->regmap, ADIHDMI_REG_I2S_CONFIG, 0x84); + + /* Setup clocks for 48KHz. */ + regmap_write(adihdmi->regmap, ADIHDMI_REG_N0, 0x00); + regmap_write(adihdmi->regmap, ADIHDMI_REG_N1, 0x18); + regmap_write(adihdmi->regmap, ADIHDMI_REG_N2, 0x00); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_I2C_FREQ_ID_CFG, 0xF0, 0x20); + + /* Set audio word length to 24 bits. */ + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_AUDIO_CFG3, 0x0F, 0x0B); + + /* Update audio infoframe. */ + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_INFOFRAME_UPDATE, 0x20, 0x20); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_AUDIO_INFOFRAME(0), 0x07, 0x01); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_AUDIO_INFOFRAME(3), 0x1F, 0x00); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_INFOFRAME_UPDATE, 0x20, 0x00); +} + +static void adihdmi_set_colormap(struct adihdmi *adihdmi, bool enable, + const uint16_t *coeff, + unsigned int scaling_factor) +{ + unsigned int i; + + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_CSC_UPPER(1), + ADIHDMI_CSC_UPDATE_MODE, ADIHDMI_CSC_UPDATE_MODE); + + if (enable) { + for (i = 0; i < 12; ++i) { + regmap_update_bits(adihdmi->regmap, + ADIHDMI_REG_CSC_UPPER(i), + 0x1f, coeff[i] >> 8); + regmap_write(adihdmi->regmap, + ADIHDMI_REG_CSC_LOWER(i), + coeff[i] & 0xff); + } + } + + if (enable) + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_CSC_UPPER(0), + 0xe0, 0x80 | (scaling_factor << 5)); + else + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_CSC_UPPER(0), + 0x80, 0x00); + + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_CSC_UPPER(1), + ADIHDMI_CSC_UPDATE_MODE, 0); +} + +static int adihdmi_packet_enable(struct adihdmi *adihdmi, unsigned int packet) +{ + if (packet & 0xff) + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_PACKET_ENABLE0, + packet, 0xff); + + if (packet & 0xff00) { + packet >>= 8; + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_PACKET_ENABLE1, + packet, 0xff); + } + + return 0; +} + +static int adihdmi_packet_disable(struct adihdmi *adihdmi, unsigned int packet) +{ + if (packet & 0xff) + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_PACKET_ENABLE0, + packet, 0x00); + + if (packet & 0xff00) { + packet >>= 8; + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_PACKET_ENABLE1, + packet, 0x00); + } + + return 0; +} + +/* Coefficients for adihdmi color space conversion */ +static const uint16_t adihdmi_csc_ycbcr_to_rgb[] = { + 0x0734, 0x04ad, 0x0000, 0x1c1b, + 0x1ddc, 0x04ad, 0x1f24, 0x0135, + 0x0000, 0x04ad, 0x087c, 0x1b77, +}; + +static void adihdmi_set_config_csc(struct adihdmi *adihdmi, + struct drm_connector *connector, + bool rgb) +{ + struct adihdmi_video_config config; + bool output_format_422, output_format_ycbcr; + unsigned int mode; + uint8_t infoframe[17]; + + if (adihdmi->edid) + config.hdmi_mode = drm_detect_hdmi_monitor(adihdmi->edid); + else + config.hdmi_mode = false; + + hdmi_avi_infoframe_init(&config.avi_infoframe); + + config.avi_infoframe.scan_mode = HDMI_SCAN_MODE_UNDERSCAN; + + if (rgb) { + config.csc_enable = false; + config.avi_infoframe.colorspace = HDMI_COLORSPACE_RGB; + } else { + config.csc_scaling_factor = ADIHDMI_CSC_SCALING_4; + config.csc_coefficents = adihdmi_csc_ycbcr_to_rgb; + + if ((connector->display_info.color_formats & + DRM_COLOR_FORMAT_YCRCB422) && + config.hdmi_mode) { + config.csc_enable = false; + config.avi_infoframe.colorspace = + HDMI_COLORSPACE_YUV422; + } else { + config.csc_enable = true; + config.avi_infoframe.colorspace = HDMI_COLORSPACE_RGB; + } + } + + if (config.hdmi_mode) { + mode = ADIHDMI_HDMI_CFG_MODE_HDMI; + + switch (config.avi_infoframe.colorspace) { + case HDMI_COLORSPACE_YUV444: + output_format_422 = false; + output_format_ycbcr = true; + break; + case HDMI_COLORSPACE_YUV422: + output_format_422 = true; + output_format_ycbcr = true; + break; + default: + output_format_422 = false; + output_format_ycbcr = false; + break; + } + } else { + mode = ADIHDMI_HDMI_CFG_MODE_DVI; + output_format_422 = false; + output_format_ycbcr = false; + } + + adihdmi_packet_disable(adihdmi, ADIHDMI_INFOFRAME_PACKETS); + + adihdmi_set_colormap(adihdmi, config.csc_enable, + config.csc_coefficents, + config.csc_scaling_factor); + + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_VIDEO_INPUT_CFG1, 0x81, + (output_format_422 << 7) | output_format_ycbcr); + + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_HDCP_HDMI_CFG, + ADIHDMI_HDMI_CFG_MODE_MASK, mode); + + hdmi_avi_infoframe_pack(&config.avi_infoframe, infoframe, + sizeof(infoframe)); + + /* The AVI infoframe id is not configurable */ + regmap_bulk_write(adihdmi->regmap, ADIHDMI_REG_AVI_INFOFRAME_VERSION, + infoframe + 1, sizeof(infoframe) - 1); + + adihdmi_packet_enable(adihdmi, ADIHDMI_INFOFRAME_PACKETS); +} + +static void adihdmi_set_link_config(struct adihdmi *adihdmi, + const struct adihdmi_link_config *config) +{ + /* + * The input style values documented in the datasheet don't match the + * hardware register field values :-( + */ + static const unsigned int input_styles[4] = { 0, 2, 1, 3 }; + + unsigned int clock_delay; + unsigned int color_depth; + unsigned int input_id; + + clock_delay = (config->clock_delay + 1200) / 400; + color_depth = config->input_color_depth == 8 ? 3 + : (config->input_color_depth == 10 ? 1 : 2); + + /* TODO Support input ID 6 */ + if (config->input_colorspace != HDMI_COLORSPACE_YUV422) + input_id = config->input_clock == ADIHDMI_INPUT_CLOCK_DDR + ? 5 : 0; + else if (config->input_clock == ADIHDMI_INPUT_CLOCK_DDR) + input_id = config->embedded_sync ? 8 : 7; + else if (config->input_clock == ADIHDMI_INPUT_CLOCK_2X) + input_id = config->embedded_sync ? 4 : 3; + else + input_id = config->embedded_sync ? 2 : 1; + + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_I2C_FREQ_ID_CFG, 0xf, + input_id); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_VIDEO_INPUT_CFG1, 0x7e, + (color_depth << 4) | + (input_styles[config->input_style] << 2)); + regmap_write(adihdmi->regmap, ADIHDMI_REG_VIDEO_INPUT_CFG2, + config->input_justification << 3); + regmap_write(adihdmi->regmap, ADIHDMI_REG_TIMING_GEN_SEQ, + config->sync_pulse << 2); + + regmap_write(adihdmi->regmap, 0xba, clock_delay << 5); + + adihdmi->embedded_sync = config->embedded_sync; + adihdmi->hsync_polarity = config->hsync_polarity; + adihdmi->vsync_polarity = config->vsync_polarity; + adihdmi->rgb = config->input_colorspace == HDMI_COLORSPACE_RGB; +} + +static void adihdmi_power_on(struct adihdmi *adihdmi) +{ + adihdmi->current_edid_segment = -1; + + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(0), + ADIHDMI_INT0_EDID_READY); + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(1), + ADIHDMI_INT1_DDC_ERROR); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER, + ADIHDMI_POWER_POWER_DOWN, 0); + + /* + * Per spec it is allowed to pulse the HDP signal to indicate that the + * EDID information has changed. Some monitors do this when they wakeup + * from standby or are enabled. When the HDP goes low the adihdmi is + * reset and the outputs are disabled which might cause the monitor to + * go to standby again. To avoid this we ignore the HDP pin for the + * first few seconds after enabling the output. + */ + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER2, + ADIHDMI_REG_POWER2_HDP_SRC_MASK, + ADIHDMI_REG_POWER2_HDP_SRC_NONE); + + /* + * Most of the registers are reset during power down or when HPD is low. + */ + regcache_sync(adihdmi->regmap); + + adihdmi->powered = true; +} + +static void adihdmi_power_off(struct adihdmi *adihdmi) +{ + /* TODO: setup additional power down modes */ + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER, + ADIHDMI_POWER_POWER_DOWN, + ADIHDMI_POWER_POWER_DOWN); + regcache_mark_dirty(adihdmi->regmap); + + adihdmi->powered = false; +} + +/* ----------------------------------------------------------------------------- + * Interrupt and hotplug detection + */ + +static bool adihdmi_hpd(struct adihdmi *adihdmi) +{ + unsigned int irq0; + int ret; + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_INT(0), &irq0); + if (ret < 0) + return false; + + if (irq0 & ADIHDMI_INT0_HDP) { + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(0), + ADIHDMI_INT0_HDP); + return true; + } + + return false; +} + +static int adihdmi_irq_process(struct adihdmi *adihdmi) +{ + unsigned int irq0, irq1; + int ret; + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_INT(0), &irq0); + if (ret < 0) + return ret; + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_INT(1), &irq1); + if (ret < 0) + return ret; + + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(0), irq0); + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(1), irq1); + + if (irq0 & ADIHDMI_INT0_HDP) + drm_helper_hpd_irq_event(adihdmi->encoder->dev); + + if (irq0 & ADIHDMI_INT0_EDID_READY || irq1 & ADIHDMI_INT1_DDC_ERROR) { + adihdmi->edid_read = true; + + if (adihdmi->i2c_main->irq) + wake_up_all(&adihdmi->wq); + } + + return 0; +} + +static irqreturn_t adihdmi_irq_handler(int irq, void *devid) +{ + struct adihdmi *adihdmi = devid; + int ret; + + ret = adihdmi_irq_process(adihdmi); + return ret < 0 ? IRQ_NONE : IRQ_HANDLED; +} + +/* ----------------------------------------------------------------------------- + * EDID retrieval + */ + +static int adihdmi_wait_for_edid(struct adihdmi *adihdmi, int timeout) +{ + int ret; + + if (adihdmi->i2c_main->irq) { + ret = wait_event_interruptible_timeout(adihdmi->wq, + adihdmi->edid_read, msecs_to_jiffies(timeout)); + } else { + for (; timeout > 0; timeout -= 25) { + ret = adihdmi_irq_process(adihdmi); + if (ret < 0) + break; + + if (adihdmi->edid_read) + break; + + msleep(25); + } + } + + return adihdmi->edid_read ? 0 : -EIO; +} + +static int adihdmi_get_edid_block(void *data, u8 *buf, unsigned int block, + size_t len) +{ + struct adihdmi *adihdmi = data; + struct i2c_msg xfer[2]; + uint8_t offset; + unsigned int i; + int ret; + + if (len > 128) + return -EINVAL; + + if (adihdmi->current_edid_segment != block / 2) { + unsigned int status; + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_DDC_STATUS, + &status); + if (ret < 0) + return ret; + + if (status != 2) { + adihdmi->edid_read = false; + regmap_write(adihdmi->regmap, ADIHDMI_REG_EDID_SEGMENT, + block); + ret = adihdmi_wait_for_edid(adihdmi, 200); + if (ret < 0) + return ret; + } + + /* Break this apart, hopefully more I2C controllers will + * support 64 byte transfers than 256 byte transfers + */ + + xfer[0].addr = adihdmi->i2c_edid->addr; + xfer[0].flags = 0; + xfer[0].len = 1; + xfer[0].buf = &offset; + xfer[1].addr = adihdmi->i2c_edid->addr; + xfer[1].flags = I2C_M_RD; + xfer[1].len = 64; + xfer[1].buf = adihdmi->edid_buf; + + offset = 0; + + for (i = 0; i < 4; ++i) { + ret = i2c_transfer(adihdmi->i2c_edid->adapter, xfer, + ARRAY_SIZE(xfer)); + if (ret < 0) + return ret; + else if (ret != 2) + return -EIO; + + xfer[1].buf += 64; + offset += 64; + } + + adihdmi->current_edid_segment = block / 2; + } + + if (block % 2 == 0) + memcpy(buf, adihdmi->edid_buf, len); + else + memcpy(buf, adihdmi->edid_buf + 128, len); + + return 0; +} + +static int adihdmi_mode_valid(struct drm_display_mode *mode) +{ + if (mode->clock > 165000) + return MODE_CLOCK_HIGH; + + return MODE_OK; +} + +/* ----------------------------------------------------------------------------- + * DT and private structure operations + */ + +#define conn_to_adihdmi2(x) \ + container_of(x, struct adihdmi2, connector); + +#define enc_to_adihdmi2(x) \ + container_of(x, struct adihdmi2, encoder); + +#define enc_to_adihdmi(x) \ + (&(container_of(x, struct adihdmi2, encoder)->base)) + +static int adihdmi_parse_dt(struct device_node *np, + struct adihdmi_link_config *config) +{ + memset(config, 0, sizeof(*config)); + + config->input_color_depth = 8; + + config->input_colorspace = HDMI_COLORSPACE_RGB; + //config->input_colorspace = HDMI_COLORSPACE_YUV422; + //config->input_colorspace = HDMI_COLORSPACE_YUV444; + + config->input_clock = ADIHDMI_INPUT_CLOCK_1X; + //config->input_clock = ADIHDMI_INPUT_CLOCK_2X; + //config->input_clock = ADIHDMI_INPUT_CLOCK_DDR; + + if (config->input_colorspace == HDMI_COLORSPACE_YUV422 || + config->input_clock != ADIHDMI_INPUT_CLOCK_1X) { + + config->input_style = 1; + //config->input_justification = ADIHDMI_INPUT_JUSTIFICATION_LEFT; + config->input_justification = ADIHDMI_INPUT_JUSTIFICATION_EVENLY; + //config->input_justification = ADIHDMI_INPUT_JUSTIFICATION_RIGHT; + + } else { + config->input_style = 1; + config->input_justification = ADIHDMI_INPUT_JUSTIFICATION_LEFT; + } + + config->clock_delay = 0; + config->embedded_sync = 0; + + /* Hardcode the sync pulse configurations for now. */ + config->sync_pulse = ADIHDMI_INPUT_SYNC_PULSE_NONE; + config->vsync_polarity = ADIHDMI_SYNC_POLARITY_PASSTHROUGH; + config->hsync_polarity = ADIHDMI_SYNC_POLARITY_PASSTHROUGH; + + return 0; +} + +static const int edid_i2c_addr = 0x7e; +static const int packet_i2c_addr = 0x70; +static const int cec_i2c_addr = 0x78; + +static int adihdmi_create(struct i2c_client *i2c, struct adihdmi *adihdmi) +{ + struct adihdmi_link_config link_config; + struct device *dev = &i2c->dev; + unsigned int val; + int ret; + + adihdmi->powered = false; + adihdmi->status = connector_status_disconnected; + + ret = adihdmi_parse_dt(NULL, &link_config); + if (ret) + { + pr_err("%s - %d - Bad parse\n", __FUNCTION__, __LINE__); + return -EINVAL; + } + + /* + * The power down GPIO is optional. If present, toggle it from active to + * inactive to wake up the encoder. + */ + adihdmi->gpio_pd = devm_gpiod_get_optional(dev, "pd", GPIOD_OUT_HIGH); + if (IS_ERR(adihdmi->gpio_pd)) + { + pr_err("%s - %d - Bad PD GPIO\n", __FUNCTION__, __LINE__); + return PTR_ERR(adihdmi->gpio_pd); + } + + if (adihdmi->gpio_pd) { + mdelay(5); + gpiod_set_value_cansleep(adihdmi->gpio_pd, 0); + } + + adihdmi->regmap = devm_regmap_init_i2c(i2c, &adihdmi_regmap_config); + if (IS_ERR(adihdmi->regmap)) + { + pr_err("%s - %d - Bad reg map init\n", __FUNCTION__, __LINE__); + return PTR_ERR(adihdmi->regmap); + } + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_CHIP_REVISION, &val); + if (ret) + { + pr_err("%s - %d - Bad reg map read\n", __FUNCTION__, __LINE__); + return ret; + } + dev_dbg(dev, "Rev. %d\n", val); + + ret = regmap_register_patch(adihdmi->regmap, adihdmi_fixed_registers, + ARRAY_SIZE(adihdmi_fixed_registers)); + if (ret) + { + pr_err("%s - %d - Bad reg map patch\n", __FUNCTION__, __LINE__); + return ret; + } + + regmap_write(adihdmi->regmap, ADIHDMI_REG_EDID_I2C_ADDR, edid_i2c_addr); + regmap_write(adihdmi->regmap, ADIHDMI_REG_PACKET_I2C_ADDR, + packet_i2c_addr); + regmap_write(adihdmi->regmap, ADIHDMI_REG_CEC_I2C_ADDR, cec_i2c_addr); + adihdmi_packet_disable(adihdmi, 0xffff); + + adihdmi->i2c_main = i2c; + adihdmi->i2c_edid = i2c_new_dummy(i2c->adapter, edid_i2c_addr >> 1); + if (!adihdmi->i2c_edid) + { + pr_err("%s - %d - No mem for EDID\n", __FUNCTION__, __LINE__); + return -ENOMEM; + } + + if (i2c->irq) { + init_waitqueue_head(&adihdmi->wq); + + ret = devm_request_threaded_irq(dev, i2c->irq, NULL, + adihdmi_irq_handler, + IRQF_ONESHOT, dev_name(dev), + adihdmi); + if (ret) + { + pr_err("%s - %d - Bad IRQ thread request\n", __FUNCTION__, __LINE__); + goto err_i2c_unregister_device; + } + } + + /* CEC is unused for now */ + regmap_write(adihdmi->regmap, ADIHDMI_REG_CEC_CTRL, + ADIHDMI_CEC_CTRL_POWER_DOWN); + + adihdmi_power_off(adihdmi); + + adihdmi_set_link_config(adihdmi, &link_config); + + adihdmi_audio_setup(adihdmi); + + return 0; + +err_i2c_unregister_device: + i2c_unregister_device(adihdmi->i2c_edid); + + return ret; +} + +static void adihdmi_destroy(struct adihdmi *priv) +{ + i2c_unregister_device(priv->i2c_edid); +} + +/* ----------------------------------------------------------------------------- + * Encoder operations + */ + +static int adihdmi_encoder_get_modes(struct adihdmi *adihdmi, + struct drm_connector *connector) +{ + struct edid *edid; + unsigned int count; + + /* Reading the EDID only works if the device is powered */ + if (!adihdmi->powered) { + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(0), + ADIHDMI_INT0_EDID_READY); + regmap_write(adihdmi->regmap, ADIHDMI_REG_INT(1), + ADIHDMI_INT1_DDC_ERROR); + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER, + ADIHDMI_POWER_POWER_DOWN, 0); + adihdmi->current_edid_segment = -1; + } + + edid = drm_do_get_edid(connector, adihdmi_get_edid_block, adihdmi); + + if (!adihdmi->powered) + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER, + ADIHDMI_POWER_POWER_DOWN, + ADIHDMI_POWER_POWER_DOWN); + + kfree(adihdmi->edid); + adihdmi->edid = edid; + if (!edid) + { + pr_err("%s - %d - No EDID\n", __FUNCTION__, __LINE__); + return 0; + } + + drm_mode_connector_update_edid_property(connector, edid); + count = drm_add_edid_modes(connector, edid); + + adihdmi_set_config_csc(adihdmi, connector, adihdmi->rgb); + + return count; +} + +static void adihdmi_encoder_dpms(struct drm_encoder *encoder, int mode) +{ + struct adihdmi2 *priv2 = enc_to_adihdmi2(encoder); + + if (mode == DRM_MODE_DPMS_ON) + adihdmi_power_on(&priv2->base); + else + adihdmi_power_off(&priv2->base); +} + + static enum drm_connector_status +adihdmi_encoder_detect(struct adihdmi *adihdmi, + struct drm_connector *connector) +{ + enum drm_connector_status status; + unsigned int val; + bool hpd; + int ret; + + ret = regmap_read(adihdmi->regmap, ADIHDMI_REG_STATUS, &val); + if (ret < 0) + { + pr_err("%s - %d - Disconnected\n", __FUNCTION__, __LINE__); + return connector_status_disconnected; + } + + if (val & ADIHDMI_STATUS_HPD) + status = connector_status_connected; + else + status = connector_status_disconnected; + + hpd = adihdmi_hpd(adihdmi); + + /* The chip resets itself when the cable is disconnected, so in case + * there is a pending HPD interrupt and the cable is connected there was + * at least one transition from disconnected to connected and the chip + * has to be reinitialized. */ + if (status == connector_status_connected && hpd && adihdmi->powered) { + regcache_mark_dirty(adihdmi->regmap); + adihdmi_power_on(adihdmi); + adihdmi_encoder_get_modes(adihdmi, connector); + if (adihdmi->status == connector_status_connected) + status = connector_status_disconnected; + } else { + /* Renable HDP sensing */ + regmap_update_bits(adihdmi->regmap, ADIHDMI_REG_POWER2, + ADIHDMI_REG_POWER2_HDP_SRC_MASK, + ADIHDMI_REG_POWER2_HDP_SRC_BOTH); + } + + adihdmi->status = status; + return status; +} + +static bool adihdmi_encoder_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) +{ + return true; +} + +static int adihdmi_encoder_mode_valid(struct drm_encoder *encoder, struct drm_display_mode *mode) +{ + return adihdmi_mode_valid(mode); +} + +static void adihdmi_encoder_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode, + struct drm_display_mode *adj_mode) +{ + unsigned int low_refresh_rate; + unsigned int hsync_polarity = 0; + unsigned int vsync_polarity = 0; + struct adihdmi *adihdmi = enc_to_adihdmi(encoder); + + if (adihdmi->embedded_sync) { + unsigned int hsync_offset, hsync_len; + unsigned int vsync_offset, vsync_len; + + hsync_offset = adj_mode->crtc_hsync_start - + adj_mode->crtc_hdisplay; + vsync_offset = adj_mode->crtc_vsync_start - + adj_mode->crtc_vdisplay; + hsync_len = adj_mode->crtc_hsync_end - + adj_mode->crtc_hsync_start; + vsync_len = adj_mode->crtc_vsync_end - + adj_mode->crtc_vsync_start; + + /* The hardware vsync generator has a off-by-one bug */ + vsync_offset += 1; + + regmap_write(adihdmi->regmap, ADIHDMI_REG_HSYNC_PLACEMENT_MSB, + ((hsync_offset >> 10) & 0x7) << 5); + regmap_write(adihdmi->regmap, ADIHDMI_REG_SYNC_DECODER(0), + (hsync_offset >> 2) & 0xff); + regmap_write(adihdmi->regmap, ADIHDMI_REG_SYNC_DECODER(1), + ((hsync_offset & 0x3) << 6) | + ((hsync_len >> 4) & 0x3f)); + regmap_write(adihdmi->regmap, ADIHDMI_REG_SYNC_DECODER(2), + ((hsync_len & 0xf) << 4) | + ((vsync_offset >> 6) & 0xf)); + regmap_write(adihdmi->regmap, ADIHDMI_REG_SYNC_DECODER(3), + ((vsync_offset & 0x3f) << 2) | + ((vsync_len >> 8) & 0x3)); + regmap_write(adihdmi->regmap, ADIHDMI_REG_SYNC_DECODER(4), + vsync_len & 0xff); + + hsync_polarity = !(adj_mode->flags & DRM_MODE_FLAG_PHSYNC); + vsync_polarity = !(adj_mode->flags & DRM_MODE_FLAG_PVSYNC); + } else { + enum adihdmi_sync_polarity mode_hsync_polarity; + enum adihdmi_sync_polarity mode_vsync_polarity; + + /** + * If the input signal is always low or always high we want to + * invert or let it passthrough depending on the polarity of the + * current mode. + **/ + if (adj_mode->flags & DRM_MODE_FLAG_NHSYNC) + mode_hsync_polarity = ADIHDMI_SYNC_POLARITY_LOW; + else + mode_hsync_polarity = ADIHDMI_SYNC_POLARITY_HIGH; + + if (adj_mode->flags & DRM_MODE_FLAG_NVSYNC) + mode_vsync_polarity = ADIHDMI_SYNC_POLARITY_LOW; + else + mode_vsync_polarity = ADIHDMI_SYNC_POLARITY_HIGH; + + if (adihdmi->hsync_polarity != mode_hsync_polarity && + adihdmi->hsync_polarity != + ADIHDMI_SYNC_POLARITY_PASSTHROUGH) + hsync_polarity = 1; + + if (adihdmi->vsync_polarity != mode_vsync_polarity && + adihdmi->vsync_polarity != + ADIHDMI_SYNC_POLARITY_PASSTHROUGH) + vsync_polarity = 1; + } + + if (mode->vrefresh <= 24000) + low_refresh_rate = ADIHDMI_LOW_REFRESH_RATE_24HZ; + else if (mode->vrefresh <= 25000) + low_refresh_rate = ADIHDMI_LOW_REFRESH_RATE_25HZ; + else if (mode->vrefresh <= 30000) + low_refresh_rate = ADIHDMI_LOW_REFRESH_RATE_30HZ; + else + low_refresh_rate = ADIHDMI_LOW_REFRESH_RATE_NONE; + + regmap_update_bits(adihdmi->regmap, 0xfb, + 0x6, low_refresh_rate << 1); + regmap_update_bits(adihdmi->regmap, 0x17, + 0x60, (vsync_polarity << 6) | (hsync_polarity << 5)); + + /* + * TODO Test first order 4:2:2 to 4:4:4 up conversion method, which is + * supposed to give better results. + */ + + adihdmi->f_tmds = mode->clock; +} + +static void adihdmi_encoder_prepare(struct drm_encoder *encoder) +{ + adihdmi_encoder_dpms(encoder, DRM_MODE_DPMS_OFF); +} + +static void adihdmi_encoder_commit(struct drm_encoder *encoder) +{ + adihdmi_encoder_dpms(encoder, DRM_MODE_DPMS_ON); +} + +static struct drm_encoder_helper_funcs adihdmi_encoder_helper_funcs = { + .dpms = adihdmi_encoder_dpms, + .mode_fixup = adihdmi_encoder_mode_fixup, + .prepare = adihdmi_encoder_prepare, + .commit = adihdmi_encoder_commit, + .mode_set = adihdmi_encoder_mode_set, +}; + +static void adihdmi_encoder_destroy(struct drm_encoder *encoder) +{ + struct adihdmi2 *priv = enc_to_adihdmi2(encoder); + + adihdmi_destroy(&priv->base); + drm_encoder_cleanup(encoder); +} + +static const struct drm_encoder_funcs adihdmi_encoder_funcs = { + .destroy = adihdmi_encoder_destroy, +}; + +/* ----------------------------------------------------------------------------- + * Slave operations + */ + +static int adihdmi_encoder_slave_create_resources(struct drm_encoder *encoder, struct drm_connector *connector) +{ + pr_debug("%s - %d\n", __FUNCTION__, __LINE__); + return 0; +} + +static void adihdmi_encoder_slave_destroy(struct drm_encoder *encoder) +{ + pr_debug("%s - %d\n", __FUNCTION__, __LINE__); +} + + static enum drm_connector_status +adihdmi_encoder_slave_detect(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + return adihdmi_encoder_detect(enc_to_adihdmi(encoder), + connector); +} + +static int adihdmi_encoder_slave_get_modes(struct drm_encoder *encoder, + struct drm_connector *connector) +{ + return adihdmi_encoder_get_modes(enc_to_adihdmi(encoder), + connector); +} + + +static void adihdmi_encoder_slave_set_config(struct drm_encoder *encoder, void *params) +{ + pr_debug("%s - %d\n", __FUNCTION__, __LINE__); +} + +static int adihdmi_encoder_set_property(struct drm_encoder *encoder, struct drm_connector *connector, struct drm_property *property, uint64_t val) +{ + pr_debug("%s - %d\n", __FUNCTION__, __LINE__); + return 0; +} + +static struct drm_encoder_slave_funcs adihdmi_encoder_slave_funcs = { + .create_resources = adihdmi_encoder_slave_create_resources, + .destroy = adihdmi_encoder_slave_destroy, + .detect = adihdmi_encoder_slave_detect, + .dpms = adihdmi_encoder_dpms, + .get_modes = adihdmi_encoder_slave_get_modes, + .mode_fixup = adihdmi_encoder_mode_fixup, + .mode_set = adihdmi_encoder_mode_set, + .mode_valid = adihdmi_encoder_mode_valid, + .set_config = adihdmi_encoder_slave_set_config, + .set_property = adihdmi_encoder_set_property, +}; + +/* ----------------------------------------------------------------------------- + * Connector operations + */ + +static int adihdmi_connector_get_modes(struct drm_connector *connector) +{ + struct adihdmi2 *priv = conn_to_adihdmi2(connector); + + return adihdmi_encoder_get_modes(&priv->base, connector); +} + +static int adihdmi_connector_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + return adihdmi_mode_valid(mode); +} + + static struct drm_encoder * +adihdmi_connector_best_encoder(struct drm_connector *connector) +{ + struct adihdmi2 *priv = conn_to_adihdmi2(connector); + + return &priv->encoder; +} + +static struct drm_connector_helper_funcs adihdmi_connector_helper_funcs = { + .get_modes = adihdmi_connector_get_modes, + .mode_valid = adihdmi_connector_mode_valid, + .best_encoder = adihdmi_connector_best_encoder, +}; + + static enum drm_connector_status +adihdmi_connector_detect(struct drm_connector *connector, bool force) +{ + struct adihdmi2 *priv = conn_to_adihdmi2(connector); + + return adihdmi_encoder_detect(&priv->base, connector); +} + +static void adihdmi_connector_destroy(struct drm_connector *connector) +{ + drm_connector_unregister(connector); + drm_connector_cleanup(connector); +} + +static struct drm_connector_funcs adihdmi_connector_funcs = { + .dpms = drm_helper_connector_dpms, + .fill_modes = drm_helper_probe_single_connector_modes, + .detect = adihdmi_connector_detect, + .destroy = adihdmi_connector_destroy, +}; + +/* ----------------------------------------------------------------------------- + * Component operations + */ + +static int adihdmi_bind(struct device *dev, struct device *master, void *data) +{ + struct i2c_client *client = to_i2c_client(dev); + struct drm_device *drm = data; + struct adihdmi2 *priv; + uint32_t crtcs = 0; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + { + pr_err("%s - %d - No memory for ADIHDMI\n", __FUNCTION__, __LINE__); + return -ENOMEM; + } + + dev_set_drvdata(dev, priv); + + if (dev->of_node) + crtcs = drm_of_find_possible_crtcs(drm, dev->of_node); + + /* If no CRTCs were found, fall back to our old behaviour */ + if (crtcs == 0) { + dev_warn(dev, "Falling back to first CRTC\n"); + crtcs = 1 << 0; + } + + priv->base.encoder = &priv->encoder; + priv->connector.interlace_allowed = 1; + priv->encoder.possible_crtcs = crtcs; + + ret = adihdmi_create(client, &priv->base); + if (ret) + return ret; + + drm_encoder_helper_add(&priv->encoder, &adihdmi_encoder_helper_funcs); + ret = drm_encoder_init(drm, &priv->encoder, &adihdmi_encoder_funcs, + DRM_MODE_ENCODER_TMDS, NULL); + if (ret) + goto err_encoder; + + drm_connector_helper_add(&priv->connector, + &adihdmi_connector_helper_funcs); + ret = drm_connector_init(drm, &priv->connector, + &adihdmi_connector_funcs, + DRM_MODE_CONNECTOR_HDMIA); + if (ret) + goto err_connector; + + ret = drm_connector_register(&priv->connector); + if (ret) + goto err_sysfs; + + priv->connector.encoder = &priv->encoder; + drm_mode_connector_attach_encoder(&priv->connector, &priv->encoder); + + return 0; + +err_sysfs: + drm_connector_cleanup(&priv->connector); +err_connector: + drm_encoder_cleanup(&priv->encoder); +err_encoder: + adihdmi_destroy(&priv->base); + return ret; + + +} + +static void adihdmi_unbind(struct device *dev, struct device *master, void *data) +{ + struct adihdmi2 *priv = dev_get_drvdata(dev); + + drm_connector_cleanup(&priv->connector); + drm_encoder_cleanup(&priv->encoder); + adihdmi_destroy(&priv->base); +} + +static const struct component_ops adihdmi_ops = +{ + .bind = adihdmi_bind, + .unbind = adihdmi_unbind, +}; + +/* ----------------------------------------------------------------------------- + * Init operations + */ + +static int adihdmi_probe(struct i2c_client *i2c, const struct i2c_device_id *id) +{ + return component_add(&i2c->dev, &adihdmi_ops); +} + +static int adihdmi_remove(struct i2c_client *i2c) +{ + component_del(&i2c->dev, &adihdmi_ops); + + return 0; +} + +static int adihdmi_encoder_init(struct i2c_client *i2c, struct drm_device *dev, + struct drm_encoder_slave *encoder_slave) +{ + + struct adihdmi *adihdmi; + int ret; + + adihdmi = kzalloc(sizeof(*adihdmi), GFP_KERNEL); + if (!adihdmi) + return -ENOMEM; + + adihdmi->encoder = &encoder_slave->base; + + ret = adihdmi_create(i2c, adihdmi); + if (ret) { + kfree(adihdmi); + return ret; + } + + encoder_slave->slave_priv = adihdmi; + encoder_slave->slave_funcs = &adihdmi_encoder_slave_funcs; + + return 0; +} + +static const struct i2c_device_id adihdmi_i2c_ids[] = { + { "adv7511", 0 }, + { "adv7511w", 0 }, + { "adv7513", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, adihdmi_i2c_ids); + +static const struct of_device_id adihdmi_of_ids[] = { + { .compatible = "adi,adv7511", }, + { .compatible = "adi,adv7511w", }, + { .compatible = "adi,adv7513", }, + { } +}; +MODULE_DEVICE_TABLE(of, adihdmi_of_ids); + +static struct drm_i2c_encoder_driver adihdmi_driver = { + .i2c_driver = { + .driver = { + .name = "adihdmi", + .of_match_table = adihdmi_of_ids, + }, + .id_table = adihdmi_i2c_ids, + .probe = adihdmi_probe, + .remove = adihdmi_remove, + }, + + .encoder_init = adihdmi_encoder_init, +}; + +static int __init adihdmi_init(void) +{ + return drm_i2c_encoder_register(THIS_MODULE, &adihdmi_driver); +} +module_init(adihdmi_init); + +static void __exit adihdmi_exit(void) +{ + drm_i2c_encoder_unregister(&adihdmi_driver); +} +module_exit(adihdmi_exit); + +MODULE_AUTHOR("Lars-Peter Clausen "); +MODULE_DESCRIPTION("ADIHDMI HDMI transmitter driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 369a2c632e46e..d1bf886dbf29a 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -656,6 +656,13 @@ config STM32_ADC This driver can also be built as a module. If so, the module will be called stm32-adc. +config STMPE_ADC + tristate "STMicroelectronics STMPE ADC driver" + depends on OF && MFD_STMPE + help + Say yes here to build support for ST Microelectronics STMPE + built-in ADC block (stmpe811). + config STX104 tristate "Apex Embedded Systems STX104 driver" depends on PC104 && X86 && ISA_BUS_API diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile index 9572c1090f356..72298d811d647 100644 --- a/drivers/iio/adc/Makefile +++ b/drivers/iio/adc/Makefile @@ -64,6 +64,7 @@ obj-$(CONFIG_STX104) += stx104.o obj-$(CONFIG_SUN4I_GPADC) += sun4i-gpadc-iio.o obj-$(CONFIG_STM32_ADC_CORE) += stm32-adc-core.o obj-$(CONFIG_STM32_ADC) += stm32-adc.o +obj-$(CONFIG_STMPE_ADC) += stmpe-adc.o obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o obj-$(CONFIG_TI_ADC084S021) += ti-adc084s021.o diff --git a/drivers/iio/adc/stmpe-adc.c b/drivers/iio/adc/stmpe-adc.c new file mode 100644 index 0000000000000..bd72727fc417a --- /dev/null +++ b/drivers/iio/adc/stmpe-adc.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * STMicroelectronics STMPE811 IIO ADC Driver + * + * 4 channel, 10/12-bit ADC + * + * Copyright (C) 2013-2018 Toradex AG + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STMPE_REG_INT_STA 0x0B +#define STMPE_REG_ADC_INT_EN 0x0E +#define STMPE_REG_ADC_INT_STA 0x0F + +#define STMPE_REG_ADC_CTRL1 0x20 +#define STMPE_REG_ADC_CTRL2 0x21 +#define STMPE_REG_ADC_CAPT 0x22 +#define STMPE_REG_ADC_DATA_CH(channel) (0x30 + 2 * (channel)) + +#define STMPE_REG_TEMP_CTRL 0x60 +#define STMPE_TEMP_CTRL_ENABLE BIT(0) +#define STMPE_TEMP_CTRL_ACQ BIT(1) +#define STMPE_TEMP_CTRL_THRES_EN BIT(3) +#define STMPE_START_ONE_TEMP_CONV (STMPE_TEMP_CTRL_ENABLE | \ + STMPE_TEMP_CTRL_ACQ | \ + STMPE_TEMP_CTRL_THRES_EN) +#define STMPE_REG_TEMP_DATA 0x61 +#define STMPE_REG_TEMP_TH 0x63 +#define STMPE_ADC_LAST_NR 7 +#define STMPE_TEMP_CHANNEL (STMPE_ADC_LAST_NR + 1) + +#define STMPE_ADC_CH(channel) ((1 << (channel)) & 0xff) + +#define STMPE_ADC_TIMEOUT msecs_to_jiffies(1000) + +struct stmpe_adc { + struct stmpe *stmpe; + struct clk *clk; + struct device *dev; + struct mutex lock; + + /* We are allocating plus one for the temperature channel */ + struct iio_chan_spec stmpe_adc_iio_channels[STMPE_ADC_LAST_NR + 2]; + + struct completion completion; + + u8 channel; + u32 value; +}; + +static int stmpe_read_voltage(struct stmpe_adc *info, + struct iio_chan_spec const *chan, int *val) +{ + long ret; + + mutex_lock(&info->lock); + + reinit_completion(&info->completion); + + info->channel = (u8)chan->channel; + + if (info->channel > STMPE_ADC_LAST_NR) { + mutex_unlock(&info->lock); + return -EINVAL; + } + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_CAPT, + STMPE_ADC_CH(info->channel)); + + ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT); + + if (ret <= 0) { + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, + STMPE_ADC_CH(info->channel)); + mutex_unlock(&info->lock); + return -ETIMEDOUT; + } + + *val = info->value; + + mutex_unlock(&info->lock); + + return 0; +} + +static int stmpe_read_temp(struct stmpe_adc *info, + struct iio_chan_spec const *chan, int *val) +{ + long ret; + + mutex_lock(&info->lock); + + reinit_completion(&info->completion); + + info->channel = (u8)chan->channel; + + if (info->channel != STMPE_TEMP_CHANNEL) { + mutex_unlock(&info->lock); + return -EINVAL; + } + + stmpe_reg_write(info->stmpe, STMPE_REG_TEMP_CTRL, + STMPE_START_ONE_TEMP_CONV); + + ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT); + + if (ret <= 0) { + mutex_unlock(&info->lock); + return -ETIMEDOUT; + } + + /* + * absolute temp = +V3.3 * value /7.51 [K] + * scale to [milli °C] + */ + *val = ((449960l * info->value) / 1024l) - 273150; + + mutex_unlock(&info->lock); + + return 0; +} + +static int stmpe_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, + int *val2, + long mask) +{ + struct stmpe_adc *info = iio_priv(indio_dev); + long ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + case IIO_CHAN_INFO_PROCESSED: + + switch (chan->type) { + case IIO_VOLTAGE: + ret = stmpe_read_voltage(info, chan, val); + break; + + case IIO_TEMP: + ret = stmpe_read_temp(info, chan, val); + break; + default: + return -EINVAL; + } + + if (ret < 0) + return ret; + + return IIO_VAL_INT; + + case IIO_CHAN_INFO_SCALE: + *val = 3300; + *val2 = info->stmpe->mod_12b ? 12 : 10; + return IIO_VAL_FRACTIONAL_LOG2; + + default: + break; + } + + return -EINVAL; +} + +static irqreturn_t stmpe_adc_isr(int irq, void *dev_id) +{ + struct stmpe_adc *info = (struct stmpe_adc *)dev_id; + u16 data; + + if (info->channel <= STMPE_ADC_LAST_NR) { + int int_sta; + + int_sta = stmpe_reg_read(info->stmpe, STMPE_REG_ADC_INT_STA); + + /* Is the interrupt relevant */ + if (!(int_sta & STMPE_ADC_CH(info->channel))) + return IRQ_NONE; + + /* Read value */ + stmpe_block_read(info->stmpe, + STMPE_REG_ADC_DATA_CH(info->channel), 2, (u8 *) &data); + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, int_sta); + } else if (info->channel == STMPE_TEMP_CHANNEL) { + /* Read value */ + stmpe_block_read(info->stmpe, STMPE_REG_TEMP_DATA, 2, + (u8 *) &data); + } else { + return IRQ_NONE; + } + + info->value = (u32) be16_to_cpu(data); + complete(&info->completion); + + return IRQ_HANDLED; +} + +static const struct iio_info stmpe_adc_iio_info = { + .read_raw = &stmpe_read_raw, +}; + +static void stmpe_adc_voltage_chan(struct iio_chan_spec *ics, int chan) +{ + ics->type = IIO_VOLTAGE; + ics->info_mask_separate = BIT(IIO_CHAN_INFO_RAW); + ics->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE); + ics->indexed = 1; + ics->channel = chan; +} + +static void stmpe_adc_temp_chan(struct iio_chan_spec *ics, int chan) +{ + ics->type = IIO_TEMP; + ics->info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED); + ics->indexed = 1; + ics->channel = chan; +} + +static int stmpe_adc_init_hw(struct stmpe_adc *adc) +{ + int ret; + struct stmpe *stmpe = adc->stmpe; + + ret = stmpe_enable(stmpe, STMPE_BLOCK_ADC); + if (ret) { + dev_err(stmpe->dev, "Could not enable clock for ADC\n"); + return ret; + } + + ret = stmpe811_adc_common_init(stmpe); + if (ret) { + stmpe_disable(stmpe, STMPE_BLOCK_ADC); + return ret; + } + + /* use temp irq for each conversion completion */ + stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH, 0); + stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH + 1, 0); + + return 0; +} + +static int stmpe_adc_probe(struct platform_device *pdev) +{ + struct iio_dev *indio_dev; + struct stmpe_adc *info; + struct device_node *np; + u32 norequest_mask = 0; + int irq_temp, irq_adc; + int num_chan = 0; + int i = 0; + int ret; + + irq_adc = platform_get_irq_byname(pdev, "STMPE_ADC"); + if (irq_adc < 0) + return irq_adc; + + indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct stmpe_adc)); + if (!indio_dev) { + dev_err(&pdev->dev, "failed allocating iio device\n"); + return -ENOMEM; + } + + info = iio_priv(indio_dev); + mutex_init(&info->lock); + + init_completion(&info->completion); + ret = devm_request_threaded_irq(&pdev->dev, irq_adc, NULL, + stmpe_adc_isr, IRQF_ONESHOT, + "stmpe-adc", info); + if (ret < 0) { + dev_err(&pdev->dev, "failed requesting irq, irq = %d\n", + irq_adc); + return ret; + } + + irq_temp = platform_get_irq_byname(pdev, "STMPE_TEMP_SENS"); + if (irq_temp >= 0) { + ret = devm_request_threaded_irq(&pdev->dev, irq_temp, NULL, + stmpe_adc_isr, IRQF_ONESHOT, + "stmpe-adc", info); + if (ret < 0) + dev_warn(&pdev->dev, "failed requesting irq for" + " temp sensor, irq = %d\n", irq_temp); + } + + platform_set_drvdata(pdev, indio_dev); + + indio_dev->name = dev_name(&pdev->dev); + indio_dev->dev.parent = &pdev->dev; + indio_dev->info = &stmpe_adc_iio_info; + indio_dev->modes = INDIO_DIRECT_MODE; + + info->stmpe = dev_get_drvdata(pdev->dev.parent); + + np = pdev->dev.of_node; + + if (!np) + dev_err(&pdev->dev, "no device tree node found\n"); + + of_property_read_u32(np, "st,norequest-mask", &norequest_mask); + + for_each_clear_bit(i, (unsigned long *) &norequest_mask, + (STMPE_ADC_LAST_NR + 1)) { + stmpe_adc_voltage_chan(&info->stmpe_adc_iio_channels[num_chan], i); + num_chan++; + } + stmpe_adc_temp_chan(&info->stmpe_adc_iio_channels[num_chan], i); + num_chan++; + indio_dev->channels = info->stmpe_adc_iio_channels; + indio_dev->num_channels = num_chan; + + ret = stmpe_adc_init_hw(info); + if (ret) + return ret; + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_EN, + ~(norequest_mask & 0xFF)); + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, + ~(norequest_mask & 0xFF)); + + return devm_iio_device_register(&pdev->dev, indio_dev); +} + +static int __maybe_unused stmpe_adc_resume(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct stmpe_adc *info = iio_priv(indio_dev); + + stmpe_adc_init_hw(info); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(stmpe_adc_pm_ops, NULL, stmpe_adc_resume); + +static struct platform_driver stmpe_adc_driver = { + .probe = stmpe_adc_probe, + .driver = { + .name = "stmpe-adc", + .pm = &stmpe_adc_pm_ops, + }, +}; +module_platform_driver(stmpe_adc_driver); + +static const struct of_device_id stmpe_adc_ids[] = { + { .compatible = "st,stmpe-adc", }, + { }, +}; +MODULE_DEVICE_TABLE(of, stmpe_adc_ids); + +MODULE_AUTHOR("Stefan Agner "); +MODULE_DESCRIPTION("STMPEXXX ADC driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:stmpe-adc"); diff --git a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig index c03de24d3c518..68343c9ee8e38 100644 --- a/drivers/iio/proximity/Kconfig +++ b/drivers/iio/proximity/Kconfig @@ -69,4 +69,15 @@ config SRF08 To compile this driver as a module, choose M here: the module will be called srf08. +config VL53L0X_I2C + tristate "STMicroelectronics VL53L0X ToF ranger sensor (I2C)" + depends on I2C + help + Say Y here to build a driver for STMicroelectronics VL53L0X + ToF ranger sensors with i2c interface. + This driver can be used to measure the distance of objects. + + To compile this driver as a module, choose M here: the + module will be called vl53l0x-i2c. + endmenu diff --git a/drivers/iio/proximity/Makefile b/drivers/iio/proximity/Makefile index ad6686391f083..607ec7b09278c 100644 --- a/drivers/iio/proximity/Makefile +++ b/drivers/iio/proximity/Makefile @@ -9,3 +9,5 @@ obj-$(CONFIG_LIDAR_LITE_V2) += pulsedlight-lidar-lite-v2.o obj-$(CONFIG_SRF04) += srf04.o obj-$(CONFIG_SRF08) += srf08.o obj-$(CONFIG_SX9500) += sx9500.o +obj-$(CONFIG_VL53L0X_I2C) += vl53l0x-i2c.o + diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c new file mode 100644 index 0000000000000..b48216cc18589 --- /dev/null +++ b/drivers/iio/proximity/vl53l0x-i2c.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for ST VL53L0X FlightSense ToF Ranging Sensor on a i2c bus. + * + * Copyright (C) 2016 STMicroelectronics Imaging Division. + * Copyright (C) 2018 Song Qiang + * + * Datasheet available at + * + * + * Default 7-bit i2c slave address 0x29. + * + * TODO: FIFO buffer, continuous mode, interrupts, range selection, + * sensor ID check. + */ + +#include +#include +#include + +#include + +#define VL_REG_SYSRANGE_START 0x00 + +#define VL_REG_SYSRANGE_MODE_MASK GENMASK(3, 0) +#define VL_REG_SYSRANGE_MODE_SINGLESHOT 0x00 +#define VL_REG_SYSRANGE_MODE_START_STOP BIT(0) +#define VL_REG_SYSRANGE_MODE_BACKTOBACK BIT(1) +#define VL_REG_SYSRANGE_MODE_TIMED BIT(2) +#define VL_REG_SYSRANGE_MODE_HISTOGRAM BIT(3) + +#define VL_REG_RESULT_INT_STATUS 0x13 +#define VL_REG_RESULT_RANGE_STATUS 0x14 +#define VL_REG_RESULT_RANGE_STATUS_COMPLETE BIT(0) + +struct vl53l0x_data { + struct i2c_client *client; +}; + +static int vl53l0x_read_proximity(struct vl53l0x_data *data, + const struct iio_chan_spec *chan, + int *val) +{ + struct i2c_client *client = data->client; + u16 tries = 20; + u8 buffer[12]; + int ret; + + ret = i2c_smbus_write_byte_data(client, VL_REG_SYSRANGE_START, 1); + if (ret < 0) + return ret; + + do { + ret = i2c_smbus_read_byte_data(client, + VL_REG_RESULT_RANGE_STATUS); + if (ret < 0) + return ret; + + if (ret & VL_REG_RESULT_RANGE_STATUS_COMPLETE) + break; + + usleep_range(1000, 5000); + } while (--tries); + if (!tries) + return -ETIMEDOUT; + + ret = i2c_smbus_read_i2c_block_data(client, VL_REG_RESULT_RANGE_STATUS, + 12, buffer); + if (ret < 0) + return ret; + else if (ret != 12) + return -EREMOTEIO; + + /* Values should be between 30~1200 in millimeters. */ + *val = (buffer[10] << 8) + buffer[11]; + + return 0; +} + +static const struct iio_chan_spec vl53l0x_channels[] = { + { + .type = IIO_DISTANCE, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + }, +}; + +static int vl53l0x_read_raw(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + int *val, int *val2, long mask) +{ + struct vl53l0x_data *data = iio_priv(indio_dev); + int ret; + + if (chan->type != IIO_DISTANCE) + return -EINVAL; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = vl53l0x_read_proximity(data, chan, val); + if (ret < 0) + return ret; + + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + *val2 = 1000; + + return IIO_VAL_INT_PLUS_MICRO; + default: + return -EINVAL; + } +} + +static const struct iio_info vl53l0x_info = { + .read_raw = vl53l0x_read_raw, +}; + +static int vl53l0x_probe(struct i2c_client *client) +{ + struct vl53l0x_data *data; + struct iio_dev *indio_dev; + + indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); + if (!indio_dev) + return -ENOMEM; + + data = iio_priv(indio_dev); + data->client = client; + i2c_set_clientdata(client, indio_dev); + + if (!i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_READ_I2C_BLOCK | + I2C_FUNC_SMBUS_BYTE_DATA)) + return -EOPNOTSUPP; + + indio_dev->dev.parent = &client->dev; + indio_dev->name = "vl53l0x"; + indio_dev->info = &vl53l0x_info; + indio_dev->channels = vl53l0x_channels; + indio_dev->num_channels = ARRAY_SIZE(vl53l0x_channels); + indio_dev->modes = INDIO_DIRECT_MODE; + + return devm_iio_device_register(&client->dev, indio_dev); +} + +static const struct of_device_id st_vl53l0x_dt_match[] = { + { .compatible = "st,vl53l0x", }, + { } +}; +MODULE_DEVICE_TABLE(of, st_vl53l0x_dt_match); + +static struct i2c_driver vl53l0x_driver = { + .driver = { + .name = "vl53l0x-i2c", + .of_match_table = st_vl53l0x_dt_match, + }, + .probe_new = vl53l0x_probe, +}; +module_i2c_driver(vl53l0x_driver); + +MODULE_AUTHOR("Song Qiang "); +MODULE_DESCRIPTION("ST vl53l0x ToF ranging sensor driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c index a4455bb12ae0c..b6d4321d421fc 100644 --- a/drivers/input/misc/tps65218-pwrbutton.c +++ b/drivers/input/misc/tps65218-pwrbutton.c @@ -36,7 +36,7 @@ struct tps6521x_data { static const struct tps6521x_data tps65217_data = { .reg_status = TPS65217_REG_STATUS, .pb_mask = TPS65217_STATUS_PB, - .name = "tps65217_pwrbutton", + .name = "tps65217_pwr_but", }; static const struct tps6521x_data tps65218_data = { diff --git a/drivers/input/touchscreen/ar1021_i2c.c b/drivers/input/touchscreen/ar1021_i2c.c index b35b640fdadfa..a1d4019c5a743 100644 --- a/drivers/input/touchscreen/ar1021_i2c.c +++ b/drivers/input/touchscreen/ar1021_i2c.c @@ -18,6 +18,11 @@ #define AR1021_MAX_X 4095 #define AR1021_MAX_Y 4095 +#define AR1021_MAX_PRESSURE 255 + +#define AR1021_CMD 0x55 + +#define AR1021_CMD_ENABLE_TOUCH 0x12 #define AR1021_CMD 0x55 @@ -27,8 +32,29 @@ struct ar1021_i2c { struct i2c_client *client; struct input_dev *input; u8 data[AR1021_TOCUH_PKG_SIZE]; + bool invert_x; + bool invert_y; + bool swap_xy; }; +static bool ar1021_get_prop_u32(struct device *dev, + const char *property, + unsigned int default_value, + unsigned int *value) +{ + u32 val; + int error; + + error = device_property_read_u32(dev, property, &val); + if (error) { + *value = default_value; + return false; + } + + *value = val; + return true; +} + static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) { struct ar1021_i2c *ar1021 = dev_id; @@ -50,9 +76,22 @@ static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) x = ((data[2] & 0x1f) << 7) | (data[1] & 0x7f); y = ((data[4] & 0x1f) << 7) | (data[3] & 0x7f); - input_report_abs(input, ABS_X, x); - input_report_abs(input, ABS_Y, y); + if (ar1021->invert_x) + x = AR1021_MAX_X - x; + + if (ar1021->invert_y) + y = AR1021_MAX_Y - y; + + if (ar1021->swap_xy) { + input_report_abs(input, ABS_X, y); + input_report_abs(input, ABS_Y, x); + } else { + input_report_abs(input, ABS_X, x); + input_report_abs(input, ABS_Y, y); + } + input_report_key(input, BTN_TOUCH, button); + input_report_abs(input, ABS_PRESSURE, AR1021_MAX_PRESSURE); input_sync(input); out: @@ -94,6 +133,8 @@ static int ar1021_i2c_probe(struct i2c_client *client, struct ar1021_i2c *ar1021; struct input_dev *input; int error; + unsigned int offset_x, offset_y; + bool data_present; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { dev_err(&client->dev, "i2c_check_functionality error\n"); @@ -117,10 +158,44 @@ static int ar1021_i2c_probe(struct i2c_client *client, input->open = ar1021_i2c_open; input->close = ar1021_i2c_close; + ar1021->invert_x = device_property_read_bool(&client->dev, "touchscreen-inverted-x"); + ar1021->invert_y = device_property_read_bool(&client->dev, "touchscreen-inverted-y"); + ar1021->swap_xy = device_property_read_bool(&client->dev, "touchscreen-swapped-x-y"); + + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-x", + 0, + &offset_x); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-x: %d\n", offset_x); + + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-y", + 0, + &offset_y); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-y: %d\n", offset_y); + __set_bit(INPUT_PROP_DIRECT, input->propbit); - input_set_capability(input, EV_KEY, BTN_TOUCH); - input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0); - input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0); + //input_set_capability(input, EV_KEY, BTN_TOUCH); + + input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); + input->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); + + if(ar1021->swap_xy) + { + input_set_abs_params(input, ABS_X, 0, AR1021_MAX_Y, 0, 0); + input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_X, 0, 0); + } + else + { + input_set_abs_params(input, ABS_X, offset_x, AR1021_MAX_X-offset_x, 0, 0); + input_set_abs_params(input, ABS_Y, offset_y, AR1021_MAX_Y-offset_y, 0, 0); + } + + input_set_abs_params(input, ABS_PRESSURE, 0, AR1021_MAX_PRESSURE, 0, 0); input_set_drvdata(input, ar1021); diff --git a/drivers/input/touchscreen/stmpe-ts.c b/drivers/input/touchscreen/stmpe-ts.c index 2a78e27b4495d..fdf93d205bfe9 100644 --- a/drivers/input/touchscreen/stmpe-ts.c +++ b/drivers/input/touchscreen/stmpe-ts.c @@ -1,14 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * STMicroelectronics STMPE811 Touchscreen Driver * * (C) 2010 Luotao Fu * All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * */ #include @@ -30,8 +25,6 @@ * with touchscreen controller */ #define STMPE_REG_INT_STA 0x0B -#define STMPE_REG_ADC_CTRL1 0x20 -#define STMPE_REG_ADC_CTRL2 0x21 #define STMPE_REG_TSC_CTRL 0x40 #define STMPE_REG_TSC_CFG 0x41 #define STMPE_REG_FIFO_TH 0x4A @@ -49,17 +42,6 @@ #define STMPE_IRQ_TOUCH_DET 0 -#define SAMPLE_TIME(x) ((x & 0xf) << 4) -#define MOD_12B(x) ((x & 0x1) << 3) -#define REF_SEL(x) ((x & 0x1) << 1) -#define ADC_FREQ(x) (x & 0x3) -#define AVE_CTRL(x) ((x & 0x3) << 6) -#define DET_DELAY(x) ((x & 0x7) << 3) -#define SETTLING(x) (x & 0x7) -#define FRACTION_Z(x) (x & 0x7) -#define I_DRIVE(x) (x & 0x1) -#define OP_MODE(x) ((x & 0x7) << 1) - #define STMPE_TS_NAME "stmpe-ts" #define XY_MASK 0xfff @@ -69,15 +51,6 @@ * @idev: registered input device * @work: a work item used to scan the device * @dev: a pointer back to the MFD cell struct device* - * @sample_time: ADC converstion time in number of clock. - * (0 -> 36 clocks, 1 -> 44 clocks, 2 -> 56 clocks, 3 -> 64 clocks, - * 4 -> 80 clocks, 5 -> 96 clocks, 6 -> 144 clocks), - * recommended is 4. - * @mod_12b: ADC Bit mode (0 -> 10bit ADC, 1 -> 12bit ADC) - * @ref_sel: ADC reference source - * (0 -> internal reference, 1 -> external reference) - * @adc_freq: ADC Clock speed - * (0 -> 1.625 MHz, 1 -> 3.25 MHz, 2 || 3 -> 6.5 MHz) * @ave_ctrl: Sample average control * (0 -> 1 sample, 1 -> 2 samples, 2 -> 4 samples, 3 -> 8 samples) * @touch_det_delay: Touch detect interrupt delay @@ -99,15 +72,14 @@ struct stmpe_touch { struct input_dev *idev; struct delayed_work work; struct device *dev; - u8 sample_time; - u8 mod_12b; - u8 ref_sel; - u8 adc_freq; u8 ave_ctrl; u8 touch_det_delay; u8 settling; u8 fraction_z; u8 i_drive; + bool invert_x; + bool invert_y; + bool swap_xy; }; static int __stmpe_reset_fifo(struct stmpe *stmpe) @@ -181,8 +153,23 @@ static irqreturn_t stmpe_ts_handler(int irq, void *data) y = ((data_set[1] & 0xf) << 8) | data_set[2]; z = data_set[3]; - input_report_abs(ts->idev, ABS_X, x); - input_report_abs(ts->idev, ABS_Y, y); + //input_report_abs(ts->idev, ABS_X, x); + //input_report_abs(ts->idev, ABS_Y, y); + + if (ts->invert_x) + x = XY_MASK - x; + + if (ts->invert_y) + y = XY_MASK - y; + + if (ts->swap_xy) { + input_report_abs(ts->idev, ABS_X, y); + input_report_abs(ts->idev, ABS_Y, x); + } else { + input_report_abs(ts->idev, ABS_X, x); + input_report_abs(ts->idev, ABS_Y, y); + } + input_report_abs(ts->idev, ABS_PRESSURE, z); input_report_key(ts->idev, BTN_TOUCH, 1); input_sync(ts->idev); @@ -203,7 +190,7 @@ static irqreturn_t stmpe_ts_handler(int irq, void *data) static int stmpe_init_hw(struct stmpe_touch *ts) { int ret; - u8 adc_ctrl1, adc_ctrl1_mask, tsc_cfg, tsc_cfg_mask; + u8 tsc_cfg, tsc_cfg_mask; struct stmpe *stmpe = ts->stmpe; struct device *dev = ts->dev; @@ -213,27 +200,17 @@ static int stmpe_init_hw(struct stmpe_touch *ts) return ret; } - adc_ctrl1 = SAMPLE_TIME(ts->sample_time) | MOD_12B(ts->mod_12b) | - REF_SEL(ts->ref_sel); - adc_ctrl1_mask = SAMPLE_TIME(0xff) | MOD_12B(0xff) | REF_SEL(0xff); - - ret = stmpe_set_bits(stmpe, STMPE_REG_ADC_CTRL1, - adc_ctrl1_mask, adc_ctrl1); + ret = stmpe811_adc_common_init(stmpe); if (ret) { - dev_err(dev, "Could not setup ADC\n"); + stmpe_disable(stmpe, STMPE_BLOCK_TOUCHSCREEN | STMPE_BLOCK_ADC); return ret; } - ret = stmpe_set_bits(stmpe, STMPE_REG_ADC_CTRL2, - ADC_FREQ(0xff), ADC_FREQ(ts->adc_freq)); - if (ret) { - dev_err(dev, "Could not setup ADC\n"); - return ret; - } - - tsc_cfg = AVE_CTRL(ts->ave_ctrl) | DET_DELAY(ts->touch_det_delay) | - SETTLING(ts->settling); - tsc_cfg_mask = AVE_CTRL(0xff) | DET_DELAY(0xff) | SETTLING(0xff); + tsc_cfg = STMPE_AVE_CTRL(ts->ave_ctrl) | + STMPE_DET_DELAY(ts->touch_det_delay) | + STMPE_SETTLING(ts->settling); + tsc_cfg_mask = STMPE_AVE_CTRL(0xff) | STMPE_DET_DELAY(0xff) | + STMPE_SETTLING(0xff); ret = stmpe_set_bits(stmpe, STMPE_REG_TSC_CFG, tsc_cfg_mask, tsc_cfg); if (ret) { @@ -242,14 +219,14 @@ static int stmpe_init_hw(struct stmpe_touch *ts) } ret = stmpe_set_bits(stmpe, STMPE_REG_TSC_FRACTION_Z, - FRACTION_Z(0xff), FRACTION_Z(ts->fraction_z)); + STMPE_FRACTION_Z(0xff), STMPE_FRACTION_Z(ts->fraction_z)); if (ret) { dev_err(dev, "Could not config touch\n"); return ret; } ret = stmpe_set_bits(stmpe, STMPE_REG_TSC_I_DRIVE, - I_DRIVE(0xff), I_DRIVE(ts->i_drive)); + STMPE_I_DRIVE(0xff), STMPE_I_DRIVE(ts->i_drive)); if (ret) { dev_err(dev, "Could not config touch\n"); return ret; @@ -263,7 +240,7 @@ static int stmpe_init_hw(struct stmpe_touch *ts) } ret = stmpe_set_bits(stmpe, STMPE_REG_TSC_CTRL, - OP_MODE(0xff), OP_MODE(OP_MOD_XYZ)); + STMPE_OP_MODE(0xff), STMPE_OP_MODE(OP_MOD_XYZ)); if (ret) { dev_err(dev, "Could not set mode\n"); return ret; @@ -303,13 +280,13 @@ static void stmpe_ts_get_platform_info(struct platform_device *pdev, if (np) { if (!of_property_read_u32(np, "st,sample-time", &val)) - ts->sample_time = val; + ts->stmpe->sample_time = val; if (!of_property_read_u32(np, "st,mod-12b", &val)) - ts->mod_12b = val; + ts->stmpe->mod_12b = val; if (!of_property_read_u32(np, "st,ref-sel", &val)) - ts->ref_sel = val; + ts->stmpe->ref_sel = val; if (!of_property_read_u32(np, "st,adc-freq", &val)) - ts->adc_freq = val; + ts->stmpe->adc_freq = val; if (!of_property_read_u32(np, "st,ave-ctrl", &val)) ts->ave_ctrl = val; if (!of_property_read_u32(np, "st,touch-det-delay", &val)) @@ -323,6 +300,24 @@ static void stmpe_ts_get_platform_info(struct platform_device *pdev, } } +static bool stmpe_ts_get_prop_u32(struct device *dev, + const char *property, + unsigned int default_value, + unsigned int *value) +{ + u32 val; + int error; + + error = device_property_read_u32(dev, property, &val); + if (error) { + *value = default_value; + return false; + } + + *value = val; + return true; +} + static int stmpe_input_probe(struct platform_device *pdev) { struct stmpe *stmpe = dev_get_drvdata(pdev->dev.parent); @@ -330,6 +325,8 @@ static int stmpe_input_probe(struct platform_device *pdev) struct input_dev *idev; int error; int ts_irq; + unsigned int offset_x, offset_y; + bool data_present; ts_irq = platform_get_irq_byname(pdev, "FIFO_TH"); if (ts_irq < 0) @@ -371,11 +368,31 @@ static int stmpe_input_probe(struct platform_device *pdev) idev->open = stmpe_ts_open; idev->close = stmpe_ts_close; + ts->invert_x = device_property_read_bool(&pdev->dev, "touchscreen-inverted-x"); + ts->invert_y = device_property_read_bool(&pdev->dev, "touchscreen-inverted-y"); + ts->swap_xy = device_property_read_bool(&pdev->dev, "touchscreen-swapped-x-y"); + + data_present = stmpe_ts_get_prop_u32(&pdev->dev, + "touchscreen-offset-x", + 0, + &offset_x); + + if (data_present) + dev_info(&pdev->dev, "touchscreen-offset-x: %d\n", offset_x); + + data_present = stmpe_ts_get_prop_u32(&pdev->dev, + "touchscreen-offset-y", + 0, + &offset_y); + + if (data_present) + dev_info(&pdev->dev, "touchscreen-offset-y: %d\n", offset_y); + input_set_drvdata(idev, ts); input_set_capability(idev, EV_KEY, BTN_TOUCH); - input_set_abs_params(idev, ABS_X, 0, XY_MASK, 0, 0); - input_set_abs_params(idev, ABS_Y, 0, XY_MASK, 0, 0); + input_set_abs_params(idev, ABS_X, offset_x, XY_MASK-offset_x, 0, 0); + input_set_abs_params(idev, ABS_Y, offset_y, XY_MASK-offset_y, 0, 0); input_set_abs_params(idev, ABS_PRESSURE, 0x0, 0xff, 0, 0); error = input_register_device(idev); diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 43b22e071842f..569dc65c394c0 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -34,6 +34,7 @@ #define ADCFSM_STEPID 0x10 #define SEQ_SETTLE 275 #define MAX_12BIT ((1 << 12) - 1) +#define PRESSURE_MAX 1000 static const int config_pins[] = { STEPCONFIG_XPP, @@ -231,6 +232,7 @@ static void titsc_read_coordinates(struct titsc *ts_dev, for (i = 0; i < creads; i++) { xvals[i] = titsc_readl(ts_dev, REG_FIFO0); xvals[i] &= 0xfff; + pr_debug("i %d xval %d yval %d z1 %d z2 %d\n", i, xvals[i], yvals[i], *z1, *z2); } /* @@ -309,13 +311,13 @@ static irqreturn_t titsc_irq(int irq, void *dev) * Resistance(touch) = x plate resistance * * x postion/4096 * ((z2 / z1) - 1) */ - z = z1 - z2; + z = z2 - z1; z *= x; z *= ts_dev->x_plate_resistance; - z /= z2; + z /= z1; z = (z + 2047) >> 12; - - if (z <= MAX_12BIT) { + pr_debug("x %d y %d z1 %d z2 %d z %d\n", x, y, z1, z2, z); + if (z <= PRESSURE_MAX) { input_report_abs(input_dev, ABS_X, x); input_report_abs(input_dev, ABS_Y, y); input_report_abs(input_dev, ABS_PRESSURE, z); @@ -456,6 +458,7 @@ static int titsc_probe(struct platform_device *pdev) input_dev->name = "ti-tsc"; input_dev->dev.parent = &pdev->dev; + __set_bit(INPUT_PROP_DIRECT, input_dev->propbit); input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig index 3f9ddb9fafa77..a2559b4fdfffa 100644 --- a/drivers/leds/trigger/Kconfig +++ b/drivers/leds/trigger/Kconfig @@ -77,6 +77,15 @@ config LEDS_TRIGGER_CPU If unsure, say N. +config LEDS_TRIGGER_ACTIVITY + tristate "LED activity Trigger" + depends on LEDS_TRIGGERS + help + This allows LEDs to be controlled by an immediate CPU usage. + The flash frequency and duty cycle varies from faint flashes to + intense brightness depending on the instant CPU load. + If unsure, say N. + config LEDS_TRIGGER_GPIO tristate "LED GPIO Trigger" depends on LEDS_TRIGGERS @@ -126,4 +135,11 @@ config LEDS_TRIGGER_PANIC a different trigger. If unsure, say Y. +config LEDS_TRIGGER_NETDEV + tristate "LED Netdev Trigger" + depends on NET && LEDS_TRIGGERS + help + This allows LEDs to be controlled by network device activity. + If unsure, say Y. + endif # LEDS_TRIGGERS diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile index 9f2e868811e29..f3cfe19505385 100644 --- a/drivers/leds/trigger/Makefile +++ b/drivers/leds/trigger/Makefile @@ -7,7 +7,9 @@ obj-$(CONFIG_LEDS_TRIGGER_HEARTBEAT) += ledtrig-heartbeat.o obj-$(CONFIG_LEDS_TRIGGER_BACKLIGHT) += ledtrig-backlight.o obj-$(CONFIG_LEDS_TRIGGER_GPIO) += ledtrig-gpio.o obj-$(CONFIG_LEDS_TRIGGER_CPU) += ledtrig-cpu.o +obj-$(CONFIG_LEDS_TRIGGER_ACTIVITY) += ledtrig-activity.o obj-$(CONFIG_LEDS_TRIGGER_DEFAULT_ON) += ledtrig-default-on.o obj-$(CONFIG_LEDS_TRIGGER_TRANSIENT) += ledtrig-transient.o obj-$(CONFIG_LEDS_TRIGGER_CAMERA) += ledtrig-camera.o obj-$(CONFIG_LEDS_TRIGGER_PANIC) += ledtrig-panic.o +obj-$(CONFIG_LEDS_TRIGGER_NETDEV) += ledtrig-netdev.o diff --git a/drivers/leds/trigger/ledtrig-activity.c b/drivers/leds/trigger/ledtrig-activity.c new file mode 100644 index 0000000000000..5081894082bd6 --- /dev/null +++ b/drivers/leds/trigger/ledtrig-activity.c @@ -0,0 +1,275 @@ +/* + * Activity LED trigger + * + * Copyright (C) 2017 Willy Tarreau + * Partially based on Atsushi Nemoto's ledtrig-heartbeat.c. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../leds.h" + +static int panic_detected; + +struct activity_data { + struct timer_list timer; + struct led_classdev *led_cdev; + u64 last_used; + u64 last_boot; + int time_left; + int state; + int invert; +}; + +static void led_activity_function(struct timer_list *t) +{ + struct activity_data *activity_data = from_timer(activity_data, t, + timer); + struct led_classdev *led_cdev = activity_data->led_cdev; + struct timespec boot_time; + unsigned int target; + unsigned int usage; + int delay; + u64 curr_used; + u64 curr_boot; + s32 diff_used; + s32 diff_boot; + int cpus; + int i; + + if (test_and_clear_bit(LED_BLINK_BRIGHTNESS_CHANGE, &led_cdev->work_flags)) + led_cdev->blink_brightness = led_cdev->new_blink_brightness; + + if (unlikely(panic_detected)) { + /* full brightness in case of panic */ + led_set_brightness_nosleep(led_cdev, led_cdev->blink_brightness); + return; + } + + get_monotonic_boottime(&boot_time); + + cpus = 0; + curr_used = 0; + + for_each_possible_cpu(i) { + curr_used += kcpustat_cpu(i).cpustat[CPUTIME_USER] + + kcpustat_cpu(i).cpustat[CPUTIME_NICE] + + kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM] + + kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ] + + kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + cpus++; + } + + /* We come here every 100ms in the worst case, so that's 100M ns of + * cumulated time. By dividing by 2^16, we get the time resolution + * down to 16us, ensuring we won't overflow 32-bit computations below + * even up to 3k CPUs, while keeping divides cheap on smaller systems. + */ + curr_boot = timespec_to_ns(&boot_time) * cpus; + diff_boot = (curr_boot - activity_data->last_boot) >> 16; + diff_used = (curr_used - activity_data->last_used) >> 16; + activity_data->last_boot = curr_boot; + activity_data->last_used = curr_used; + + if (diff_boot <= 0 || diff_used < 0) + usage = 0; + else if (diff_used >= diff_boot) + usage = 100; + else + usage = 100 * diff_used / diff_boot; + + /* + * Now we know the total boot_time multiplied by the number of CPUs, and + * the total idle+wait time for all CPUs. We'll compare how they evolved + * since last call. The % of overall CPU usage is : + * + * 1 - delta_idle / delta_boot + * + * What we want is that when the CPU usage is zero, the LED must blink + * slowly with very faint flashes that are detectable but not disturbing + * (typically 10ms every second, or 10ms ON, 990ms OFF). Then we want + * blinking frequency to increase up to the point where the load is + * enough to saturate one core in multi-core systems or 50% in single + * core systems. At this point it should reach 10 Hz with a 10/90 duty + * cycle (10ms ON, 90ms OFF). After this point, the blinking frequency + * remains stable (10 Hz) and only the duty cycle increases to report + * the activity, up to the point where we have 90ms ON, 10ms OFF when + * all cores are saturated. It's important that the LED never stays in + * a steady state so that it's easy to distinguish an idle or saturated + * machine from a hung one. + * + * This gives us : + * - a target CPU usage of min(50%, 100%/#CPU) for a 10% duty cycle + * (10ms ON, 90ms OFF) + * - below target : + * ON_ms = 10 + * OFF_ms = 90 + (1 - usage/target) * 900 + * - above target : + * ON_ms = 10 + (usage-target)/(100%-target) * 80 + * OFF_ms = 90 - (usage-target)/(100%-target) * 80 + * + * In order to keep a good responsiveness, we cap the sleep time to + * 100 ms and keep track of the sleep time left. This allows us to + * quickly change it if needed. + */ + + activity_data->time_left -= 100; + if (activity_data->time_left <= 0) { + activity_data->time_left = 0; + activity_data->state = !activity_data->state; + led_set_brightness_nosleep(led_cdev, + (activity_data->state ^ activity_data->invert) ? + led_cdev->blink_brightness : LED_OFF); + } + + target = (cpus > 1) ? (100 / cpus) : 50; + + if (usage < target) + delay = activity_data->state ? + 10 : /* ON */ + 990 - 900 * usage / target; /* OFF */ + else + delay = activity_data->state ? + 10 + 80 * (usage - target) / (100 - target) : /* ON */ + 90 - 80 * (usage - target) / (100 - target); /* OFF */ + + + if (!activity_data->time_left || delay <= activity_data->time_left) + activity_data->time_left = delay; + + delay = min_t(int, activity_data->time_left, 100); + mod_timer(&activity_data->timer, jiffies + msecs_to_jiffies(delay)); +} + +static ssize_t led_invert_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct activity_data *activity_data = led_cdev->trigger_data; + + return sprintf(buf, "%u\n", activity_data->invert); +} + +static ssize_t led_invert_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct activity_data *activity_data = led_cdev->trigger_data; + unsigned long state; + int ret; + + ret = kstrtoul(buf, 0, &state); + if (ret) + return ret; + + activity_data->invert = !!state; + + return size; +} + +static DEVICE_ATTR(invert, 0644, led_invert_show, led_invert_store); + +static void activity_activate(struct led_classdev *led_cdev) +{ + struct activity_data *activity_data; + int rc; + + activity_data = kzalloc(sizeof(*activity_data), GFP_KERNEL); + if (!activity_data) + return; + + led_cdev->trigger_data = activity_data; + rc = device_create_file(led_cdev->dev, &dev_attr_invert); + if (rc) { + kfree(led_cdev->trigger_data); + return; + } + + activity_data->led_cdev = led_cdev; + timer_setup(&activity_data->timer, led_activity_function, 0); + if (!led_cdev->blink_brightness) + led_cdev->blink_brightness = led_cdev->max_brightness; + led_activity_function(&activity_data->timer); + set_bit(LED_BLINK_SW, &led_cdev->work_flags); + led_cdev->activated = true; +} + +static void activity_deactivate(struct led_classdev *led_cdev) +{ + struct activity_data *activity_data = led_cdev->trigger_data; + + if (led_cdev->activated) { + del_timer_sync(&activity_data->timer); + device_remove_file(led_cdev->dev, &dev_attr_invert); + kfree(activity_data); + clear_bit(LED_BLINK_SW, &led_cdev->work_flags); + led_cdev->activated = false; + } +} + +static struct led_trigger activity_led_trigger = { + .name = "activity", + .activate = activity_activate, + .deactivate = activity_deactivate, +}; + +static int activity_reboot_notifier(struct notifier_block *nb, + unsigned long code, void *unused) +{ + led_trigger_unregister(&activity_led_trigger); + return NOTIFY_DONE; +} + +static int activity_panic_notifier(struct notifier_block *nb, + unsigned long code, void *unused) +{ + panic_detected = 1; + return NOTIFY_DONE; +} + +static struct notifier_block activity_reboot_nb = { + .notifier_call = activity_reboot_notifier, +}; + +static struct notifier_block activity_panic_nb = { + .notifier_call = activity_panic_notifier, +}; + +static int __init activity_init(void) +{ + int rc = led_trigger_register(&activity_led_trigger); + + if (!rc) { + atomic_notifier_chain_register(&panic_notifier_list, + &activity_panic_nb); + register_reboot_notifier(&activity_reboot_nb); + } + return rc; +} + +static void __exit activity_exit(void) +{ + unregister_reboot_notifier(&activity_reboot_nb); + atomic_notifier_chain_unregister(&panic_notifier_list, + &activity_panic_nb); + led_trigger_unregister(&activity_led_trigger); +} + +module_init(activity_init); +module_exit(activity_exit); + +MODULE_AUTHOR("Willy Tarreau "); +MODULE_DESCRIPTION("Activity LED trigger"); +MODULE_LICENSE("GPL"); diff --git a/drivers/leds/trigger/ledtrig-heartbeat.c b/drivers/leds/trigger/ledtrig-heartbeat.c index e95ea65380c86..f0896de410b87 100644 --- a/drivers/leds/trigger/ledtrig-heartbeat.c +++ b/drivers/leds/trigger/ledtrig-heartbeat.c @@ -25,19 +25,23 @@ static int panic_heartbeats; struct heartbeat_trig_data { + struct led_classdev *led_cdev; unsigned int phase; unsigned int period; struct timer_list timer; unsigned int invert; }; -static void led_heartbeat_function(unsigned long data) +static void led_heartbeat_function(struct timer_list *t) { - struct led_classdev *led_cdev = (struct led_classdev *) data; - struct heartbeat_trig_data *heartbeat_data = led_cdev->trigger_data; + struct heartbeat_trig_data *heartbeat_data = + from_timer(heartbeat_data, t, timer); + struct led_classdev *led_cdev; unsigned long brightness = LED_OFF; unsigned long delay = 0; + led_cdev = heartbeat_data->led_cdev; + if (unlikely(panic_heartbeats)) { led_set_brightness_nosleep(led_cdev, LED_OFF); return; @@ -127,18 +131,18 @@ static void heartbeat_trig_activate(struct led_classdev *led_cdev) return; led_cdev->trigger_data = heartbeat_data; + heartbeat_data->led_cdev = led_cdev; rc = device_create_file(led_cdev->dev, &dev_attr_invert); if (rc) { kfree(led_cdev->trigger_data); return; } - setup_timer(&heartbeat_data->timer, - led_heartbeat_function, (unsigned long) led_cdev); + timer_setup(&heartbeat_data->timer, led_heartbeat_function, 0); heartbeat_data->phase = 0; if (!led_cdev->blink_brightness) led_cdev->blink_brightness = led_cdev->max_brightness; - led_heartbeat_function(heartbeat_data->timer.data); + led_heartbeat_function(&heartbeat_data->timer); set_bit(LED_BLINK_SW, &led_cdev->work_flags); led_cdev->activated = true; } diff --git a/drivers/leds/trigger/ledtrig-netdev.c b/drivers/leds/trigger/ledtrig-netdev.c new file mode 100644 index 0000000000000..6df4781a6308a --- /dev/null +++ b/drivers/leds/trigger/ledtrig-netdev.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2017 Ben Whitten +// Copyright 2007 Oliver Jowett +// +// LED Kernel Netdev Trigger +// +// Toggles the LED to reflect the link and traffic state of a named net device +// +// Derived from ledtrig-timer.c which is: +// Copyright 2005-2006 Openedhand Ltd. +// Author: Richard Purdie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../leds.h" + +/* + * Configurable sysfs attributes: + * + * device_name - network device name to monitor + * interval - duration of LED blink, in milliseconds + * link - LED's normal state reflects whether the link is up + * (has carrier) or not + * tx - LED blinks on transmitted data + * rx - LED blinks on receive data + * + */ + +struct led_netdev_data { + spinlock_t lock; + + struct delayed_work work; + struct notifier_block notifier; + + struct led_classdev *led_cdev; + struct net_device *net_dev; + + char device_name[IFNAMSIZ]; + atomic_t interval; + unsigned int last_activity; + + unsigned long mode; +#define NETDEV_LED_LINK 0 +#define NETDEV_LED_TX 1 +#define NETDEV_LED_RX 2 +#define NETDEV_LED_MODE_LINKUP 3 +}; + +enum netdev_led_attr { + NETDEV_ATTR_LINK, + NETDEV_ATTR_TX, + NETDEV_ATTR_RX +}; + +static void set_baseline_state(struct led_netdev_data *trigger_data) +{ + int current_brightness; + struct led_classdev *led_cdev = trigger_data->led_cdev; + + current_brightness = led_cdev->brightness; + if (current_brightness) + led_cdev->blink_brightness = current_brightness; + if (!led_cdev->blink_brightness) + led_cdev->blink_brightness = led_cdev->max_brightness; + + if (!test_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode)) + led_set_brightness(led_cdev, LED_OFF); + else { + if (test_bit(NETDEV_LED_LINK, &trigger_data->mode)) + led_set_brightness(led_cdev, + led_cdev->blink_brightness); + else + led_set_brightness(led_cdev, LED_OFF); + + /* If we are looking for RX/TX start periodically + * checking stats + */ + if (test_bit(NETDEV_LED_TX, &trigger_data->mode) || + test_bit(NETDEV_LED_RX, &trigger_data->mode)) + schedule_delayed_work(&trigger_data->work, 0); + } +} + +static ssize_t device_name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + ssize_t len; + + spin_lock_bh(&trigger_data->lock); + len = sprintf(buf, "%s\n", trigger_data->device_name); + spin_unlock_bh(&trigger_data->lock); + + return len; +} + +static ssize_t device_name_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t size) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + + if (size >= IFNAMSIZ) + return -EINVAL; + + cancel_delayed_work_sync(&trigger_data->work); + + spin_lock_bh(&trigger_data->lock); + + if (trigger_data->net_dev) { + dev_put(trigger_data->net_dev); + trigger_data->net_dev = NULL; + } + + strncpy(trigger_data->device_name, buf, size); + if (size > 0 && trigger_data->device_name[size - 1] == '\n') + trigger_data->device_name[size - 1] = 0; + + if (trigger_data->device_name[0] != 0) + trigger_data->net_dev = + dev_get_by_name(&init_net, trigger_data->device_name); + + clear_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode); + if (trigger_data->net_dev != NULL) + if (netif_carrier_ok(trigger_data->net_dev)) + set_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode); + + trigger_data->last_activity = 0; + + set_baseline_state(trigger_data); + spin_unlock_bh(&trigger_data->lock); + + return size; +} + +static DEVICE_ATTR_RW(device_name); + +static ssize_t netdev_led_attr_show(struct device *dev, char *buf, + enum netdev_led_attr attr) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + int bit; + + switch (attr) { + case NETDEV_ATTR_LINK: + bit = NETDEV_LED_LINK; + break; + case NETDEV_ATTR_TX: + bit = NETDEV_LED_TX; + break; + case NETDEV_ATTR_RX: + bit = NETDEV_LED_RX; + break; + default: + return -EINVAL; + } + + return sprintf(buf, "%u\n", test_bit(bit, &trigger_data->mode)); +} + +static ssize_t netdev_led_attr_store(struct device *dev, const char *buf, + size_t size, enum netdev_led_attr attr) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + unsigned long state; + int ret; + int bit; + + ret = kstrtoul(buf, 0, &state); + if (ret) + return ret; + + switch (attr) { + case NETDEV_ATTR_LINK: + bit = NETDEV_LED_LINK; + break; + case NETDEV_ATTR_TX: + bit = NETDEV_LED_TX; + break; + case NETDEV_ATTR_RX: + bit = NETDEV_LED_RX; + break; + default: + return -EINVAL; + } + + cancel_delayed_work_sync(&trigger_data->work); + + if (state) + set_bit(bit, &trigger_data->mode); + else + clear_bit(bit, &trigger_data->mode); + + set_baseline_state(trigger_data); + + return size; +} + +static ssize_t link_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return netdev_led_attr_show(dev, buf, NETDEV_ATTR_LINK); +} + +static ssize_t link_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size) +{ + return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_LINK); +} + +static DEVICE_ATTR_RW(link); + +static ssize_t tx_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return netdev_led_attr_show(dev, buf, NETDEV_ATTR_TX); +} + +static ssize_t tx_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size) +{ + return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_TX); +} + +static DEVICE_ATTR_RW(tx); + +static ssize_t rx_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return netdev_led_attr_show(dev, buf, NETDEV_ATTR_RX); +} + +static ssize_t rx_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size) +{ + return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_RX); +} + +static DEVICE_ATTR_RW(rx); + +static ssize_t interval_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + + return sprintf(buf, "%u\n", + jiffies_to_msecs(atomic_read(&trigger_data->interval))); +} + +static ssize_t interval_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t size) +{ + struct led_classdev *led_cdev = dev_get_drvdata(dev); + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + unsigned long value; + int ret; + + ret = kstrtoul(buf, 0, &value); + if (ret) + return ret; + + /* impose some basic bounds on the timer interval */ + if (value >= 5 && value <= 10000) { + cancel_delayed_work_sync(&trigger_data->work); + + atomic_set(&trigger_data->interval, msecs_to_jiffies(value)); + set_baseline_state(trigger_data); /* resets timer */ + } + + return size; +} + +static DEVICE_ATTR_RW(interval); + +static int netdev_trig_notify(struct notifier_block *nb, + unsigned long evt, void *dv) +{ + struct net_device *dev = + netdev_notifier_info_to_dev((struct netdev_notifier_info *)dv); + struct led_netdev_data *trigger_data = container_of(nb, + struct + led_netdev_data, + notifier); + + if (evt != NETDEV_UP && evt != NETDEV_DOWN && evt != NETDEV_CHANGE + && evt != NETDEV_REGISTER && evt != NETDEV_UNREGISTER + && evt != NETDEV_CHANGENAME) + return NOTIFY_DONE; + + if (strcmp(dev->name, trigger_data->device_name)) + return NOTIFY_DONE; + + cancel_delayed_work_sync(&trigger_data->work); + + spin_lock_bh(&trigger_data->lock); + + clear_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode); + switch (evt) { + case NETDEV_REGISTER: + if (trigger_data->net_dev) + dev_put(trigger_data->net_dev); + dev_hold(dev); + trigger_data->net_dev = dev; + break; + case NETDEV_CHANGENAME: + case NETDEV_UNREGISTER: + if (trigger_data->net_dev) { + dev_put(trigger_data->net_dev); + trigger_data->net_dev = NULL; + } + break; + case NETDEV_UP: + case NETDEV_CHANGE: + if (netif_carrier_ok(dev)) + set_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode); + break; + } + + set_baseline_state(trigger_data); + + spin_unlock_bh(&trigger_data->lock); + + return NOTIFY_DONE; +} + +/* here's the real work! */ +static void netdev_trig_work(struct work_struct *work) +{ + struct led_netdev_data *trigger_data = container_of(work, + struct + led_netdev_data, + work.work); + struct rtnl_link_stats64 *dev_stats; + unsigned int new_activity; + struct rtnl_link_stats64 temp; + unsigned long interval; + int invert; + + /* If we dont have a device, insure we are off */ + if (!trigger_data->net_dev) { + led_set_brightness(trigger_data->led_cdev, LED_OFF); + return; + } + + /* If we are not looking for RX/TX then return */ + if (!test_bit(NETDEV_LED_TX, &trigger_data->mode) && + !test_bit(NETDEV_LED_RX, &trigger_data->mode)) + return; + + dev_stats = dev_get_stats(trigger_data->net_dev, &temp); + new_activity = + (test_bit(NETDEV_LED_TX, &trigger_data->mode) ? + dev_stats->tx_packets : 0) + + (test_bit(NETDEV_LED_RX, &trigger_data->mode) ? + dev_stats->rx_packets : 0); + + if (trigger_data->last_activity != new_activity) { + led_stop_software_blink(trigger_data->led_cdev); + + invert = test_bit(NETDEV_LED_LINK, &trigger_data->mode); + interval = jiffies_to_msecs( + atomic_read(&trigger_data->interval)); + /* base state is ON (link present) */ + led_blink_set_oneshot(trigger_data->led_cdev, + &interval, + &interval, + invert); + trigger_data->last_activity = new_activity; + } + + schedule_delayed_work(&trigger_data->work, + (atomic_read(&trigger_data->interval)*2)); +} + +static void netdev_trig_activate(struct led_classdev *led_cdev) +{ + struct led_netdev_data *trigger_data; + int rc; + + trigger_data = kzalloc(sizeof(struct led_netdev_data), GFP_KERNEL); + if (!trigger_data) + return; + + spin_lock_init(&trigger_data->lock); + + trigger_data->notifier.notifier_call = netdev_trig_notify; + trigger_data->notifier.priority = 10; + + INIT_DELAYED_WORK(&trigger_data->work, netdev_trig_work); + + trigger_data->led_cdev = led_cdev; + trigger_data->net_dev = NULL; + trigger_data->device_name[0] = 0; + + trigger_data->mode = 0; + atomic_set(&trigger_data->interval, msecs_to_jiffies(50)); + trigger_data->last_activity = 0; + + led_cdev->trigger_data = trigger_data; + + rc = device_create_file(led_cdev->dev, &dev_attr_device_name); + if (rc) + goto err_out; + rc = device_create_file(led_cdev->dev, &dev_attr_link); + if (rc) + goto err_out_device_name; + rc = device_create_file(led_cdev->dev, &dev_attr_rx); + if (rc) + goto err_out_link; + rc = device_create_file(led_cdev->dev, &dev_attr_tx); + if (rc) + goto err_out_rx; + rc = device_create_file(led_cdev->dev, &dev_attr_interval); + if (rc) + goto err_out_tx; + rc = register_netdevice_notifier(&trigger_data->notifier); + if (rc) + goto err_out_interval; + return; + +err_out_interval: + device_remove_file(led_cdev->dev, &dev_attr_interval); +err_out_tx: + device_remove_file(led_cdev->dev, &dev_attr_tx); +err_out_rx: + device_remove_file(led_cdev->dev, &dev_attr_rx); +err_out_link: + device_remove_file(led_cdev->dev, &dev_attr_link); +err_out_device_name: + device_remove_file(led_cdev->dev, &dev_attr_device_name); +err_out: + led_cdev->trigger_data = NULL; + kfree(trigger_data); +} + +static void netdev_trig_deactivate(struct led_classdev *led_cdev) +{ + struct led_netdev_data *trigger_data = led_cdev->trigger_data; + + if (trigger_data) { + unregister_netdevice_notifier(&trigger_data->notifier); + + device_remove_file(led_cdev->dev, &dev_attr_device_name); + device_remove_file(led_cdev->dev, &dev_attr_link); + device_remove_file(led_cdev->dev, &dev_attr_rx); + device_remove_file(led_cdev->dev, &dev_attr_tx); + device_remove_file(led_cdev->dev, &dev_attr_interval); + + cancel_delayed_work_sync(&trigger_data->work); + + if (trigger_data->net_dev) + dev_put(trigger_data->net_dev); + + kfree(trigger_data); + } +} + +static struct led_trigger netdev_led_trigger = { + .name = "netdev", + .activate = netdev_trig_activate, + .deactivate = netdev_trig_deactivate, +}; + +static int __init netdev_trig_init(void) +{ + return led_trigger_register(&netdev_led_trigger); +} + +static void __exit netdev_trig_exit(void) +{ + led_trigger_unregister(&netdev_led_trigger); +} + +module_init(netdev_trig_init); +module_exit(netdev_trig_exit); + +MODULE_AUTHOR("Ben Whitten "); +MODULE_AUTHOR("Oliver Jowett "); +MODULE_DESCRIPTION("Netdev LED trigger"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/leds/trigger/ledtrig-transient.c b/drivers/leds/trigger/ledtrig-transient.c index 7e6011bd36465..9d1769073562f 100644 --- a/drivers/leds/trigger/ledtrig-transient.c +++ b/drivers/leds/trigger/ledtrig-transient.c @@ -1,22 +1,15 @@ -/* - * LED Kernel Transient Trigger - * - * Copyright (C) 2012 Shuah Khan - * - * Based on Richard Purdie's ledtrig-timer.c and Atsushi Nemoto's - * ledtrig-heartbeat.c - * Design and use-case input from Jonas Bonn and - * Neil Brown - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - */ -/* - * Transient trigger allows one shot timer activation. Please refer to - * Documentation/leds/ledtrig-transient.txt for details -*/ +// SPDX-License-Identifier: GPL-2.0 +// +// LED Kernel Transient Trigger +// +// Transient trigger allows one shot timer activation. Please refer to +// Documentation/leds/ledtrig-transient.txt for details +// Copyright (C) 2012 Shuah Khan +// +// Based on Richard Purdie's ledtrig-timer.c and Atsushi Nemoto's +// ledtrig-heartbeat.c +// Design and use-case input from Jonas Bonn and +// Neil Brown #include #include @@ -33,12 +26,14 @@ struct transient_trig_data { int restore_state; unsigned long duration; struct timer_list timer; + struct led_classdev *led_cdev; }; -static void transient_timer_function(unsigned long data) +static void transient_timer_function(struct timer_list *t) { - struct led_classdev *led_cdev = (struct led_classdev *) data; - struct transient_trig_data *transient_data = led_cdev->trigger_data; + struct transient_trig_data *transient_data = + from_timer(transient_data, t, timer); + struct led_classdev *led_cdev = transient_data->led_cdev; transient_data->activate = 0; led_set_brightness_nosleep(led_cdev, transient_data->restore_state); @@ -169,6 +164,7 @@ static void transient_trig_activate(struct led_classdev *led_cdev) return; } led_cdev->trigger_data = tdata; + tdata->led_cdev = led_cdev; rc = device_create_file(led_cdev->dev, &dev_attr_activate); if (rc) @@ -182,8 +178,7 @@ static void transient_trig_activate(struct led_classdev *led_cdev) if (rc) goto err_out_state; - setup_timer(&tdata->timer, transient_timer_function, - (unsigned long) led_cdev); + timer_setup(&tdata->timer, transient_timer_function, 0); led_cdev->activated = true; return; @@ -236,4 +231,4 @@ module_exit(transient_trig_exit); MODULE_AUTHOR("Shuah Khan "); MODULE_DESCRIPTION("Transient LED trigger"); -MODULE_LICENSE("GPL"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c index 566caca4efd8e..7b2f9b85e3497 100644 --- a/drivers/mfd/stmpe.c +++ b/drivers/mfd/stmpe.c @@ -1,9 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * ST Microelectronics MFD: stmpe's driver * * Copyright (C) ST-Ericsson SA 2010 * - * License Terms: GNU General Public License, version 2 * Author: Rabin Vincent for ST-Ericsson */ @@ -463,6 +463,28 @@ static const struct mfd_cell stmpe_ts_cell = { .num_resources = ARRAY_SIZE(stmpe_ts_resources), }; +/* + * ADC (STMPE811) + */ + +static struct resource stmpe_adc_resources[] = { + { + .name = "STMPE_TEMP_SENS", + .flags = IORESOURCE_IRQ, + }, + { + .name = "STMPE_ADC", + .flags = IORESOURCE_IRQ, + }, +}; + +static const struct mfd_cell stmpe_adc_cell = { + .name = "stmpe-adc", + .of_compatible = "st,stmpe-adc", + .resources = stmpe_adc_resources, + .num_resources = ARRAY_SIZE(stmpe_adc_resources), +}; + /* * STMPE811 or STMPE610 */ @@ -497,6 +519,11 @@ static struct stmpe_variant_block stmpe811_blocks[] = { .irq = STMPE811_IRQ_TOUCH_DET, .block = STMPE_BLOCK_TOUCHSCREEN, }, + { + .cell = &stmpe_adc_cell, + .irq = STMPE811_IRQ_TEMP_SENS, + .block = STMPE_BLOCK_ADC, + }, }; static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks, @@ -517,6 +544,35 @@ static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks, enable ? 0 : mask); } +int stmpe811_adc_common_init(struct stmpe *stmpe) +{ + int ret; + u8 adc_ctrl1, adc_ctrl1_mask; + + adc_ctrl1 = STMPE_SAMPLE_TIME(stmpe->sample_time) | + STMPE_MOD_12B(stmpe->mod_12b) | + STMPE_REF_SEL(stmpe->ref_sel); + adc_ctrl1_mask = STMPE_SAMPLE_TIME(0xff) | STMPE_MOD_12B(0xff) | + STMPE_REF_SEL(0xff); + + ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL1, + adc_ctrl1_mask, adc_ctrl1); + if (ret) { + dev_err(stmpe->dev, "Could not setup ADC\n"); + return ret; + } + + ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL2, + STMPE_ADC_FREQ(0xff), STMPE_ADC_FREQ(stmpe->adc_freq)); + if (ret) { + dev_err(stmpe->dev, "Could not setup ADC\n"); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(stmpe811_adc_common_init); + static int stmpe811_get_altfunc(struct stmpe *stmpe, enum stmpe_block block) { /* 0 for touchscreen, 1 for GPIO */ @@ -1325,6 +1381,7 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) struct device_node *np = ci->dev->of_node; struct stmpe *stmpe; int ret; + u32 val; pdata = devm_kzalloc(ci->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -1342,6 +1399,15 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) mutex_init(&stmpe->irq_lock); mutex_init(&stmpe->lock); + if (!of_property_read_u32(np, "st,sample-time", &val)) + stmpe->sample_time = val; + if (!of_property_read_u32(np, "st,mod-12b", &val)) + stmpe->mod_12b = val; + if (!of_property_read_u32(np, "st,ref-sel", &val)) + stmpe->ref_sel = val; + if (!of_property_read_u32(np, "st,adc-freq", &val)) + stmpe->adc_freq = val; + stmpe->dev = ci->dev; stmpe->client = ci->client; stmpe->pdata = pdata; @@ -1433,6 +1499,8 @@ int stmpe_remove(struct stmpe *stmpe) if (!IS_ERR(stmpe->vcc)) regulator_disable(stmpe->vcc); + __stmpe_disable(stmpe, STMPE_BLOCK_ADC); + mfd_remove_devices(stmpe->dev); return 0; diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 8136dc7e863d7..f1c4c891cda3c 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -506,6 +506,36 @@ config PCI_ENDPOINT_TEST Enable this configuration option to enable the host side test driver for PCI Endpoint. +config BONE_CAPEMGR + tristate "Beaglebone cape manager" + depends on ARCH_OMAP2PLUS && OF + select EEPROM + select OF_OVERLAY + help + Say Y here to include support for automatic loading of + beaglebone capes. Select M to build as a module which + will be named bone_capemgr. + +config DEV_OVERLAYMGR + tristate "Device overlay manager" + depends on OF + select OF_OVERLAY + default n + help + Say Y here to include support for the automagical dev + overlay manager. + +config TIEQEP + tristate "EQEP Hardware quadrature encoder controller" + depends on SOC_AM33XX + select PWM_TIPWMSS + help + Driver support for the EQEP quadrature encoder controller AM33XX + TI SOC + + To compile this driver as a module, choose M here: the module + will be called tieqep. + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" @@ -516,6 +546,7 @@ source "drivers/misc/mei/Kconfig" source "drivers/misc/vmw_vmci/Kconfig" source "drivers/misc/mic/Kconfig" source "drivers/misc/genwqe/Kconfig" +source "drivers/misc/cape/Kconfig" source "drivers/misc/echo/Kconfig" source "drivers/misc/cxl/Kconfig" endmenu diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index ad0e64fdba348..3225f1e37dfc0 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -43,6 +43,9 @@ obj-y += ti-st/ obj-y += lis3lv02d/ obj-$(CONFIG_USB_SWITCH_FSA9480) += fsa9480.o obj-$(CONFIG_ALTERA_STAPL) +=altera-stapl/ +obj-$(CONFIG_TIEQEP) += tieqep.o +obj-$(CONFIG_BONE_CAPEMGR) += bone_capemgr.o +obj-$(CONFIG_DEV_OVERLAYMGR) += devovmgr.o obj-$(CONFIG_INTEL_MEI) += mei/ obj-$(CONFIG_VMWARE_VMCI) += vmw_vmci/ obj-$(CONFIG_LATTICE_ECP3_CONFIG) += lattice-ecp3-config.o @@ -50,6 +53,7 @@ obj-$(CONFIG_SRAM) += sram.o obj-$(CONFIG_SRAM_EXEC) += sram-exec.o obj-y += mic/ obj-$(CONFIG_GENWQE) += genwqe/ +obj-y += cape/ obj-$(CONFIG_ECHO) += echo/ obj-$(CONFIG_VEXPRESS_SYSCFG) += vexpress-syscfg.o obj-$(CONFIG_CXL_BASE) += cxl/ diff --git a/drivers/misc/bone_capemgr.c b/drivers/misc/bone_capemgr.c new file mode 100644 index 0000000000000..0262a46f2a917 --- /dev/null +++ b/drivers/misc/bone_capemgr.c @@ -0,0 +1,1898 @@ +/* + * TI Beaglebone cape manager + * + * Copyright (C) 2012 Texas Instruments Inc. + * Copyright (C) 2012-2015 Konsulko Group. + * Author: Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* disabled capes */ +static char *disable_partno; +module_param(disable_partno, charp, 0444); +MODULE_PARM_DESC(disable_partno, + "Comma delimited list of PART-NUMBER[:REV] of disabled capes"); + +/* enable capes */ +static char *enable_partno; +module_param(enable_partno, charp, 0444); +MODULE_PARM_DESC(enable_partno, + "Comma delimited list of PART-NUMBER[:REV] of enabled capes"); + +/* delay to scan on boot until rootfs appears */ +static int boot_scan_period = 1000; +module_param(boot_scan_period, int, 0444); +MODULE_PARM_DESC(boot_scan_period, + "boot scan period until rootfs firmware is available"); + +static int uboot_capemgr_enabled = 0; +module_param(uboot_capemgr_enabled, int, 0444); +MODULE_PARM_DESC(uboot_capemgr_enabled, + "U-Boot Cape Manager is enabled (0=Kernel Cape Manager [default], 1=Disable Kernel Cape Manager)"); + +struct capemgr_info; + +struct slot_ee_attribute { + struct device_attribute devattr; + unsigned int field; + struct bone_cape_slot *slot; /* this is filled when instantiated */ +}; +#define to_slot_ee_attribute(x) \ + container_of((x), struct slot_ee_attribute, devattr) + +struct bbrd_ee_attribute { + struct device_attribute devattr; + unsigned int field; +}; +#define to_bbrd_ee_attribute(x) \ + container_of((x), struct bbrd_ee_attribute, devattr) + +struct bone_cape_slot { + struct list_head node; + struct capemgr_info *info; + int slotno; + struct nvmem_cell *nvmem_cell; + + char text_id[256]; + char signature[256]; + /* quick access */ + char board_name[32+1]; + char version[4+1]; + char manufacturer[16+1]; + char part_number[16+1]; + + /* attribute group */ + char *ee_attr_name; + int ee_attrs_count; + struct slot_ee_attribute *ee_attrs; + struct attribute **ee_attrs_tab; + struct attribute_group attrgroup; + + /* state flags */ + unsigned int probed : 1; + unsigned int probe_failed : 1; + unsigned int override : 1; + unsigned int loading : 1; + unsigned int loaded : 1; + unsigned int retry_loading : 1; + unsigned int disabled : 1; + + char *dtbo; + const struct firmware *fw; + struct device_node *overlay; + int overlay_id; + + /* loader thread */ + struct task_struct *loader_thread; + + /* load priority */ + int priority; +}; + +struct bone_baseboard { + + /* from the matched boardmap node */ + char *compatible_name; + + /* filled in by reading the eeprom */ + char signature[256]; + char text_id[64+1]; + + /* quick access */ + char board_name[8+1]; + char revision[4+1]; + char serial_number[12+1]; + + /* access to the eeprom */ + struct nvmem_cell *nvmem_cell; +}; + +struct capemgr_info { + struct platform_device *pdev; + + atomic_t next_slot_nr; + struct list_head slot_list; + struct mutex slots_list_mutex; + + /* baseboard EEPROM data */ + struct bone_baseboard baseboard; + + /* wait queue for keeping the priorities straight */ + wait_queue_head_t load_wq; +}; + +static int bone_slot_fill_override(struct bone_cape_slot *slot, + const char *part_number, const char *version); +static struct bone_cape_slot *capemgr_add_slot( + struct capemgr_info *info, const char *slot_name, + const char *part_number, const char *version, int prio); +static int capemgr_remove_slot_no_lock(struct bone_cape_slot *slot); +static int capemgr_remove_slot(struct bone_cape_slot *slot); +static int capemgr_load_slot(struct bone_cape_slot *slot); +static int capemgr_unload_slot(struct bone_cape_slot *slot); + +/* baseboard EEPROM field definition */ +#define BBRD_EE_FIELD_HEADER 0 +#define BBRD_EE_FIELD_BOARD_NAME 1 +#define BBRD_EE_FIELD_REVISION 2 +#define BBRD_EE_FIELD_SERIAL_NUMBER 3 +#define BBRD_EE_FIELD_CONFIG_OPTION 4 +#define BBRD_EE_FILED_RSVD1 5 +#define BBRD_EE_FILED_RSVD2 6 +#define BBRD_EE_FILED_RSVD3 7 + +/* cape EEPROM field definitions */ +#define CAPE_EE_FIELD_HEADER 0 +#define CAPE_EE_FIELD_EEPROM_REV 1 +#define CAPE_EE_FIELD_BOARD_NAME 2 +#define CAPE_EE_FIELD_VERSION 3 +#define CAPE_EE_FIELD_MANUFACTURER 4 +#define CAPE_EE_FIELD_PART_NUMBER 5 +#define CAPE_EE_FIELD_NUMBER_OF_PINS 6 +#define CAPE_EE_FIELD_SERIAL_NUMBER 7 +#define CAPE_EE_FIELD_PIN_USAGE 8 +#define CAPE_EE_FIELD_VDD_3V3EXP 9 +#define CAPE_EE_FIELD_VDD_5V 10 +#define CAPE_EE_FIELD_SYS_5V 11 +#define CAPE_EE_FIELD_DC_SUPPLIED 12 +#define CAPE_EE_FIELD_FIELDS_NR 13 + +#define EE_FIELD_MAKE_HEADER(p) \ + ({ \ + const u8 *_p = (p); \ + (((u32)_p[0] << 24) | ((u32)_p[1] << 16) | \ + ((u32)_p[2] << 8) | (u32)_p[3]); \ + }) + +#define EE_FIELD_HEADER_VALID 0xaa5533ee + +struct ee_field { + const char *name; + int start; + int size; + unsigned int ascii : 1; + unsigned int strip_trailing_dots : 1; + const char *override; +}; + +/* baseboard EEPROM definitions */ +static const struct ee_field bbrd_sig_fields[] = { + [BBRD_EE_FIELD_HEADER] = { + .name = "header", + .start = 0, + .size = 4, + .ascii = 0, + .override = "\xaa\x55\x33\xee", /* AA 55 33 EE */ + }, + [BBRD_EE_FIELD_BOARD_NAME] = { + .name = "board-name", + .start = 4, + .size = 8, + .ascii = 1, + .strip_trailing_dots = 1, + .override = "Board Name", + }, + [BBRD_EE_FIELD_REVISION] = { + .name = "revision", + .start = 12, + .size = 4, + .ascii = 1, + .override = "00A0", + }, + [BBRD_EE_FIELD_SERIAL_NUMBER] = { + .name = "serial-number", + .start = 16, + .size = 12, + .ascii = 1, + .override = "0000000000", + }, + [BBRD_EE_FIELD_CONFIG_OPTION] = { + .name = "config-option", + .start = 28, + .size = 32, + }, +}; + +/* cape EEPROM definitions */ +static const struct ee_field cape_sig_fields[] = { + [CAPE_EE_FIELD_HEADER] = { + .name = "header", + .start = 0, + .size = 4, + .ascii = 0, + .override = "\xaa\x55\x33\xee", /* AA 55 33 EE */ + }, + [CAPE_EE_FIELD_EEPROM_REV] = { + .name = "eeprom-format-revision", + .start = 4, + .size = 2, + .ascii = 1, + .override = "A0", + }, + [CAPE_EE_FIELD_BOARD_NAME] = { + .name = "board-name", + .start = 6, + .size = 32, + .ascii = 1, + .strip_trailing_dots = 1, + .override = "Override Board Name", + }, + [CAPE_EE_FIELD_VERSION] = { + .name = "version", + .start = 38, + .size = 4, + .ascii = 1, + .override = "00A0", + }, + [CAPE_EE_FIELD_MANUFACTURER] = { + .name = "manufacturer", + .start = 42, + .size = 16, + .ascii = 1, + .strip_trailing_dots = 1, + .override = "Override Manuf", + }, + [CAPE_EE_FIELD_PART_NUMBER] = { + .name = "part-number", + .start = 58, + .size = 16, + .ascii = 1, + .strip_trailing_dots = 1, + .override = "Override Part#", + }, + [CAPE_EE_FIELD_NUMBER_OF_PINS] = { + .name = "number-of-pins", + .start = 74, + .size = 2, + .ascii = 0, + .override = NULL, + }, + [CAPE_EE_FIELD_SERIAL_NUMBER] = { + .name = "serial-number", + .start = 76, + .size = 12, + .ascii = 1, + .override = "0000000000", + }, + [CAPE_EE_FIELD_PIN_USAGE] = { + .name = "pin-usage", + .start = 88, + .size = 140, + .ascii = 0, + .override = NULL, + }, + [CAPE_EE_FIELD_VDD_3V3EXP] = { + .name = "vdd-3v3exp", + .start = 228, + .size = 2, + .ascii = 0, + .override = NULL, + }, + [CAPE_EE_FIELD_VDD_5V] = { + .name = "vdd-5v", + .start = 230, + .size = 2, + .ascii = 0, + .override = NULL, + }, + [CAPE_EE_FIELD_SYS_5V] = { + .name = "sys-5v", + .start = 232, + .size = 2, + .ascii = 0, + .override = NULL, + }, + [CAPE_EE_FIELD_DC_SUPPLIED] = { + .name = "dc-supplied", + .start = 234, + .size = 2, + .ascii = 0, + .override = NULL, + }, +}; + +static char *ee_field_get(const struct ee_field *sig_field, + const void *data, int field, char *buf, int bufsz) +{ + int len; + + /* enough space? */ + if (bufsz < sig_field->size + sig_field->ascii) + return NULL; + + memcpy(buf, (char *)data + sig_field->start, sig_field->size); + + /* terminate ascii field */ + if (sig_field->ascii) + buf[sig_field->size] = '\0'; + + if (sig_field->strip_trailing_dots) { + len = strlen(buf); + while (len > 1 && buf[len - 1] == '.') + buf[--len] = '\0'; + } + + return buf; +} + +char *bbrd_ee_field_get(const void *data, + int field, char *buf, int bufsz) +{ + if ((unsigned int)field >= ARRAY_SIZE(bbrd_sig_fields)) + return NULL; + + return ee_field_get(&bbrd_sig_fields[field], data, field, buf, bufsz); +} + +char *cape_ee_field_get(const void *data, + int field, char *buf, int bufsz) +{ + if ((unsigned int)field >= ARRAY_SIZE(cape_sig_fields)) + return NULL; + + return ee_field_get(&cape_sig_fields[field], data, field, buf, bufsz); +} + +#ifdef CONFIG_OF +static const struct of_device_id capemgr_of_match[] = { + { + .compatible = "ti,bone-capemgr", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, capemgr_of_match); + +#endif + +static int bone_baseboard_scan(struct bone_baseboard *bbrd) +{ + struct capemgr_info *info = container_of(bbrd, + struct capemgr_info, baseboard); + const u8 *p; + int ret; + size_t len; + + p = nvmem_cell_read(bbrd->nvmem_cell, &len); + if (IS_ERR(p)) { + ret = PTR_ERR(p); + dev_err(&info->pdev->dev, + "Cannot read cell (ret=%d)\n", ret); + return ret; + } + if (len < sizeof(bbrd->signature)) { + dev_info(&info->pdev->dev, + "Short read %d (should be >= %d bytes)\n", + len, sizeof(bbrd->signature)); + return -EINVAL; + } + memcpy(bbrd->signature, p, sizeof(bbrd->signature)); + + p = bbrd->signature; + if (EE_FIELD_MAKE_HEADER(p) != EE_FIELD_HEADER_VALID) { + dev_err(&info->pdev->dev, "Invalid board signature '%08x'\n", + EE_FIELD_MAKE_HEADER(p)); + return -ENODEV; + } + + bbrd_ee_field_get(bbrd->signature, + BBRD_EE_FIELD_BOARD_NAME, + bbrd->board_name, sizeof(bbrd->board_name)); + bbrd_ee_field_get(bbrd->signature, + BBRD_EE_FIELD_REVISION, + bbrd->revision, sizeof(bbrd->revision)); + bbrd_ee_field_get(bbrd->signature, + BBRD_EE_FIELD_SERIAL_NUMBER, + bbrd->serial_number, sizeof(bbrd->serial_number)); + + /* board_name,version,manufacturer,part_number */ + snprintf(bbrd->text_id, sizeof(bbrd->text_id) - 1, + "%s,%s,%s", bbrd->board_name, bbrd->revision, + bbrd->serial_number); + + /* terminate always */ + bbrd->text_id[sizeof(bbrd->text_id) - 1] = '\0'; + + return 0; +} + +static int bone_slot_scan(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + const u8 *p; + int r; + ssize_t len; + + /* need to read EEPROM? */ + if (slot->probed) + goto slot_fail_check; + + if (uboot_capemgr_enabled) + goto slot_fail_check; + + slot->probed = 1; + + if (!slot->override) { + + p = nvmem_cell_read(slot->nvmem_cell, &len); + if (IS_ERR(p)) { + r = PTR_ERR(p); + slot->probe_failed = 1; + + /* timeout is normal when no cape is present */ + if (r != -ETIMEDOUT) + dev_err(&info->pdev->dev, + "Cannot read cell (ret=%d)\n", r); + return r; + } + if (len < sizeof(slot->signature)) { + dev_info(&info->pdev->dev, + "Short read %d (should be >= %d bytes)\n", + len, sizeof(slot->signature)); + return -EINVAL; + } + memcpy(slot->signature, p, sizeof(slot->signature)); + + } else + dev_info(&info->pdev->dev, + "Using override eeprom data at slot %d\n", + slot->slotno); + + p = slot->signature; + if (EE_FIELD_MAKE_HEADER(p) != EE_FIELD_HEADER_VALID) { + dev_err(&info->pdev->dev, + "Invalid signature '%08x' at slot %d\n", + EE_FIELD_MAKE_HEADER(p), slot->slotno); + slot->probe_failed = 1; + return -ENODEV; + } + + cape_ee_field_get(slot->signature, + CAPE_EE_FIELD_BOARD_NAME, + slot->board_name, sizeof(slot->board_name)); + cape_ee_field_get(slot->signature, + CAPE_EE_FIELD_VERSION, + slot->version, sizeof(slot->version)); + cape_ee_field_get(slot->signature, + CAPE_EE_FIELD_MANUFACTURER, + slot->manufacturer, sizeof(slot->manufacturer)); + cape_ee_field_get(slot->signature, + CAPE_EE_FIELD_PART_NUMBER, + slot->part_number, sizeof(slot->part_number)); + + /* board_name,version,manufacturer,part_number */ + snprintf(slot->text_id, sizeof(slot->text_id) - 1, + "%s,%s,%s,%s", slot->board_name, slot->version, + slot->manufacturer, slot->part_number); + + /* terminate always */ + slot->text_id[sizeof(slot->text_id) - 1] = '\0'; + +slot_fail_check: + /* slot has failed and we don't support hotpluging */ + if (slot->probe_failed) + return -ENODEV; + + return 0; +} + +/* return 0 if not matched,, 1 if matched */ +static int bone_match_cape(const char *match, + const char *part_number, const char *version) +{ + char *tmp_part_number, *tmp_version; + char *buf, *s, *e, *sn; + int found; + + if (match == NULL || part_number == NULL) + return 0; + + /* copy the argument to work on it */ + buf = kstrdup(match, GFP_KERNEL); + + /* no memory, too bad... */ + if (buf == NULL) + return 0; + + found = 0; + s = buf; + e = s + strlen(s); + while (s < e) { + /* find comma separator */ + sn = strchr(s, ','); + if (sn != NULL) + *sn++ = '\0'; + else + sn = e; + tmp_part_number = s; + tmp_version = strchr(tmp_part_number, ':'); + if (tmp_version != NULL) + *tmp_version++ = '\0'; + s = sn; + + /* the part names must match */ + if (strcmp(tmp_part_number, part_number) != 0) + continue; + + /* if there's no version, match any */ + if (version == NULL || tmp_version == NULL || + strcmp(version, tmp_version) == 0) { + found = 1; + break; + } + } + + kfree(buf); + + return found; +} + +/* helper method */ +static int of_multi_prop_cmp(const struct property *prop, const char *value) +{ + const char *cp; + int cplen, vlen, l; + + /* check if it's directly compatible */ + cp = prop->value; + cplen = prop->length; + vlen = strlen(value); + + while (cplen > 0) { + /* compatible? */ + if (of_compat_cmp(cp, value, vlen) == 0) + return 0; + l = strlen(cp) + 1; + cp += l; + cplen -= l; + } + return -1; +} + +static ssize_t slot_ee_attr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct slot_ee_attribute *ee_attr = to_slot_ee_attribute(attr); + struct bone_cape_slot *slot = ee_attr->slot; + const struct ee_field *sig_field; + int i, len; + char *p, *s; + u16 val; + + /* add newline for ascii fields */ + sig_field = &cape_sig_fields[ee_attr->field]; + + len = sig_field->size + sig_field->ascii; + p = kmalloc(len, GFP_KERNEL); + if (p == NULL) + return -ENOMEM; + + s = cape_ee_field_get(slot->signature, ee_attr->field, p, len); + if (s == NULL) + return -EINVAL; + + /* add newline for ascii fields and return */ + if (sig_field->ascii) { + len = sprintf(buf, "%s\n", s); + goto out; + } + + /* case by case handling */ + switch (ee_attr->field) { + case CAPE_EE_FIELD_HEADER: + len = sprintf(buf, "%02x %02x %02x %02x\n", + s[0], s[1], s[2], s[3]); + break; + + /* 2 bytes */ + case CAPE_EE_FIELD_NUMBER_OF_PINS: + case CAPE_EE_FIELD_VDD_3V3EXP: + case CAPE_EE_FIELD_VDD_5V: + case CAPE_EE_FIELD_SYS_5V: + case CAPE_EE_FIELD_DC_SUPPLIED: + /* the bone is LE */ + val = s[0] & (s[1] << 8); + len = sprintf(buf, "%u\n", (unsigned int)val & 0xffff); + break; + + case CAPE_EE_FIELD_PIN_USAGE: + + len = 0; + for (i = 0; i < sig_field->size / 2; i++) { + /* the bone is LE */ + val = s[0] & (s[1] << 8); + sprintf(buf, "%04x\n", val); + buf += 5; + len += 5; + s += 2; + } + + break; + + default: + *buf = '\0'; + len = 0; + break; + } + +out: + kfree(p); + + return len; +} + +#define SLOT_EE_ATTR(_name, _field) \ + { \ + .devattr = __ATTR(_name, S_IRUGO, slot_ee_attr_show, NULL), \ + .field = CAPE_EE_FIELD_##_field, \ + .slot = NULL, \ + } + +static const struct slot_ee_attribute slot_ee_attrs[] = { + SLOT_EE_ATTR(header, HEADER), + SLOT_EE_ATTR(eeprom-format-revision, EEPROM_REV), + SLOT_EE_ATTR(board-name, BOARD_NAME), + SLOT_EE_ATTR(version, VERSION), + SLOT_EE_ATTR(manufacturer, MANUFACTURER), + SLOT_EE_ATTR(part-number, PART_NUMBER), + SLOT_EE_ATTR(number-of-pins, NUMBER_OF_PINS), + SLOT_EE_ATTR(serial-number, SERIAL_NUMBER), + SLOT_EE_ATTR(pin-usage, PIN_USAGE), + SLOT_EE_ATTR(vdd-3v3exp, VDD_3V3EXP), + SLOT_EE_ATTR(vdd-5v, VDD_5V), + SLOT_EE_ATTR(sys-5v, SYS_5V), + SLOT_EE_ATTR(dc-supplied, DC_SUPPLIED), +}; + +static int bone_cape_slot_sysfs_register(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + struct slot_ee_attribute *ee_attr; + struct attribute_group *attrgroup; + int i, err, sz; + + slot->ee_attr_name = kasprintf(GFP_KERNEL, "slot-%d", slot->slotno); + if (slot->ee_attr_name == NULL) { + dev_err(dev, "slot #%d: Failed to allocate ee_attr_name\n", + slot->slotno); + err = -ENOMEM; + goto err_fail_no_ee_attr_name; + } + + slot->ee_attrs_count = ARRAY_SIZE(slot_ee_attrs); + + sz = slot->ee_attrs_count * sizeof(*slot->ee_attrs); + slot->ee_attrs = kmalloc(sz, GFP_KERNEL); + if (slot->ee_attrs == NULL) { + dev_err(dev, "slot #%d: Failed to allocate ee_attrs\n", + slot->slotno); + err = -ENOMEM; + goto err_fail_no_ee_attrs; + } + + attrgroup = &slot->attrgroup; + memset(attrgroup, 0, sizeof(*attrgroup)); + attrgroup->name = slot->ee_attr_name; + + sz = sizeof(*slot->ee_attrs_tab) * (slot->ee_attrs_count + 1); + attrgroup->attrs = kmalloc(sz, GFP_KERNEL); + if (attrgroup->attrs == NULL) { + dev_err(dev, "slot #%d: Failed to allocate ee_attrs_tab\n", + slot->slotno); + err = -ENOMEM; + goto err_fail_no_ee_attrs_tab; + } + /* copy everything over */ + memcpy(slot->ee_attrs, slot_ee_attrs, sizeof(slot_ee_attrs)); + + /* bind this attr to the slot */ + for (i = 0; i < slot->ee_attrs_count; i++) { + ee_attr = &slot->ee_attrs[i]; + ee_attr->slot = slot; + attrgroup->attrs[i] = &ee_attr->devattr.attr; + } + attrgroup->attrs[i] = NULL; + + /* make lockdep happy */ + for (i = 0; i < slot->ee_attrs_count; i++) { + ee_attr = &slot->ee_attrs[i]; + sysfs_attr_init(&ee_attr->devattr.attr); + } + + err = sysfs_create_group(&dev->kobj, attrgroup); + if (err != 0) { + dev_err(dev, "slot #%d: Failed to allocate ee_attrs_tab\n", + slot->slotno); + err = -ENOMEM; + goto err_fail_no_ee_attrs_group; + } + + return 0; + +err_fail_no_ee_attrs_group: + kfree(slot->ee_attrs_tab); +err_fail_no_ee_attrs_tab: + kfree(slot->ee_attrs); +err_fail_no_ee_attrs: + kfree(slot->ee_attr_name); +err_fail_no_ee_attr_name: + return err; +} + +static void bone_cape_slot_sysfs_unregister(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + + sysfs_remove_group(&dev->kobj, &slot->attrgroup); + kfree(slot->ee_attrs_tab); + kfree(slot->ee_attrs); + kfree(slot->ee_attr_name); +} + +static ssize_t bbrd_ee_attr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct bbrd_ee_attribute *ee_attr = to_bbrd_ee_attribute(attr); + struct platform_device *pdev = to_platform_device(dev); + struct capemgr_info *info = platform_get_drvdata(pdev); + struct bone_baseboard *bbrd = &info->baseboard; + const struct ee_field *sig_field; + u16 val; + int i, len; + char *p, *s; + + /* add newline for ascii fields */ + sig_field = &bbrd_sig_fields[ee_attr->field]; + + len = sig_field->size + sig_field->ascii; + p = kmalloc(len, GFP_KERNEL); + if (p == NULL) + return -ENOMEM; + + s = bbrd_ee_field_get(bbrd->signature, ee_attr->field, p, len); + if (s == NULL) + return -EINVAL; + + /* add newline for ascii fields and return */ + if (sig_field->ascii) { + len = sprintf(buf, "%s\n", s); + goto out; + } + + /* case by case handling */ + switch (ee_attr->field) { + case BBRD_EE_FIELD_HEADER: + len = sprintf(buf, "%02x %02x %02x %02x\n", + s[0], s[1], s[2], s[3]); + break; + + case BBRD_EE_FIELD_CONFIG_OPTION: + len = 0; + for (i = 0; i < sig_field->size / 2; i++) { + /* the bone is LE */ + val = s[0] & (s[1] << 8); + sprintf(buf, "%04x\n", val); + buf += 5; + len += 5; + s += 2; + } + break; + + default: + *buf = '\0'; + len = 0; + break; + } + +out: + kfree(p); + + return len; +} + +#define BBRD_EE_ATTR(_name, _field) \ + { \ + .devattr = __ATTR(_name, 0440, bbrd_ee_attr_show, NULL), \ + .field = BBRD_EE_FIELD_##_field, \ + } + +static struct bbrd_ee_attribute bbrd_ee_attrs[] = { + BBRD_EE_ATTR(header, HEADER), + BBRD_EE_ATTR(board-name, BOARD_NAME), + BBRD_EE_ATTR(revision, REVISION), + BBRD_EE_ATTR(serial-number, SERIAL_NUMBER), + BBRD_EE_ATTR(config-option, CONFIG_OPTION), +}; + +static struct attribute *bbrd_attrs_flat[] = { + &bbrd_ee_attrs[BBRD_EE_FIELD_HEADER].devattr.attr, + &bbrd_ee_attrs[BBRD_EE_FIELD_BOARD_NAME].devattr.attr, + &bbrd_ee_attrs[BBRD_EE_FIELD_REVISION].devattr.attr, + &bbrd_ee_attrs[BBRD_EE_FIELD_SERIAL_NUMBER].devattr.attr, + &bbrd_ee_attrs[BBRD_EE_FIELD_CONFIG_OPTION].devattr.attr, + NULL, +}; + +static const struct attribute_group bbrd_attr_group = { + .name = "baseboard", + .attrs = bbrd_attrs_flat, +}; + +static ssize_t slots_show(struct device *dev, struct device_attribute *attr, + char *buf); +static ssize_t slots_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count); + +static DEVICE_ATTR(slots, 0644, slots_show, slots_store); + +static struct attribute *root_attrs_flat[] = { + &dev_attr_slots.attr, + NULL, +}; + +static const struct attribute_group root_attr_group = { + .attrs = root_attrs_flat, +}; + +static const struct attribute_group *attr_groups[] = { + &root_attr_group, + &bbrd_attr_group, + NULL, +}; + +static ssize_t slots_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct capemgr_info *info = platform_get_drvdata(pdev); + struct bone_cape_slot *slot; + ssize_t len, sz; + + mutex_lock(&info->slots_list_mutex); + sz = 0; + list_for_each_entry(slot, &info->slot_list, node) { + + len = sprintf(buf, "%2d: %c%c%c%c%c%c %3d %s\n", + slot->slotno, + slot->probed ? 'P' : '-', + slot->probe_failed ? 'F' : '-', + slot->override ? 'O' : '-', + slot->loading ? 'l' : '-', + slot->loaded ? 'L' : '-', + slot->disabled ? 'D' : '-', + slot->overlay_id, slot->text_id); + + buf += len; + sz += len; + } + mutex_unlock(&info->slots_list_mutex); + + return sz; +} + +static ssize_t slots_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + struct capemgr_info *info = platform_get_drvdata(pdev); + struct bone_cape_slot *slot; + struct device_node *pnode, *node; + char *s, *part_number, *version; + int ret; + int slotno; + + /* check for remove slot */ + if (strlen(buf) > 0 && buf[0] == '-') { + ret = kstrtoint(buf + 1, 10, &slotno); + if (ret != 0) + return ret; + + /* now load each (take lock to be sure */ + mutex_lock(&info->slots_list_mutex); + list_for_each_entry(slot, &info->slot_list, node) { + if (slotno == slot->slotno) + goto found; + } + + mutex_unlock(&info->slots_list_mutex); + return -ENODEV; +found: + /* the hardware slots just get unloaded */ + if (!slot->override) { + ret = capemgr_unload_slot(slot); + if (ret == 0) + dev_info(&pdev->dev, + "Unloaded slot #%d\n", slotno); + else + dev_err(&pdev->dev, + "Failed to unload slot #%d\n", slotno); + } else { + ret = capemgr_remove_slot_no_lock(slot); + if (ret == 0) + dev_info(&pdev->dev, + "Removed slot #%d\n", slotno); + else + dev_err(&pdev->dev, + "Failed to remove slot #%d\n", slotno); + } + mutex_unlock(&info->slots_list_mutex); + + return ret == 0 ? strlen(buf) : ret; + } + + part_number = kstrdup(buf, GFP_KERNEL); + if (part_number == NULL) + return -ENOMEM; + + /* remove trailing spaces dots and newlines */ + s = part_number + strlen(part_number); + while (s > part_number && + (isspace(s[-1]) || s[-1] == '\n' || s[-1] == '.')) + *--s = '\0'; + + version = strchr(part_number, ':'); + if (version != NULL) + *version++ = '\0'; + + dev_info(&pdev->dev, "part_number '%s', version '%s'\n", + part_number, version ? version : "N/A"); + + pnode = pdev->dev.of_node; + node = NULL; + slot = NULL; + ret = 0; + + /* no specific slot found, try immediate */ + slot = capemgr_add_slot(info, NULL, part_number, version, 0); + + if (IS_ERR_OR_NULL(slot)) { + dev_err(&pdev->dev, "Failed to add slot #%d\n", + atomic_read(&info->next_slot_nr) - 1); + ret = slot ? PTR_ERR(slot) : -ENODEV; + slot = NULL; + goto err_fail; + } + + kfree(part_number); + + ret = capemgr_load_slot(slot); + if (ret != 0) + capemgr_remove_slot(slot); + + return ret == 0 ? strlen(buf) : ret; +err_fail: + of_node_put(node); + kfree(part_number); + return ret; +} + +/* verify the overlay */ +static int capemgr_verify_overlay(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + struct bone_baseboard *bbrd = &info->baseboard; + struct device_node *node = slot->overlay; + struct property *prop; + struct bone_cape_slot *slotn; + int err, counta, countb, i, j; + const char *ra, *rb; + + /* validate */ + if (node == NULL) { + dev_err(dev, "slot #%d: No overlay for '%s'\n", + slot->slotno, slot->part_number); + return -EINVAL; + } + + /* check if the slot is compatible with the board */ + prop = of_find_property(node, "compatible", NULL); + + /* no compatible property? */ + if (prop == NULL) { + dev_err(dev, "slot #%d: No compatible property for '%s'\n", + slot->slotno, slot->part_number); + return -EINVAL; + } + + /* verify that the cape is baseboard compatible */ + if (of_multi_prop_cmp(prop, bbrd->compatible_name) != 0) { + dev_err(dev, "slot #%d: Incompatible with baseboard for '%s'\n", + slot->slotno, slot->part_number); + return -EINVAL; + } + + /* count the strings */ + counta = of_property_count_strings(node, "exclusive-use"); + /* no valid property, or no resources; no matter, it's OK */ + if (counta <= 0) + return 0; + + /* and now check if there's a resource conflict */ + err = 0; + mutex_lock(&info->slots_list_mutex); + for (i = 0; i < counta; i++) { + + ra = NULL; + err = of_property_read_string_index(node, "exclusive-use", + i, &ra); + if (err != 0) { + dev_err(dev, "slot #%d: Could not read string #%d\n", + slot->slotno, i); + break; + } + + list_for_each_entry(slotn, &info->slot_list, node) { + + /* don't check against self */ + if (slot == slotn) + continue; + + /* only check against loaded or loading slots */ + if (!slotn->loaded && !slotn->loading) + continue; + + countb = of_property_count_strings(slotn->overlay, + "exclusive-use"); + /* no valid property, or resources; it's OK */ + if (countb <= 0) + continue; + + + for (j = 0; j < countb; j++) { + + /* count the resources */ + rb = NULL; + err = of_property_read_string_index( + slotn->overlay, "exclusive-use", + j, &rb); + if (err != 0) { + /* error, but we don't care */ + err = 0; + break; + } + + /* ignore case; just in case ;) */ + if (strcasecmp(ra, rb) == 0) { + + /* resource conflict */ + err = -EEXIST; + dev_err(dev, + "slot #%d: %s conflict %s (#%d:%s)\n", + slot->slotno, + slot->part_number, ra, + slotn->slotno, + slotn->part_number); + goto out; + } + } + } + } +out: + mutex_unlock(&info->slots_list_mutex); + + return err; +} + +static int capemgr_load_slot(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + const char *dtbo; + int err; + + if (slot->probe_failed) { + dev_err(dev, "slot #%d: probe failed for '%s'\n", + slot->slotno, slot->part_number); + return -ENODEV; + } + + if (slot->loaded) { + dev_err(dev, "slot #%d: already loaded for '%s'\n", + slot->slotno, slot->part_number); + return -EAGAIN; + } + + /* make sure we don't leak this on repeated calls */ + kfree(slot->dtbo); + slot->dtbo = NULL; + + dev_dbg(dev, "slot #%d: Requesting part number/version based '%s-%s.dtbo\n", + slot->slotno, slot->part_number, slot->version); + + /* request the part number + .dtbo*/ + slot->dtbo = kasprintf(GFP_KERNEL, "%s-%s.dtbo", + slot->part_number, slot->version); + if (slot->dtbo == NULL) { + dev_err(dev, "slot #%d: Failed to get dtbo '%s'\n", + slot->slotno, dtbo); + return -ENOMEM; + } + + dev_dbg(dev, "slot #%d: Requesting firmware '%s' for board-name '%s', version '%s'%s\n", + slot->slotno, + slot->dtbo, slot->board_name, slot->version, + system_state == SYSTEM_BOOTING ? " - booting" : ""); + + err = request_firmware_direct(&slot->fw, slot->dtbo, dev); + if (err != 0) { + dev_dbg(dev, "failed to load firmware '%s'\n", slot->dtbo); + goto err_fail_no_fw; + } + + dev_dbg(dev, "slot #%d: dtbo '%s' loaded; converting to live tree\n", + slot->slotno, slot->dtbo); + + of_fdt_unflatten_tree((unsigned long *)slot->fw->data, NULL, + &slot->overlay); + if (slot->overlay == NULL) { + dev_err(dev, "slot #%d: Failed to unflatten\n", + slot->slotno); + err = -EINVAL; + goto err_fail; + } + + /* mark it as detached */ + of_node_set_flag(slot->overlay, OF_DETACHED); + + /* perform resolution */ + err = of_resolve_phandles(slot->overlay); + if (err != 0) { + dev_err(dev, "slot #%d: Failed to resolve tree\n", + slot->slotno); + goto err_fail; + } + + err = capemgr_verify_overlay(slot); + if (err != 0) { + dev_err(dev, "slot #%d: Failed verification\n", + slot->slotno); + goto err_fail; + } + + err = of_overlay_create(slot->overlay); + if (err < 0) { + dev_err(dev, "slot #%d: Failed to create overlay\n", + slot->slotno); + goto err_fail; + } + slot->overlay_id = err; + + slot->loading = 0; + slot->loaded = 1; + + dev_info(dev, "slot #%d: dtbo '%s' loaded; overlay id #%d\n", + slot->slotno, slot->dtbo, slot->overlay_id); + + return 0; + +err_fail: + + /* TODO: free the overlay, we can't right now cause + * the unflatten method does not track it */ + slot->overlay = NULL; + + release_firmware(slot->fw); + slot->fw = NULL; + +err_fail_no_fw: + slot->loading = 0; + return err; +} + +static int capemgr_unload_slot(struct bone_cape_slot *slot) +{ + if (!slot->loaded || slot->overlay_id == -1) + return -EINVAL; + + of_overlay_destroy(slot->overlay_id); + slot->overlay_id = -1; + + slot->loaded = 0; + + return 0; + +} + +/* slots_list_mutex must be taken */ +static int capemgr_remove_slot_no_lock(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + int ret; + + if (slot == NULL) + return 0; + + if (slot->loaded && slot->overlay_id >= 0) { + /* unload just in case */ + ret = capemgr_unload_slot(slot); + if (ret != 0) { + dev_err(dev, "Unable to unload slot #%d\n", + slot->slotno); + return ret; + } + } + + /* if probed OK, remove the sysfs nodes */ + if (slot->probed && !slot->probe_failed) + bone_cape_slot_sysfs_unregister(slot); + + /* remove it from the list */ + list_del(&slot->node); + + if (slot->nvmem_cell) + nvmem_cell_put(slot->nvmem_cell); + devm_kfree(dev, slot); + return 0; +} + +static int capemgr_remove_slot(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + int ret; + + mutex_lock(&info->slots_list_mutex); + ret = capemgr_remove_slot_no_lock(slot); + mutex_unlock(&info->slots_list_mutex); + + return ret; +} + +static int bone_slot_fill_override(struct bone_cape_slot *slot, + const char *part_number, const char *version) +{ + const struct ee_field *sig_field; + int i, len, has_part_number; + char *p; + + slot->probe_failed = 0; + slot->probed = 0; + + /* zero out signature */ + memset(slot->signature, 0, + sizeof(slot->signature)); + + /* first, fill in all with override defaults */ + for (i = 0; i < ARRAY_SIZE(cape_sig_fields); i++) { + + sig_field = &cape_sig_fields[i]; + + /* point to the entry */ + p = slot->signature + sig_field->start; + + if (sig_field->override) + memcpy(p, sig_field->override, + sig_field->size); + else + memset(p, 0, sig_field->size); + } + + /* if a part_number is supplied use it */ + len = part_number ? strlen(part_number) : 0; + if (len > 0) { + sig_field = &cape_sig_fields[CAPE_EE_FIELD_PART_NUMBER]; + + /* point to the entry */ + p = slot->signature + sig_field->start; + + /* copy and zero out any remainder */ + if (len > sig_field->size) + len = sig_field->size; + memcpy(p, part_number, len); + if (len < sig_field->size) + memset(p + len, 0, sig_field->size - len); + + has_part_number = 1; + } + + /* if a version is supplied use it */ + len = version ? strlen(version) : 0; + if (len > 0) { + sig_field = &cape_sig_fields[CAPE_EE_FIELD_VERSION]; + + /* point to the entry */ + p = slot->signature + sig_field->start; + + /* copy and zero out any remainder */ + if (len > sig_field->size) + len = sig_field->size; + memcpy(p, version, len); + if (len < sig_field->size) + memset(p + len, 0, sig_field->size - len); + } + + /* we must have a part number */ + if (!has_part_number) + return -EINVAL; + + slot->override = 1; + + return 0; +} + +static struct bone_cape_slot * +capemgr_add_slot(struct capemgr_info *info, const char *slot_name, + const char *part_number, const char *version, int prio) +{ + struct bone_cape_slot *slot; + struct device *dev = &info->pdev->dev; + int slotno; + int ret; + + slotno = atomic_inc_return(&info->next_slot_nr) - 1; + + slot = devm_kzalloc(dev, sizeof(*slot), GFP_KERNEL); + if (slot == NULL) + return ERR_PTR(-ENOMEM); + + slot->info = info; + slot->slotno = slotno; + slot->priority = prio; + slot->overlay_id = -1; + + if (slot_name) { + slot->nvmem_cell = nvmem_cell_get(dev, slot_name); + if (IS_ERR(slot->nvmem_cell)) { + ret = PTR_ERR(slot->nvmem_cell); + if (ret != -EPROBE_DEFER) + dev_info(dev, "Failed to get slot eeprom cell\n"); + slot->nvmem_cell = NULL; + goto err_out; + } + } else { + dev_info(dev, "slot #%d: override\n", slotno); + + /* fill in everything with defaults first */ + ret = bone_slot_fill_override(slot, part_number, version); + if (ret != 0) { + dev_info(dev, "slot #%d: override failed\n", slotno); + goto err_out; + } + } + + ret = bone_slot_scan(slot); + if (ret != 0) { + + if (!slot->probe_failed) { + dev_info(dev, "slot #%d: scan failed\n", + slotno); + goto err_out; + } + + dev_info(dev, "slot #%d: No cape found\n", slotno); + /* but all is fine */ + } else { + if (uboot_capemgr_enabled == 0) { + dev_info(dev, "slot #%d: '%s'\n", + slotno, slot->text_id); + + ret = bone_cape_slot_sysfs_register(slot); + if (ret != 0) { + dev_info(dev, "slot #%d: sysfs register failed\n", + slotno); + goto err_out; + } + } else { + dev_info(dev, "slot #%d: auto loading handled by U-Boot\n", slotno); + } + } + + /* add to the slot list */ + mutex_lock(&info->slots_list_mutex); + list_add_tail(&slot->node, &info->slot_list); + mutex_unlock(&info->slots_list_mutex); + + return slot; + +err_out: + if (slot->nvmem_cell) + nvmem_cell_put(slot->nvmem_cell); + devm_kfree(dev, slot); + return ERR_PTR(ret); +} + +/* return 1 if it makes sense to retry loading */ +static int retry_loading_condition(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + struct bone_cape_slot *slotn; + int ret; + + dev_dbg(dev, "loader: retry_loading slot-%d %s:%s (prio %d)\n", + slot->slotno, slot->part_number, slot->version, + slot->priority); + + mutex_lock(&info->slots_list_mutex); + ret = 0; + list_for_each_entry(slotn, &info->slot_list, node) { + /* if same slot or not loading skip */ + if (!slotn->loading || slotn->retry_loading) + continue; + /* at least one cape is still loading (without retrying) */ + ret = 1; + } + mutex_unlock(&info->slots_list_mutex); + return ret; +} + +/* return 1 if this slot is clear to try to load now */ +static int clear_to_load_condition(struct bone_cape_slot *slot) +{ + struct capemgr_info *info = slot->info; + int my_prio = slot->priority; + struct device *dev = &info->pdev->dev; + int ret; + + dev_dbg(dev, "loader: check slot-%d %s:%s (prio %d)\n", slot->slotno, + slot->part_number, slot->version, slot->priority); + + mutex_lock(&info->slots_list_mutex); + ret = 1; + list_for_each_entry(slot, &info->slot_list, node) { + /* if any slot is loading with lowest priority */ + if (!slot->loading) + continue; + if (slot->priority < my_prio) { + ret = 0; + break; + } + } + mutex_unlock(&info->slots_list_mutex); + return ret; +} + +static int capemgr_loader(void *data) +{ + struct bone_cape_slot *slot = data; + struct capemgr_info *info = slot->info; + struct device *dev = &info->pdev->dev; + int ret, done, other_loading, booting; + + done = 0; + + slot->retry_loading = 0; + + dev_dbg(dev, "loader: before slot-%d %s:%s (prio %d)\n", slot->slotno, + slot->part_number, slot->version, slot->priority); + + /* + * We have a basic priority based arbitration system + * Slots have priorities, so the lower priority ones + * should start loading first. So each time we end up + * here. + */ + ret = wait_event_interruptible(info->load_wq, + clear_to_load_condition(slot)); + if (ret < 0) { + dev_warn(dev, "loader, Signal pending\n"); + return ret; + } + + dev_dbg(dev, "loader: after slot-%d %s:%s (prio %d)\n", slot->slotno, + slot->part_number, slot->version, slot->priority); + + /* using the return value */ + ret = capemgr_load_slot(slot); + + /* wake up all just in case */ + wake_up_interruptible_all(&info->load_wq); + + if (ret == 0) + goto done; + + dev_dbg(dev, "loader: retrying slot-%d %s:%s (prio %d)\n", slot->slotno, + slot->part_number, slot->version, slot->priority); + + /* first attempt has failed; now try each time there's any change */ + slot->retry_loading = 1; + + for (;;) { + booting = (system_state == SYSTEM_BOOTING); + other_loading = retry_loading_condition(slot); + if (!booting && !other_loading) + break; + + /* simple wait for someone to kick us */ + if (other_loading) { + DEFINE_WAIT(__wait); + + prepare_to_wait(&info->load_wq, &__wait, + TASK_INTERRUPTIBLE); + finish_wait(&info->load_wq, &__wait); + } else { + /* always delay when booting */ + msleep(boot_scan_period); + } + + if (signal_pending(current)) { + dev_warn(dev, "loader, Signal pending\n"); + ret = -ERESTARTSYS; + goto done; + } + + /* using the return value */ + ret = capemgr_load_slot(slot); + if (ret == 0) + goto done; + + /* wake up all just in case */ + wake_up_interruptible_all(&info->load_wq); + } + +done: + slot->loading = 0; + slot->retry_loading = 0; + + if (ret == 0) { + dev_dbg(dev, "loader: done slot-%d %s:%s (prio %d)\n", + slot->slotno, slot->part_number, slot->version, + slot->priority); + } else { + dev_err(dev, "loader: failed to load slot-%d %s:%s (prio %d)\n", + slot->slotno, slot->part_number, slot->version, + slot->priority); + + /* if it's a override slot remove it */ + if (slot->override) + capemgr_remove_slot(slot); + } + + return ret; +} + +static int +capemgr_probe(struct platform_device *pdev) +{ + struct capemgr_info *info; + struct bone_baseboard *bbrd; + struct bone_cape_slot *slot; + struct device_node *pnode = pdev->dev.of_node; + struct device_node *baseboardmaps_node; + struct device_node *node; + const char *part_number; + const char *version; + const char *board_name; + const char *compatible_name; + char slot_name[16]; + u32 slots_nr; + int i, ret, len, prio; + long val; + char *wbuf, *s, *p, *e; + + if (uboot_capemgr_enabled) + return 0; + + /* we don't use platform_data at all; we require OF */ + if (pnode == NULL) + return -ENOTSUPP; + + info = devm_kzalloc(&pdev->dev, + sizeof(struct capemgr_info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->pdev = pdev; + platform_set_drvdata(pdev, info); + + atomic_set(&info->next_slot_nr, 0); + INIT_LIST_HEAD(&info->slot_list); + mutex_init(&info->slots_list_mutex); + + init_waitqueue_head(&info->load_wq); + + baseboardmaps_node = NULL; + + /* find the baseboard */ + bbrd = &info->baseboard; + + baseboardmaps_node = of_get_child_by_name(pnode, "baseboardmaps"); + if (baseboardmaps_node == NULL) { + dev_err(&pdev->dev, "Failed to get baseboardmaps node"); + ret = -ENODEV; + goto err_exit; + } + + bbrd->nvmem_cell = nvmem_cell_get(&pdev->dev, "baseboard"); + if (IS_ERR(bbrd->nvmem_cell)) { + ret = PTR_ERR(bbrd->nvmem_cell); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Failed to get baseboard eeprom cell\n"); + bbrd->nvmem_cell = NULL; + goto err_exit; + } + + ret = bone_baseboard_scan(bbrd); + if (ret != 0) { + dev_err(&pdev->dev, "Failed to scan baseboard eeprom\n"); + goto err_exit; + } + + dev_info(&pdev->dev, "Baseboard: '%s'\n", bbrd->text_id); + + board_name = NULL; + compatible_name = NULL; + for_each_child_of_node(baseboardmaps_node, node) { + /* there must be board-name */ + if (of_property_read_string(node, "board-name", + &board_name) != 0 || + of_property_read_string(node, "compatible-name", + &compatible_name) != 0) + continue; + + if (strcmp(bbrd->board_name, board_name) == 0) + break; + } + of_node_put(baseboardmaps_node); + baseboardmaps_node = NULL; + + if (node == NULL) { + dev_err(&pdev->dev, "Failed to find compatible map for %s\n", + bbrd->board_name); + ret = -ENODEV; + goto err_exit; + } + bbrd->compatible_name = kstrdup(compatible_name, GFP_KERNEL); + if (bbrd->compatible_name == NULL) { + ret = -ENOMEM; + goto err_exit; + } + of_node_put(node); + + /* get slot number */ + ret = of_property_read_u32(pnode, "#slots", &slots_nr); + if (ret != 0) + slots_nr = 0; + + dev_info(&pdev->dev, "compatible-baseboard=%s - #slots=%d\n", + bbrd->compatible_name, slots_nr); + + for (i = 0; i < slots_nr; i++) { + snprintf(slot_name, sizeof(slot_name), "slot%d", i); + slot = capemgr_add_slot(info, slot_name, NULL, NULL, 0); + if (IS_ERR(slot)) { + dev_err(&pdev->dev, "Failed to add slot #%d\n", + atomic_read(&info->next_slot_nr)); + ret = PTR_ERR(slot); + goto err_exit; + } + } + + /* iterate over enable_partno (if there) */ + if (enable_partno && strlen(enable_partno) > 0) { + + /* allocate a temporary buffer */ + wbuf = devm_kzalloc(&pdev->dev, PAGE_SIZE, GFP_KERNEL); + if (wbuf == NULL) { + ret = -ENOMEM; + goto err_exit; + } + + /* add any enable_partno capes */ + s = enable_partno; + while (*s) { + /* form is PART[:REV[:PRIO]],PART.. */ + p = strchr(s, ','); + if (p == NULL) + e = s + strlen(s); + else + e = p; + + /* copy to temp buffer */ + len = e - s; + if (len >= PAGE_SIZE - 1) + len = PAGE_SIZE - 1; + memcpy(wbuf, s, len); + wbuf[len] = '\0'; + + /* move to the next */ + s = *e ? e + 1 : e; + + part_number = wbuf; + + /* default version is NULL & prio is 0 */ + version = NULL; + prio = 0; + + /* now split the rev & prio part */ + p = strchr(wbuf, ':'); + if (p != NULL) { + *p++ = '\0'; + if (*p != ':') + version = p; + p = strchr(p, ':'); + if (p != NULL) { + *p++ = '\0'; + ret = kstrtol(p, 10, &val); + if (ret == 0) + prio = val; + } + } + + dev_info(&pdev->dev, + "enabled_partno PARTNO '%s' VER '%s' PR '%d'\n", + part_number, + version ? version : "N/A", prio); + + /* only immediate slots are allowed here */ + slot = capemgr_add_slot(info, NULL, + part_number, version, prio); + + /* we continue even in case of an error */ + if (IS_ERR_OR_NULL(slot)) { + dev_warn(&pdev->dev, "Failed to add slot #%d\n", + atomic_read(&info->next_slot_nr) - 1); + } + } + + devm_kfree(&pdev->dev, wbuf); + } + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (IS_ERR_VALUE(ret)) { + dev_err(&pdev->dev, "Failed to pm_runtime_get_sync()\n"); + goto err_exit; + } + + pm_runtime_put(&pdev->dev); + + /* it is safe to create the attribute groups */ + ret = sysfs_create_groups(&pdev->dev.kobj, attr_groups); + if (ret != 0) { + dev_err(&pdev->dev, "Failed to create sysfs attributes\n"); + goto err_exit; + } + /* automatically cleared by driver core now */ + pdev->dev.groups = attr_groups; + + /* now load each (take lock to be sure */ + mutex_lock(&info->slots_list_mutex); + + list_for_each_entry(slot, &info->slot_list, node) { + + /* if matches the disabled ones skip */ + if (bone_match_cape(disable_partno, slot->part_number, + slot->version)) { + dev_info(&pdev->dev, + "Skipping loading of disabled cape with part# %s\n", + slot->part_number); + slot->disabled = 1; + continue; + } + + if (!slot->probe_failed && !slot->loaded) + slot->loading = 1; + } + + /* now start the loader thread(s) (all at once) */ + list_for_each_entry(slot, &info->slot_list, node) { + + if (!slot->loading) + continue; + + slot->loader_thread = kthread_run(capemgr_loader, + slot, "capemgr-loader-%d", + slot->slotno); + if (IS_ERR(slot->loader_thread)) { + dev_warn(&pdev->dev, "slot #%d: Failed to start loader\n", + slot->slotno); + slot->loader_thread = NULL; + } + } + mutex_unlock(&info->slots_list_mutex); + + dev_info(&pdev->dev, "initialized OK.\n"); + + return 0; + +err_exit: + if (bbrd->nvmem_cell) + nvmem_cell_put(bbrd->nvmem_cell); + of_node_put(baseboardmaps_node); + platform_set_drvdata(pdev, NULL); + devm_kfree(&pdev->dev, info); + + return ret; +} + +static int capemgr_remove(struct platform_device *pdev) +{ + struct capemgr_info *info = platform_get_drvdata(pdev); + struct bone_baseboard *bbrd = &info->baseboard; + struct bone_cape_slot *slot, *slotn; + int ret; + + mutex_lock(&info->slots_list_mutex); + list_for_each_entry_safe(slot, slotn, &info->slot_list, node) + capemgr_remove_slot_no_lock(slot); + mutex_unlock(&info->slots_list_mutex); + + platform_set_drvdata(pdev, NULL); + + ret = pm_runtime_get_sync(&pdev->dev); + if (IS_ERR_VALUE(ret)) + return ret; + + pm_runtime_put(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + if (bbrd->nvmem_cell) + nvmem_cell_put(bbrd->nvmem_cell); + devm_kfree(&pdev->dev, info); + + return 0; +} + +static struct platform_driver capemgr_driver = { + .probe = capemgr_probe, + .remove = capemgr_remove, + .driver = { + .name = "bone_capemgr", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(capemgr_of_match), + }, +}; + +module_platform_driver(capemgr_driver); + +MODULE_AUTHOR("Pantelis Antoniou"); +MODULE_DESCRIPTION("Beaglebone cape manager"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:bone_capemgr"); diff --git a/drivers/misc/cape/Kconfig b/drivers/misc/cape/Kconfig new file mode 100644 index 0000000000000..a2ef85e0415c5 --- /dev/null +++ b/drivers/misc/cape/Kconfig @@ -0,0 +1,5 @@ +# +# Capes +# + +source "drivers/misc/cape/beaglebone/Kconfig" diff --git a/drivers/misc/cape/Makefile b/drivers/misc/cape/Makefile new file mode 100644 index 0000000000000..7c4eb96ab29e4 --- /dev/null +++ b/drivers/misc/cape/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for cape like devices +# + +obj-y += beaglebone/ diff --git a/drivers/misc/cape/beaglebone/Kconfig b/drivers/misc/cape/beaglebone/Kconfig new file mode 100644 index 0000000000000..eeb67828fd829 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Kconfig @@ -0,0 +1,10 @@ +# +# Beaglebone capes +# + +config BEAGLEBONE_PINMUX_HELPER + tristate "Beaglebone Pinmux Helper" + depends on ARCH_OMAP2PLUS && OF + default n + help + Say Y here to include support for the pinmux helper diff --git a/drivers/misc/cape/beaglebone/Makefile b/drivers/misc/cape/beaglebone/Makefile new file mode 100644 index 0000000000000..7f4617a01e262 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for beaglebone capes +# + +obj-$(CONFIG_BEAGLEBONE_PINMUX_HELPER) += bone-pinmux-helper.o diff --git a/drivers/misc/cape/beaglebone/bone-pinmux-helper.c b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c new file mode 100644 index 0000000000000..d81363a77b103 --- /dev/null +++ b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c @@ -0,0 +1,242 @@ +/* + * Pinmux helper driver + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct of_device_id bone_pinmux_helper_of_match[] = { + { + .compatible = "bone-pinmux-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, bone_pinmux_helper_of_match); + +struct pinmux_helper_data { + struct pinctrl *pinctrl; + char *selected_state_name; +}; + +static ssize_t pinmux_helper_show_state(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + const char *name; + + name = data->selected_state_name; + if (name == NULL || strlen(name) == 0) + name = "none"; + return sprintf(buf, "%s\n", name); +} + +static ssize_t pinmux_helper_store_state(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct pinctrl_state *state; + char *state_name; + char *s; + int err; + + /* duplicate (as a null terminated string) */ + state_name = kmalloc(count + 1, GFP_KERNEL); + if (state_name == NULL) + return -ENOMEM; + memcpy(state_name, buf, count); + state_name[count] = '\0'; + + /* and chop off newline */ + s = strchr(state_name, '\n'); + if (s != NULL) + *s = '\0'; + + /* try to select default state at first (if it exists) */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) + dev_err(dev, "Failed to select state %s\n", + state_name); + } else { + dev_err(dev, "Failed to find state %s\n", state_name); + err = PTR_RET(state); + } + + if (err == 0) { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + } + + return err ? err : count; +} + +static DEVICE_ATTR(state, S_IWUSR | S_IRUGO, + pinmux_helper_show_state, pinmux_helper_store_state); + +static struct attribute *pinmux_helper_attributes[] = { + &dev_attr_state.attr, + NULL +}; + +static const struct attribute_group pinmux_helper_attr_group = { + .attrs = pinmux_helper_attributes, +}; + +static int bone_pinmux_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pinmux_helper_data *data; + struct pinctrl_state *state; + char *state_name; + const char *mode_name; + int mode_len; + int err; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (data == NULL) { + dev_err(dev, "Failed to allocate data\n"); + err = -ENOMEM; + goto err_no_mem; + } + + state_name = kmalloc(strlen(PINCTRL_STATE_DEFAULT) + 1, + GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_state_mem; + } + data->selected_state_name = state_name; + strcpy(data->selected_state_name, PINCTRL_STATE_DEFAULT); + + platform_set_drvdata(pdev, data); + + data->pinctrl = devm_pinctrl_get(dev); + if (IS_ERR(data->pinctrl)) { + dev_err(dev, "Failed to get pinctrl\n"); + err = PTR_RET(data->pinctrl); + goto err_no_pinctrl; + } + + /* See if an initial mode is specified in the device tree */ + mode_name = of_get_property(dev->of_node, "mode", &mode_len); + + err = -1; + if (mode_name != NULL ) { + state_name = kmalloc(mode_len + 1, GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_mode_mem; + } + strncpy(state_name, mode_name, mode_len); + + /* try to select requested mode */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_warn(dev, "Unable to select requested mode %s\n", state_name); + kfree(state_name); + } else { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + dev_notice(dev, "Set initial pinmux mode to %s\n", state_name); + } + } + } + + /* try to select default state if mode_name failed */ + if ( err != 0) { + state = pinctrl_lookup_state(data->pinctrl, + data->selected_state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_err(dev, "Failed to select default state\n"); + goto err_no_state; + } + } else { + data->selected_state_name = '\0'; + } + } + + /* Register sysfs hooks */ + err = sysfs_create_group(&dev->kobj, &pinmux_helper_attr_group); + if (err) { + dev_err(dev, "Failed to create sysfs group\n"); + goto err_no_sysfs; + } + + return 0; + +err_no_sysfs: +err_no_state: +err_no_mode_mem: + devm_pinctrl_put(data->pinctrl); +err_no_pinctrl: + devm_kfree(dev, data->selected_state_name); +err_no_state_mem: + devm_kfree(dev, data); +err_no_mem: + return err; +} + +static int bone_pinmux_helper_remove(struct platform_device *pdev) +{ + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + + sysfs_remove_group(&dev->kobj, &pinmux_helper_attr_group); + kfree(data->selected_state_name); + devm_pinctrl_put(data->pinctrl); + devm_kfree(dev, data); + + return 0; +} + +struct platform_driver bone_pinmux_helper_driver = { + .probe = bone_pinmux_helper_probe, + .remove = bone_pinmux_helper_remove, + .driver = { + .name = "bone-pinmux-helper", + .owner = THIS_MODULE, + .of_match_table = bone_pinmux_helper_of_match, + }, +}; + +module_platform_driver(bone_pinmux_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou"); +MODULE_DESCRIPTION("Beaglebone pinmux helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:bone-pinmux-helper"); diff --git a/drivers/misc/devovmgr.c b/drivers/misc/devovmgr.c new file mode 100644 index 0000000000000..d5c8d1db58a73 --- /dev/null +++ b/drivers/misc/devovmgr.c @@ -0,0 +1,1306 @@ +/* + * Device overlay manager + * + * Copyright (C) 2015 Konsulko Group + * Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum dovmgr_type { + ITEM_PCI, + ITEM_USB +}; + +struct dovmgr_item; + +struct dovmgr_dev_item { + struct dovmgr_item *item; + struct list_head node; + struct device *dev; + const struct firmware *fw; + struct device_node *overlay; + int overlay_id; + struct work_struct work; +}; + +struct dovmgr_item { + struct config_item item; + char *path; + bool enable; + char *overlay_name; + struct mutex dev_item_mutex; + struct list_head dev_item_list; + enum dovmgr_type type; + union { + struct pci_device_id pci; + struct usb_device_id usb; + }; +}; + +struct config_group dovmgr_pci_group; +struct config_group dovmgr_usb_group; + +static inline struct dovmgr_item *to_dovmgr_item(struct config_item *cfsitem) +{ + if (!cfsitem) + return NULL; + + return container_of(cfsitem, struct dovmgr_item, item); +} + +static int dovmgr_notifier_action(struct config_group *group, + unsigned long action, struct device *dev, + int (*do_match)(struct dovmgr_item *item, struct device *dev), + int (*do_action)(struct dovmgr_item *item, unsigned long action, + struct device *dev)) +{ + struct config_item *cfsitem; + struct dovmgr_item *item; + int ret; + + /* only handle device notifiers */ + if (action != BUS_NOTIFY_ADD_DEVICE && + action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE) + return 0; + + ret = 0; + list_for_each_entry(cfsitem, &group->cg_children, ci_entry) { + item = to_dovmgr_item(cfsitem); + if (!item->enable || !(*do_match)(item, dev)) + continue; + + ret = (*do_action)(item, action, dev); + if (ret != 0) + break; + } + return ret; +} + +#if IS_ENABLED(CONFIG_PCI) +/* copy of drivers/pci/pci.h */ +static inline const struct pci_device_id * +pci_match_one_device(const struct pci_device_id *id, const struct pci_dev *dev) +{ + if ((id->vendor == PCI_ANY_ID || id->vendor == dev->vendor) && + (id->device == PCI_ANY_ID || id->device == dev->device) && + (id->subvendor == PCI_ANY_ID || + id->subvendor == dev->subsystem_vendor) && + (id->subdevice == PCI_ANY_ID || + id->subdevice == dev->subsystem_device) && + !((id->class ^ dev->class) & id->class_mask)) + return id; + return NULL; +} + +static int dovmgr_pci_item_match(struct dovmgr_item *item, struct device *dev) +{ + struct pci_dev *pdev; + + BUG_ON(item->type != ITEM_PCI); + pdev = to_pci_dev(dev); + + return pci_match_one_device(&item->pci, pdev) != NULL; +} +#endif + +#if IS_ENABLED(CONFIG_USB) +/* in drivers/usb/core/driver.c */ +extern int usb_match_device(struct usb_device *dev, + const struct usb_device_id *id); + +static int dovmgr_usb_item_match(struct dovmgr_item *item, struct device *dev) +{ + struct usb_device *udev; + + BUG_ON(item->type != ITEM_USB); + udev = to_usb_device(dev); + + return usb_match_device(udev, &item->usb); +} +#endif + +static struct dovmgr_dev_item *dovmgr_lookup_dev_item(struct dovmgr_item *item, + struct device *dev) +{ + struct dovmgr_dev_item *ditem; + + list_for_each_entry(ditem, &item->dev_item_list, node) + if (ditem->dev == dev) + return ditem; + return NULL; +} + +static void dovmgr_item_work_func(struct work_struct *work) +{ + struct dovmgr_dev_item *ditem = container_of(work, + struct dovmgr_dev_item, work); + struct dovmgr_item *item = ditem->item; + struct device *dev; + struct device_node *np; + int err; + + mutex_lock(&item->dev_item_mutex); + + dev = ditem->dev; + np = dev->of_node; + if (!dev || !np || !item->overlay_name || ditem->overlay_id >= 0) + goto out_unlock; + + pr_info("%s: %s %s\n", __func__, + kobject_name(&dev->kobj), of_node_full_name(np)); + + err = request_firmware_direct(&ditem->fw, item->overlay_name, dev); + if (err != 0) { + pr_err("%s: %s failed to load firmware '%s'\n", __func__, + kobject_name(&dev->kobj), item->overlay_name); + goto out_unlock; + } + + of_fdt_unflatten_tree((void *)ditem->fw->data, &ditem->overlay); + if (ditem->overlay == NULL) { + pr_err("%s: %s failed to load firmware '%s'\n", __func__, + kobject_name(&dev->kobj), item->overlay_name); + goto out_release_fw; + } + + /* mark it as detached */ + of_node_set_flag(ditem->overlay, OF_DETACHED); + + /* perform resolution */ + err = of_resolve_phandles(ditem->overlay); + if (err != 0) { + pr_err("%s: %s failed to resolve tree\n", __func__, + kobject_name(&dev->kobj)); + goto out_release_overlay; + } + + err = of_overlay_create_target_root(ditem->overlay, np); + if (err < 0) { + pr_err("%s: %s failed to create overlay\n", __func__, + kobject_name(&dev->kobj)); + goto out_release_overlay; + } + ditem->overlay_id = err; + +out_unlock: + mutex_unlock(&item->dev_item_mutex); + return; + +out_release_overlay: + /* TODO: free the overlay, we can't right now cause + * the unflatten method does not track it */ + ditem->overlay = NULL; +out_release_fw: + release_firmware(ditem->fw); + ditem->fw = NULL; + goto out_unlock; +} + +/* dev item list mutex lock must be held */ +static int dovmgr_add_dev_item(struct dovmgr_item *item, struct device *dev) +{ + struct dovmgr_dev_item *ditem; + + /* first make sure there's no duplicate */ + if (dovmgr_lookup_dev_item(item, dev)) + return -EEXIST; + + /* add the device item */ + ditem = kzalloc(sizeof(*ditem), GFP_KERNEL); + if (!ditem) + return -ENOMEM; + ditem->overlay_id = -1; + ditem->dev = get_device(dev); + INIT_WORK(&ditem->work, dovmgr_item_work_func); + ditem->item = item; + + list_add_tail(&ditem->node, &item->dev_item_list); + + pr_info("%s: added device %s from item's %s list\n", __func__, + kobject_name(&dev->kobj), + config_item_name(&item->item)); + + /* now schedule the overlay application */ + if (item->overlay_name) + schedule_work(&ditem->work); + + return 0; +} + +static int dovmgr_remove_dev_item(struct dovmgr_item *item, struct device *dev) +{ + struct dovmgr_dev_item *ditem; + + /* find it */ + ditem = dovmgr_lookup_dev_item(item, dev); + if (!ditem) + return -ENODEV; + + if (work_pending(&ditem->work)) + cancel_work_sync(&ditem->work); + + if (ditem->overlay_id >= 0) { + of_overlay_destroy(ditem->overlay_id); + ditem->overlay_id = -1; + + } + + if (ditem->overlay) { + /* TODO: free the overlay, we can't right now cause + * the unflatten method does not track it */ + ditem->overlay = NULL; + } + + if (ditem->fw) { + /* TODO release_firmware(ditem->fw); */ + release_firmware(ditem->fw); + ditem->fw = NULL; + } + + put_device(ditem->dev); + list_del(&ditem->node); + + kfree(ditem); + + pr_info("%s: removed device %s from item's %s list\n", __func__, + kobject_name(&dev->kobj), + config_item_name(&item->item)); + + return 0; +} + +static int dovmgr_item_notify(struct dovmgr_item *item, + unsigned long action, struct device *dev) +{ + int ret; + + ret = 0; + mutex_lock(&item->dev_item_mutex); + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + pr_info("%s: BUS_NOTIFY_ADD_DEVICE for %s\n", __func__, + kobject_name(&dev->kobj)); + + ret = dovmgr_add_dev_item(item, dev); + if (ret != 0) + goto out_unlock; + + break; + + case BUS_NOTIFY_DEL_DEVICE: + pr_info("%s: BUS_NOTIFY_DEL_DEVICE for %s\n", __func__, + kobject_name(&dev->kobj)); + break; + + case BUS_NOTIFY_REMOVED_DEVICE: + pr_info("%s: BUS_NOTIFY_REMOVE_DEVICE for %s\n", __func__, + kobject_name(&dev->kobj)); + + ret = dovmgr_remove_dev_item(item, dev); + if (ret != 0) + goto out_unlock; + + break; + } + +out_unlock: + mutex_unlock(&item->dev_item_mutex); + + return ret; +} + +#if IS_ENABLED(CONFIG_PCI) +static int dovmgr_pci_add_iterator(struct device *dev, void *data) +{ + struct dovmgr_item *item = data; + + /* do add match */ + if (!item->enable || !dovmgr_pci_item_match(item, dev)) + return 0; + + pr_info("%s: dev=%s\n", __func__, kobject_name(&dev->kobj)); + + return dovmgr_item_notify(item, BUS_NOTIFY_ADD_DEVICE, dev); +} + +static int dovmgr_pci_removed_iterator(struct device *dev, void *data) +{ + struct dovmgr_item *item = data; + + /* do add match */ + if (item->enable || !dovmgr_pci_item_match(item, dev)) + return 0; + + pr_info("%s: dev=%s\n", __func__, kobject_name(&dev->kobj)); + + return dovmgr_item_notify(item, BUS_NOTIFY_REMOVED_DEVICE, dev); +} +#endif + +#if IS_ENABLED(CONFIG_USB) +static int dovmgr_usb_add_iterator(struct device *dev, void *data) +{ + struct dovmgr_item *item = data; + + /* do add match */ + if (item->enable || !dovmgr_usb_item_match(item, dev)) + return 0; + + pr_info("%s: dev=%s\n", __func__, kobject_name(&dev->kobj)); + + return dovmgr_item_notify(item, BUS_NOTIFY_ADD_DEVICE, dev); +} + +static int dovmgr_usb_removed_iterator(struct device *dev, void *data) +{ + struct dovmgr_item *item = data; + + /* do add match */ + if (!item->enable || !dovmgr_usb_item_match(item, dev)) + return 0; + + pr_info("%s: dev=%s\n", __func__, kobject_name(&dev->kobj)); + + return dovmgr_item_notify(item, BUS_NOTIFY_REMOVED_DEVICE, dev); +} +#endif + +static int dovmgr_item_set_enable(struct dovmgr_item *item, bool new_enable) +{ + int ret; + + if (new_enable == item->enable) + return 0; + + item->enable = new_enable; + switch (item->type) { +#if IS_ENABLED(CONFIG_PCI) + case ITEM_PCI: + ret = bus_for_each_dev(&pci_bus_type, NULL, item, + new_enable ? dovmgr_pci_add_iterator : + dovmgr_pci_removed_iterator); + if (ret != 0) + return ret; + break; +#endif +#if IS_ENABLED(CONFIG_USB) + case ITEM_USB: + ret = bus_for_each_dev(&usb_bus_type, NULL, item, + new_enable ? dovmgr_usb_add_iterator : + dovmgr_usb_removed_iterator); + if (ret != 0) + return ret; + break; +#endif + default: + break; + } + return 0; +} + + +static ssize_t dovmgr_item_str_show(struct dovmgr_item *item, + char *page, char **strp) +{ + return snprintf(page, PAGE_SIZE, "%s\n", + *strp ? *strp : ""); +} + +static ssize_t dovmgr_item_str_store(struct dovmgr_item *item, + const char *page, size_t count, char **strp) +{ + const char *s; + int len; + + /* copy to path buffer (and make sure it's always zero terminated */ + len = strnlen(page, PAGE_SIZE); + if (len >= PAGE_SIZE) + return -EINVAL; + s = page + len; + while (len > 0 && *--s == '\n') + len--; + if (len == 0) + return -EINVAL; + + if (*strp) + kfree(*strp); + *strp = kmalloc(len + 1, GFP_KERNEL); + if (!*strp) + return -ENOMEM; + memcpy(*strp, page, len); + (*strp)[len + 1] = '\0'; + + return count; +} + +static ssize_t dovmgr_item_path_show(struct config_item *citem, char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return dovmgr_item_str_show(item, page, &item->path); +} + +static ssize_t dovmgr_item_path_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return dovmgr_item_str_store(item, page, count, &item->path); +} + +static ssize_t dovmgr_item_enable_show(struct config_item *citem, char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "%u\n", !!item->enable); +} + +static ssize_t dovmgr_item_enable_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + + ret = dovmgr_item_set_enable(item, !!val); + if (ret != 0) + return ret; + + return count; +} + +static ssize_t dovmgr_item_overlay_show(struct config_item *citem, char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + ssize_t ret; + + mutex_lock(&item->dev_item_mutex); + ret = snprintf(page, PAGE_SIZE, "%s\n", item->overlay_name ? + item->overlay_name : ""); + mutex_unlock(&item->dev_item_mutex); + return ret; +}; + + +static ssize_t dovmgr_item_overlay_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + ssize_t ret; + + mutex_lock(&item->dev_item_mutex); + kfree(item->overlay_name); + item->overlay_name = kstrndup(page, PAGE_SIZE, GFP_KERNEL); + if (!item->overlay_name) + ret = -ENOMEM; + else + ret = count; + mutex_unlock(&item->dev_item_mutex); + return ret; +} + +static ssize_t dovmgr_item_status_show(struct config_item *citem, char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + struct dovmgr_dev_item *ditem; + char *p, *e; + int len; + + p = page; + e = page + PAGE_SIZE; + + mutex_lock(&item->dev_item_mutex); + list_for_each_entry(ditem, &item->dev_item_list, node) { + len = snprintf(p, e - p, "%s:%s:%d\n", + kobject_name(&ditem->dev->kobj), + of_node_full_name(ditem->dev->of_node), + ditem->overlay_id); + p += len; + if (p >= e - 1) + break; + } + mutex_unlock(&item->dev_item_mutex); + + return p - page; +} + +CONFIGFS_ATTR(dovmgr_item_, path); +CONFIGFS_ATTR_RO(dovmgr_item_, status); +CONFIGFS_ATTR(dovmgr_item_, enable); +CONFIGFS_ATTR(dovmgr_item_, overlay); + +#if IS_ENABLED(CONFIG_PCI) +static ssize_t dovmgr_item_pci_device_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", item->pci.device); +} + +static ssize_t dovmgr_item_pci_device_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.device = val; + return count; +} + +static ssize_t dovmgr_item_pci_vendor_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", item->pci.vendor); +} + +static ssize_t dovmgr_item_pci_vendor_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.vendor = val; + return count; +} + +static ssize_t dovmgr_item_pci_subdevice_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%08x\n", item->pci.subdevice); +} + +static ssize_t dovmgr_item_pci_subdevice_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.subdevice = val; + return count; +} + +static ssize_t dovmgr_item_pci_subvendor_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%08x\n", item->pci.subvendor); +} + +static ssize_t dovmgr_item_pci_subvendor_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.subvendor = val; + return count; +} + +static ssize_t dovmgr_item_pci_class_show(struct config_item *citem, char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", item->pci.class); +} + +static ssize_t dovmgr_item_pci_class_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.class = val; + return count; +} + +static ssize_t dovmgr_item_pci_class_mask_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", item->pci.class_mask); +} + +static ssize_t dovmgr_item_pci_class_mask_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->pci.class_mask = val; + return count; +} + +CONFIGFS_ATTR(dovmgr_item_pci_, device); +CONFIGFS_ATTR(dovmgr_item_pci_, vendor); +CONFIGFS_ATTR(dovmgr_item_pci_, subdevice); +CONFIGFS_ATTR(dovmgr_item_pci_, subvendor); +CONFIGFS_ATTR(dovmgr_item_pci_, class); +CONFIGFS_ATTR(dovmgr_item_pci_, class_mask); +#endif + +#if IS_ENABLED(CONFIG_USB) +static ssize_t dovmgr_item_usb_idProduct_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", + item->usb.idProduct); +} + +static ssize_t dovmgr_item_usb_idProduct_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->usb.idProduct = val; + return count; +} + +static ssize_t dovmgr_item_usb_idVendor_show(struct config_item *citem, + char *page) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + return snprintf(page, PAGE_SIZE, "0x%04x\n", + item->usb.idVendor); +} + +static ssize_t dovmgr_item_usb_idVendor_store(struct config_item *citem, + const char *page, size_t count) +{ + struct dovmgr_item *item = to_dovmgr_item(citem); + int ret; + unsigned int val; + + /* cannot modify when item is enabled */ + if (item->enable) + return -EBUSY; + + ret = kstrtouint(page, 0, &val); + if (ret != 0) + return ret; + item->usb.idVendor = val; + return count; +} + +CONFIGFS_ATTR(dovmgr_item_usb_, idProduct); +CONFIGFS_ATTR(dovmgr_item_usb_, idVendor); +#endif + +#if IS_ENABLED(CONFIG_PCI) +static struct configfs_attribute *dovmgr_pci_attrs[] = { + &dovmgr_item_attr_path, + &dovmgr_item_attr_status, + &dovmgr_item_attr_enable, + &dovmgr_item_attr_overlay, + &dovmgr_item_pci_attr_device, + &dovmgr_item_pci_attr_vendor, + &dovmgr_item_pci_attr_subdevice, + &dovmgr_item_pci_attr_subvendor, + &dovmgr_item_pci_attr_class, + &dovmgr_item_pci_attr_class_mask, + NULL, +}; +#endif + +#if IS_ENABLED(CONFIG_USB) +static struct configfs_attribute *dovmgr_usb_attrs[] = { + &dovmgr_item_attr_path, + &dovmgr_item_attr_enable, + &dovmgr_item_attr_status, + &dovmgr_item_attr_overlay, + &dovmgr_item_usb_attr_idVendor, + &dovmgr_item_usb_attr_idProduct, + NULL, +}; +#endif + +static void dovmgr_release(struct config_item *cfsitem) +{ + struct dovmgr_item *item = to_dovmgr_item(cfsitem); + + /* disable item (this removes the overlay and all) */ + dovmgr_item_set_enable(item, false); + + kfree(item->path); + kfree(item); +} + +static struct configfs_item_operations dovmgr_item_ops = { + .release = dovmgr_release, +}; + +#if IS_ENABLED(CONFIG_PCI) +static struct config_item_type dovmgr_pci_item_type = { + .ct_item_ops = &dovmgr_item_ops, + .ct_attrs = dovmgr_pci_attrs, + .ct_owner = THIS_MODULE, +}; +#endif + +#if IS_ENABLED(CONFIG_USB) +static struct config_item_type dovmgr_usb_item_type = { + .ct_item_ops = &dovmgr_item_ops, + .ct_attrs = dovmgr_usb_attrs, + .ct_owner = THIS_MODULE, +}; +#endif + +static struct config_item *dovmgr_group_make_item( + struct config_group *group, const char *name, + enum dovmgr_type type) +{ + struct dovmgr_item *item; + struct config_item_type *item_type; + + switch (type) { +#if IS_ENABLED(CONFIG_PCI) + case ITEM_PCI: + item_type = &dovmgr_pci_item_type; + break; +#endif +#if IS_ENABLED(CONFIG_USB) + case ITEM_USB: + item_type = &dovmgr_usb_item_type; + break; +#endif + default: + return ERR_PTR(-EINVAL); + }; + + item = kzalloc(sizeof(*item), GFP_KERNEL); + if (!item) + return ERR_PTR(-ENOMEM); + + item->type = type; + item->enable = false; + mutex_init(&item->dev_item_mutex); + INIT_LIST_HEAD(&item->dev_item_list); + + switch (type) { +#if IS_ENABLED(CONFIG_PCI) + case ITEM_PCI: + /* default for matching device/vendor */ + item->pci.vendor = PCI_ANY_ID; + item->pci.device = PCI_ANY_ID; + item->pci.subvendor = PCI_ANY_ID; + item->pci.subdevice = PCI_ANY_ID; + item->pci.class = 0; + item->pci.class_mask = 0; + break; +#endif +#if IS_ENABLED(CONFIG_USB) + case ITEM_USB: + /* default */ + item->usb.match_flags = USB_DEVICE_ID_MATCH_DEVICE; + break; +#endif + default: + return ERR_PTR(-EINVAL); + }; + + config_item_init_type_name(&item->item, name, item_type); + return &item->item; +} + +#if IS_ENABLED(CONFIG_PCI) +static struct config_item *dovmgr_group_pci_make_item( + struct config_group *group, const char *name) +{ + return dovmgr_group_make_item(group, name, ITEM_PCI); +} +#endif + +#if IS_ENABLED(CONFIG_USB) +static struct config_item *dovmgr_group_usb_make_item( + struct config_group *group, const char *name) +{ + return dovmgr_group_make_item(group, name, ITEM_USB); +} +#endif + +static void dovmgr_group_drop_item(struct config_group *group, + struct config_item *cfsitem) +{ + struct dovmgr_item *item = to_dovmgr_item(cfsitem); + + switch (item->type) { +#if IS_ENABLED(CONFIG_PCI) + case ITEM_PCI: + break; +#endif +#if IS_ENABLED(CONFIG_USB) + case ITEM_USB: + break; +#endif + default: + break; + } + config_item_put(&item->item); +} + +#if IS_ENABLED(CONFIG_PCI) +static struct configfs_group_operations dovmgr_pci_group_ops = { + .make_item = dovmgr_group_pci_make_item, + .drop_item = dovmgr_group_drop_item, +}; + +static struct config_item_type dovmgr_pci_type = { + .ct_group_ops = &dovmgr_pci_group_ops, + .ct_owner = THIS_MODULE, +}; +#endif + +#if IS_ENABLED(CONFIG_USB) +static struct configfs_group_operations dovmgr_usb_group_ops = { + .make_item = dovmgr_group_usb_make_item, + .drop_item = dovmgr_group_drop_item, +}; + +static struct config_item_type dovmgr_usb_type = { + .ct_group_ops = &dovmgr_usb_group_ops, + .ct_owner = THIS_MODULE, +}; +#endif + +static struct configfs_group_operations dovmgr_ops = { + /* empty - we don't allow anything to be created */ +}; + +static struct config_item_type dovmgr_type = { + .ct_group_ops = &dovmgr_ops, + .ct_owner = THIS_MODULE, +}; + +struct config_group *dovmgr_def_groups[] = { +#if IS_ENABLED(CONFIG_PCI) + &dovmgr_pci_group, +#endif +#if IS_ENABLED(CONFIG_USB) + &dovmgr_usb_group, +#endif + NULL +}; + +static struct configfs_subsystem dovmgr_subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "dovmgr", + .ci_type = &dovmgr_type, + }, + .default_groups = dovmgr_def_groups, + }, + .su_mutex = __MUTEX_INITIALIZER(dovmgr_subsys.su_mutex), +}; + +#if IS_ENABLED(CONFIG_PCI) +static int pci_dev_instantiate(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct device *bus_dev; + struct of_changeset cset; + struct device_node *np, *npb; + int ret; + + npb = NULL; + + /* already instantiated */ + if (dev->of_node) { + pr_debug("%s: dev=%s of_node=%s\n", __func__, + kobject_name(&dev->kobj), + of_node_full_name(dev->of_node)); + return 0; + } + + bus_dev = &pdev->bus->dev; + + pr_debug("%s: %s: %02x:%02x.%02x - node %s%s\n", __func__, + kobject_name(&dev->kobj), + pdev->bus->number, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn), + bus_dev->of_node ? of_node_full_name(bus_dev->of_node) : "", + pci_is_bridge(pdev) ? " bridge" : ""); + + /* to create the node, the bus must be present */ + if (!bus_dev->of_node) { + pr_err("%s: No node for %s because no bus device node\n", + __func__, kobject_name(&dev->kobj)); + return 0; + } + + of_changeset_init(&cset); + + np = of_changeset_create_device_node(&cset, bus_dev->of_node, + "%s/pci-%04x-%02x-%02x.%d", + of_node_full_name(bus_dev->of_node), + pci_domain_nr(pdev->bus), pdev->bus->number, + PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + if (IS_ERR(np)) { + ret = PTR_ERR(np); + goto out_cset_fail; + } + + ret = of_changeset_add_property_stringf(&cset, np, "compatible", + "pciclass,%04x", (pdev->class >> 8) & 0xffffff); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_u32(&cset, np, "vendor", + pdev->vendor); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_u32(&cset, np, "device", + pdev->device); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_string(&cset, np, "status", "okay"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_bool(&cset, np, "auto-generated"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_attach_node(&cset, np); + if (ret != 0) + goto out_cset_fail; + + /* are we creating a bridge; swell */ + npb = NULL; + if (pci_is_bridge(pdev) && !pdev->subordinate->dev.of_node) { + + pr_debug("%s: %s: bus->dev=%s subordinate=%s\n", __func__, + kobject_name(&dev->kobj), + kobject_name(&pdev->bus->dev.kobj), + kobject_name(&pdev->subordinate->dev.kobj)); + + npb = of_changeset_create_device_node(&cset, bus_dev->of_node, + "%s/pci-%04x-%02x", + of_node_full_name(bus_dev->of_node), + pci_domain_nr(pdev->subordinate), + pdev->subordinate->number); + if (IS_ERR(npb)) { + ret = PTR_ERR(npb); + goto out_cset_fail; + } + + ret = of_changeset_add_property_string(&cset, npb, "compatible", "generic,pci-bus"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_string(&cset, npb, "device_type", "pci"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_string(&cset, npb, "status", "okay"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_add_property_bool(&cset, npb, "auto-generated"); + if (ret != 0) + goto out_cset_fail; + + ret = of_changeset_attach_node(&cset, npb); + if (ret != 0) + goto out_cset_fail; + } + + ret = of_changeset_apply(&cset); + if (ret != 0) + goto out_cset_fail; + + /* permanently commit */ + of_changeset_destroy(&cset); + + /* bind the node to the device */ + dev->of_node = np; + ret = sysfs_create_link(&dev->kobj, &dev->of_node->kobj, + "of_node"); + if (ret) + pr_warn("%s: %s Error %d creating of_node link\n", + __func__, kobject_name(&dev->kobj), ret); + + if (npb) { + pdev->subordinate->dev.of_node = npb; + ret = sysfs_create_link(&pdev->subordinate->dev.kobj, &npb->kobj, + "of_node"); + if (ret) + pr_warn("%s: %s Error %d creating of_node link\n", + __func__, kobject_name(&dev->kobj), ret); + } + + + return 0; + +out_cset_fail: + pr_err("%s: %s Failed to apply changeset (err=%d)\n", __func__, + kobject_name(&dev->kobj), ret); + of_changeset_destroy(&cset); + return ret; +} + +static int pci_dev_uninstantiate(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np, *npb; + struct of_changeset cset; + int ret; + + /* device node must exist */ + np = dev->of_node; + if (!np) + return 0; + + /* and the auto-generated property */ + if (!of_property_read_bool(np, "auto-generated")) + return 0; + + of_changeset_init(&cset); + + ret = of_changeset_detach_node(&cset, np); + if (ret != 0) + goto out_cset_fail; + + npb = NULL; + if (pci_is_bridge(pdev)) + npb = pdev->subordinate->dev.of_node; + + if (npb != NULL) { + ret = of_changeset_detach_node(&cset, npb); + if (ret != 0) + goto out_cset_fail; + } + + ret = of_changeset_apply(&cset); + if (ret != 0) + goto out_cset_fail; + + dev->of_node = NULL; + if (npb != NULL) + pdev->subordinate->dev.of_node = NULL; + + pr_debug("%s: %s: %02x:%02x.%02x\n", __func__, + kobject_name(&dev->kobj), + pdev->bus->number, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + + /* TODO iterate over the properties and free */ + + return 0; + +out_cset_fail: + of_changeset_destroy(&cset); + + return ret; +} + +static int dovmgr_pci_notify(struct notifier_block *nb, + unsigned long action, void *arg) +{ + int ret; + + if (action == BUS_NOTIFY_ADD_DEVICE) + pci_dev_instantiate(to_pci_dev(arg)); + + ret = dovmgr_notifier_action(&dovmgr_pci_group, action, arg, + dovmgr_pci_item_match, dovmgr_item_notify); + + if (action == BUS_NOTIFY_REMOVED_DEVICE) + pci_dev_uninstantiate(to_pci_dev(arg)); + + return ret; +} + +static struct notifier_block dovmgr_pci_notifier = { + .notifier_call = dovmgr_pci_notify, +}; + +static int pci_instantiate_iterator(struct device *dev, void *data) +{ + return pci_dev_instantiate(to_pci_dev(dev)); +} + +static int dovmgr_pci_init(void) +{ + int ret; + + config_group_init_type_name(&dovmgr_pci_group, "pci", &dovmgr_pci_type); + ret = bus_register_notifier(&pci_bus_type, &dovmgr_pci_notifier); + if (ret != 0) { + pr_err("%s: bus_register_notifier() failed\n", __func__); + return ret; + } + + ret = bus_for_each_dev(&pci_bus_type, NULL, NULL, + pci_instantiate_iterator); + if (ret != 0) { + pr_err("%s: bus_for_each_dev() failed\n", __func__); + return ret; + } + + return 0; +} + +static void dovmgr_pci_cleanup(void) +{ + bus_unregister_notifier(&pci_bus_type, &dovmgr_pci_notifier); +} +#endif + +#if IS_ENABLED(CONFIG_USB) +static int dovmgr_usb_notify(struct notifier_block *nb, + unsigned long action, void *arg) +{ + return dovmgr_notifier_action(&dovmgr_usb_group, action, arg, + dovmgr_usb_item_match, dovmgr_item_notify); +} + +static struct notifier_block dovmgr_usb_notifier = { + .notifier_call = dovmgr_usb_notify, +}; + +static int dovmgr_usb_init(void) +{ + int ret; + + config_group_init_type_name(&dovmgr_usb_group, "usb", &dovmgr_usb_type); + ret = bus_register_notifier(&usb_bus_type, &dovmgr_usb_notifier); + if (ret != 0) { + pr_err("%s: bus_register_notifier() failed\n", __func__); + return ret; + } + return 0; +} + +static void dovmgr_usb_cleanup(void) +{ + bus_unregister_notifier(&usb_bus_type, &dovmgr_usb_notifier); +} +#endif + +static int __init dovmgr_init(void) +{ + int ret; + + config_group_init(&dovmgr_subsys.su_group); +#if IS_ENABLED(CONFIG_PCI) + configfs_add_default_group(&dovmgr_pci_group, + &dovmgr_subsys.su_group); +#endif +#if IS_ENABLED(CONFIG_USB) + configfs_add_default_group(&dovmgr_usb_group, + &dovmgr_subsys.su_group); +#endif + +#if IS_ENABLED(CONFIG_PCI) + ret = dovmgr_pci_init(); + if (ret != 0) + goto err_no_pci_init; +#endif +#if IS_ENABLED(CONFIG_USB) + ret = dovmgr_usb_init(); + if (ret != 0) + goto err_no_usb_init; +#endif + + ret = configfs_register_subsystem(&dovmgr_subsys); + if (ret != 0) { + pr_err("%s: failed to register subsys\n", __func__); + goto err_no_configfs; + } + pr_info("%s: OK\n", __func__); + return 0; + +err_no_configfs: +#if IS_ENABLED(CONFIG_USB) + dovmgr_usb_cleanup(); +err_no_usb_init: +#endif +#if IS_ENABLED(CONFIG_PCI) + dovmgr_pci_cleanup(); +err_no_pci_init: +#endif + return ret; +} +late_initcall(dovmgr_init); diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c index 59dcd97ee3de4..0e9d4131d0b33 100644 --- a/drivers/misc/eeprom/at24.c +++ b/drivers/misc/eeprom/at24.c @@ -618,7 +618,6 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id) struct at24_data *at24; int err; unsigned i, num_addresses; - u8 test_byte; if (client->dev.platform_data) { chip = *(struct at24_platform_data *)client->dev.platform_data; @@ -772,18 +771,6 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id) } } - i2c_set_clientdata(client, at24); - - /* - * Perform a one-byte test read to verify that the - * chip is functional. - */ - err = at24_read(at24, 0, &test_byte, 1); - if (err) { - err = -ENODEV; - goto err_clients; - } - at24->nvmem_config.name = dev_name(&client->dev); at24->nvmem_config.dev = &client->dev; at24->nvmem_config.read_only = !writable; @@ -805,6 +792,8 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id) goto err_clients; } + i2c_set_clientdata(client, at24); + dev_info(&client->dev, "%u byte %s EEPROM, %s, %u bytes/write\n", chip.byte_len, client->name, writable ? "writable" : "read-only", at24->write_max); diff --git a/drivers/misc/tieqep.c b/drivers/misc/tieqep.c new file mode 100644 index 0000000000000..bb69ad4468e26 --- /dev/null +++ b/drivers/misc/tieqep.c @@ -0,0 +1,754 @@ +/* + * TI eQEP driver for AM33xx devices + * + * Copyright (C) 2013 Nathaniel R. Lewis - http://teknoman117.wordpress.com/ + * Copyright (C) 2015 SoftPLC Corporation, Dick Hollenbeck + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * + * sysfs entries + * - position = absolute - current position; relative - last latched value + * - mode => 0 - absolute; 1 - relative + * - period => sampling period for the hardware + * - enable => 0 - eQEP disabled, 1 - eQEP enabled + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* eQEP register offsets from its base IO address */ +#define QPOSCNT 0x0000 +#define QPOSINIT 0x0004 +#define QPOSMAX 0x0008 +#define QPOSCMP 0x000C +#define QPOSILAT 0x0010 +#define QPOSSLAT 0x0014 +#define QPOSLAT 0x0018 +#define QUTMR 0x001C +#define QUPRD 0x0020 +#define QWDTMR 0x0024 +#define QWDPRD 0x0026 +#define QDECCTL 0x0028 +#define QEPCTL 0x002A +#define QCAPCTL 0x002C +#define QPOSCTL 0x002E +#define QEINT 0x0030 +#define QFLG 0x0032 +#define QCLR 0x0034 +#define QFRC 0x0036 +#define QEPSTS 0x0038 +#define QCTMR 0x003A +#define QCPRD 0x003C +#define QCTMRLAT 0x003E +#define QCPRDLAT 0x0040 +#define QREVID 0x005C + +#if 0 /* if you wanted another way to modify IP registers... */ +typedef volatile u32 REG32; +typedef volatile u16 REG16; +struct EQEP_REGS { + REG32 q_poscnt; /* 0x00 position counter */ + REG32 q_posinit; /* 0x04 position counter initialization */ + REG32 q_posmax; /* 0x08 maximum position count */ + REG32 q_poscmp; /* 0x0C position compare */ + REG32 q_posilat; /* 0x10 index position latch */ + REG32 q_posslat; /* 0x14 strobe position latch */ + REG32 q_poslat; /* 0x18 position counter latch */ + REG32 q_utmr; /* 0x1C unit timer */ + REG32 q_uprd; /* 0x20 unit period */ + REG16 q_wdtmr; /* 0x24 watchdog timer */ + REG16 q_wdprd; /* 0x26 watchdog period */ + REG16 q_decctl; /* 0x28 decoder control */ + REG16 q_epctl; /* 0x2A control register */ + REG16 q_capctl; /* 0x2C capture control */ + REG16 q_posctl; /* 0x2E position compare control */ + REG16 q_eint; /* 0x30 interrupt enable */ + REG16 q_flg; /* 0x32 interrupt flag */ + REG16 q_clr; /* 0x34 interrupt clear */ + REG16 q_frc; /* 0x36 interrupt force */ + REG16 q_epsts; /* 0x38 status */ + REG16 q_ctmr; /* 0x3A capture timer */ + REG16 q_cprd; /* 0x3C capture period */ + REG16 q_ctmrlat; /* 0x3E capture timer latch */ + REG16 q_prdlat; /* 0x40 capture period latch */ + char q_fill1[0x5c-0x40]; + REG32 q_revid; /* 0x5C revision id */ +}; +#endif + + +/* Bits for the QDECTL register */ +#define QSRC1 (1 << 15) +#define QSRC0 (1 << 14) +#define SOEN (1 << 13) +#define SPSEL (1 << 12) +#define XCR (1 << 11) +#define SWAP (1 << 10) +#define IGATE (1 << 9) +#define QAP (1 << 8) +#define QBP (1 << 7) +#define QIP (1 << 6) +#define QSP (1 << 5) + +/* Bits for the QEPCTL register */ +#define FREESOFT1 (1 << 15) +#define FREESOFT0 (1 << 14) +#define PCRM1 (1 << 13) +#define PCRM0 (1 << 12) +#define SEI1 (1 << 11) +#define SEI0 (1 << 10) +#define IEI1 (1 << 9) +#define IEI0 (1 << 8) +#define SWI (1 << 7) +#define SEL (1 << 6) +#define IEL1 (1 << 5) +#define IEL0 (1 << 4) +#define PHEN (1 << 3) +#define QCLM (1 << 2) +#define UTE (1 << 1) +#define WDE (1 << 0) + +/* Bits for the QCAPCTL register */ +#define CEN (1 << 15) +#define CCPS2 (1 << 6) +#define CCPS0 (1 << 5) +#define CCPS1 (1 << 4) +#define UPPS3 (1 << 3) +#define UPPS2 (1 << 2) +#define UPPS1 (1 << 1) +#define UPPS0 (1 << 0) + +/* Bits for the QPOSCTL register */ +#define PCSHDW (1 << 15) +#define PCLOAD (1 << 14) +#define PCPOL (1 << 13) +#define PCE (1 << 12) +#define PCSPW11 (1 << 11) +#define PCSPW10 (1 << 10) +#define PCSPW9 (1 << 9) +#define PCSPW8 (1 << 8) +#define PCSPW7 (1 << 7) +#define PCSPW6 (1 << 6) +#define PCSPW5 (1 << 5) +#define PCSPW4 (1 << 4) +#define PCSPW3 (1 << 3) +#define PCSPW2 (1 << 2) +#define PCSPW1 (1 << 1) +#define PCSPW0 (1 << 0) + +/* Bits for the interrupt registers */ +#define EQEP_INTERRUPT_MASK 0x0FFF +#define UTOF (1 << 11) + +/* Bits to control the clock in the PWMSS subsystem */ +#define PWMSS_EQEPCLK_EN BIT(4) +#define PWMSS_EQEPCLK_STOP_REQ BIT(5) +#define PWMSS_EQEPCLK_EN_ACK BIT(4) + +/* + * Modes for the eQEP unit + * Absolute - the position entry represents the current position of the encoder. + * Poll this value and it will be notified every period nanoseconds + * Relative - the position entry represents the last latched position of the encoder + * This value is latched every period nanoseconds and the internal counter + * is subsequenty reset + */ +#define TIEQEP_MODE_ABSOLUTE 0 +#define TIEQEP_MODE_RELATIVE 1 + +/* Structure defining the characteristics of the eQEP unit */ +struct eqep_chip +{ + /* Platform device for this eQEP unit */ + struct platform_device *pdev; + + /* Pointer to the base of the memory of the eQEP unit */ + void __iomem *mmio_base; + + /* SYSCLKOUT to the eQEP unit */ + u32 clk_rate; + + /* IRQ for the eQEP unit */ + u16 irq; + + /* Mode of the eQEP unit */ + u8 op_mode; + + /* work stuct for the notify userspace work */ + struct work_struct notify_work; + + /* Backup for driver suspension */ + u16 prior_qepctl; + u16 prior_qeint; +}; + +/* Notify userspace work */ +static void notify_handler(struct work_struct *work) +{ + /* Get a reference to the eQEP driver */ + struct eqep_chip *eqep = container_of(work, struct eqep_chip, notify_work); + + /* Notify the userspace */ + sysfs_notify(&eqep->pdev->dev.kobj, NULL, "position"); +} + +/* eQEP Interrupt handler */ +static irqreturn_t eqep_irq_handler(int irq, void *dev_id) +{ + /* Get the instance information */ + struct platform_device *pdev = dev_id; + struct eqep_chip *eqep = platform_get_drvdata(pdev); + + /* Get the interrupt flags */ + u16 iflags = readw(eqep->mmio_base + QFLG) & EQEP_INTERRUPT_MASK; + + /* Check the interrupt source(s) */ + if (iflags & UTOF) { + /* Handle the unit timer overflow interrupt by notifying any potential pollers */ + schedule_work(&eqep->notify_work); + } + + /* Clear interrupt flags (write back triggered flags to the clear register) */ + writew(iflags, eqep->mmio_base + QCLR); + + /* Return that the IRQ was handled successfully */ + return IRQ_HANDLED; +} + +/* Function to read whether the eQEP unit is enabled or disabled */ +static ssize_t eqep_get_enabled(struct device *dev, struct device_attribute *attr, char *buf) +{ + /* Get the instance structure */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + u16 enabled = 0; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + + /* Read the qep control register and mask all but the enabled bit */ + enabled = readw(eqep->mmio_base + QEPCTL) & PHEN; + + /* Return the target in string format */ + return sprintf(buf, "%u\n", (enabled) ? 1 : 0); +} + +/* Function to set if the eQEP is enabled */ +static ssize_t eqep_set_enabled(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + /* Get the instance structure */ + int rc; + u16 val; + u8 enabled; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + /* Convert the input string to an 8 bit uint */ + if ((rc = kstrtou8(buf, 0, &enabled))) + return rc; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + /* Get the existing state of QEPCTL */ + val = readw(eqep->mmio_base + QEPCTL); + + /* If we passed a number that is not 0, enable the eQEP */ + if (enabled) + /* Enable the eQEP (Set PHEN in QEPCTL) */ + val |= PHEN; + else + /* Disable the eQEP (Clear PHEN in QEPCTL) */ + val &= ~PHEN; + + /* Write flags back to control register */ + writew(val, eqep->mmio_base + QEPCTL); + + /* Return buffer length consumed (all) */ + return count; +} + +/* Function to read the current position of the eQEP */ +static ssize_t eqep_get_position(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + s32 position = 0; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + + if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) { + position = readl(eqep->mmio_base + QPOSCNT); + } else if (eqep->op_mode == TIEQEP_MODE_RELATIVE) { + /* in relative mode, use the last latched value of the eQEP hardware */ + position = readl(eqep->mmio_base + QPOSLAT); + dev_dbg(dev, "get_position:0x%08x\n", position); + } + + return sprintf(buf, "%d\n", position); +} + +/* Function to set the position of the eQEP hardware */ +static ssize_t eqep_set_position(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + s32 position; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtos32(buf, 0, &position))) + return rc; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + /* + * If we are in absolute mode, set the position of the encoder, + * discard relative mode because thats pointless + */ + if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) { + /* If absolute mode, set the current value of the eQEP hardware */ + writel(position, eqep->mmio_base + QPOSCNT); + } + + /* Return buffer length consumed (all) */ + return count; +} + +/* Function to read the period of the unit time event timer */ +static ssize_t eqep_get_timer_period(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + u64 period; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + /* Convert from counts per interrupt back into period_ns */ + period = readl(eqep->mmio_base + QUPRD); + period = period * NSEC_PER_SEC; + do_div(period, eqep->clk_rate); + + /* Otherwise write out the data */ + return sprintf(buf, "%llu\n", period); +} + +/* Function to set the unit timer period. 0 = off, greater than zero sets the period */ +static ssize_t eqep_set_timer_period(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + u16 tmp; + u64 period; + + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtou64(buf, 0, &period))) + return rc; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + /* Disable the unit timer before modifying its period register */ + tmp = readw(eqep->mmio_base + QEPCTL); + tmp &= ~(UTE | QCLM); + writew(tmp, eqep->mmio_base + QEPCTL); + + /* Zero the unit timer counter register */ + writel(0, eqep->mmio_base + QUTMR); + + /* If the timer is enabled (a non-zero period has been passed) */ + if (period) { + /* update the period */ + period = period * eqep->clk_rate; + do_div(period, NSEC_PER_SEC); + + dev_dbg(dev, "eqep_set_timer_period:%llu\n", period); + + writel(period, eqep->mmio_base + QUPRD); + + /* Enable unit timer, and latch QPOSLAT to QPOSCNT on timer expiration */ + tmp |= UTE | QCLM; + writew(tmp, eqep->mmio_base + QEPCTL); + } + + return count; +} + +/* Function to read the mode of the eQEP hardware */ +static ssize_t eqep_get_mode(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + return sprintf(buf, "%u\n", eqep->op_mode); +} + +/* Function to set the mode of the eQEP hardware */ +static ssize_t eqep_set_mode(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + u16 val; + u8 tmp_mode; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtou8(buf, 0, &tmp_mode))) + return rc; + + dev_dbg(dev, "eqep_set_mode:%d\n", tmp_mode); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + val = readw(eqep->mmio_base + QEPCTL); + + if (tmp_mode == TIEQEP_MODE_ABSOLUTE) { + /* + * In absolute mode, don't reset the hardware based on time, + * so disable the unit timer position reset (Set PCRM[1:0] = 0) + */ + val &= ~(PCRM1 | PCRM0); + + eqep->op_mode = TIEQEP_MODE_ABSOLUTE; + } else if (tmp_mode == TIEQEP_MODE_RELATIVE) { + /* + * In relative mode, latch the value of the eQEP hardware on the + * overflow of the unit timer. So enable the unit timer position reset + * (Set PCRM[1:0] = 3) + */ + val |= PCRM1 | PCRM0; + + eqep->op_mode = TIEQEP_MODE_RELATIVE; + } + + writew(val, eqep->mmio_base + QEPCTL); + + return count; +} + +/* Bind read/write functions to sysfs entries */ +static DEVICE_ATTR(enabled, 0644, eqep_get_enabled, eqep_set_enabled); +static DEVICE_ATTR(position, 0644, eqep_get_position, eqep_set_position); +static DEVICE_ATTR(period, 0644, eqep_get_timer_period, eqep_set_timer_period); +static DEVICE_ATTR(mode, 0644, eqep_get_mode, eqep_set_mode); + +/* Array holding all of the sysfs entries */ +static const struct attribute *eqep_attrs[] = { + &dev_attr_enabled.attr, + &dev_attr_position.attr, + &dev_attr_period.attr, + &dev_attr_mode.attr, + NULL, +}; + +/* Driver function group */ +static const struct attribute_group eqep_device_attr_group = { + .attrs = (struct attribute **) eqep_attrs, +}; + +/* Driver compatibility list */ +static struct of_device_id eqep_of_match[] = +{ + { .compatible = "ti,am33xx-eqep" }, + { } +}; + +/* Register our compatibilities for device trees */ +MODULE_DEVICE_TABLE(of, eqep_of_match); + +/* Create an instance of the eQEP driver */ +static int eqep_probe(struct platform_device *pdev) +{ + struct resource *r; + struct clk *clk; + struct eqep_chip *eqep; + struct pinctrl *pinctrl; + + u64 period; + u16 status; + u32 value; + + dev_info(&pdev->dev, "ver. 1.0\n"); + + /* Select pins provided through the device tree */ + pinctrl = devm_pinctrl_get_select_default(&pdev->dev); + if (IS_ERR(pinctrl)) + { + dev_warn(&pdev->dev, "unable to select pin group\n"); + } + + /* Allocate a eqep_driver object */ + eqep = devm_kzalloc(&pdev->dev, sizeof(struct eqep_chip), GFP_KERNEL); + if (!eqep) { + dev_err(&pdev->dev, "failed to allocate memory\n"); + return -ENOMEM; + } + + /* Get a handle to the system clock object */ + clk = devm_clk_get(pdev->dev.parent, "fck"); + if (IS_ERR(clk)) { + dev_err(&pdev->dev, "failed to get clock\n"); + return PTR_ERR(clk); + } + + /* Get the frequency of the system clock */ + eqep->clk_rate = clk_get_rate(clk); + if (!eqep->clk_rate) { + dev_err(&pdev->dev, "failed to get clock rate\n"); + return -EINVAL; + } + + /* Get a resource containing the IRQ for this eQEP controller */ + r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (unlikely(!r)) { + dev_err(&pdev->dev, "Invalid IRQ resource\n"); + return -ENODEV; + } + + /* Store the irq */ + eqep->irq = r->start; + + /* Get a resource containing the requested (from DT) memory address and range of eQEP controller */ + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!r) { + dev_err(&pdev->dev, "no memory resource defined\n"); + return -ENODEV; + } + + /* Remap the eQEP controller memory into our own memory space */ + eqep->mmio_base = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(eqep->mmio_base)) + return PTR_ERR(eqep->mmio_base); + + /* Store the platform device in our eQEP data structure for later usage */ + eqep->pdev = pdev; + + /* Subscribe to the eQEP interrupt */ + if (request_irq(eqep->irq, eqep_irq_handler, IRQF_IRQPOLL, "eqep_interrupt", pdev)) + { + dev_err(&pdev->dev, "unable to request irq for eQEP\n"); + return -ENODEV; + } + + /* Register controls to sysfs */ + if (sysfs_create_group(&pdev->dev.kobj, &eqep_device_attr_group)) + { + dev_err(&pdev->dev, "sysfs creation failed\n"); + return -EINVAL; + } + + /* set QDECCTL */ + status = 0; /* default to Quadrature count mode, QSRC1 & QSRC0 = 0 */ + + /* set QSRC1 & QSRC0 bits, one of 4 count_modes. */ + if (!of_property_read_u32(pdev->dev.of_node, "count_mode", &value) && value <= 3) { + status |= value << 14; + + /* + * in count up or count down mode, count on rising edge only + * not on both edges. + */ + if (value >= 2) + status |= XCR; + } + dev_info(&pdev->dev, "count_mode:%d\n", value); + + /* Should we invert the qa input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qa", &value)) + status = value ? status | QAP : status & ~QAP; + dev_info(&pdev->dev, "invert_qa:%d\n", value); + + /* Should we invert the qb input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qb", &value)) + status = value ? status | QBP : status & ~QBP; + dev_info(&pdev->dev, "invert_qb:%d\n", value); + + /* Should we invert the index input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qi", &value)) + status = value ? status | QIP : status & ~QIP; + dev_info(&pdev->dev, "invert_qi:%d\n", value); + + /* Should we invert the strobe input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qs", &value)) + status = value ? status | QSP : status & ~QSP; + dev_info(&pdev->dev, "invert_qs:%d\n", value); + + /* Should we swap the cha and chb inputs */ + if (!of_property_read_u32(pdev->dev.of_node, "swap_inputs", &value)) + status = value ? status | SWAP : status & ~SWAP; + dev_info(&pdev->dev, "swap_inputs:%d\n", value); + + dev_info(&pdev->dev, "QDECCTL:0x%04x\n", status); + + /* Write the decoder control settings back to the control register */ + writew(status, eqep->mmio_base + QDECCTL); + + writel( 0, eqep->mmio_base + QPOSINIT); + writel(~0, eqep->mmio_base + QPOSMAX); + writel( 0, eqep->mmio_base + QPOSCNT); + + dev_info(&pdev->dev, "QPOSINIT:0x%08x\n", readl(eqep->mmio_base + QPOSINIT)); + dev_info(&pdev->dev, "QPOSMAX:0x%08x\n", readl(eqep->mmio_base + QPOSMAX)); + dev_info(&pdev->dev, "QPOSCNT:0x%08x\n", readl(eqep->mmio_base + QPOSCNT)); + + status = UTOF; /* Enable Unit Time Period interrupt. */ + if (!of_property_read_u32(pdev->dev.of_node, "omit_interrupt", &value) && value) { + status = 0; /* no interrupt */ + } + writew(status, eqep->mmio_base + QEINT); + dev_info(&pdev->dev, "omit_interrupt:%d\n", value); + dev_info(&pdev->dev, "QEINT:0x%04x\n", status); + + /* Calculate the timer ticks per second */ + period = 1000000000; + period = period * eqep->clk_rate; + do_div(period, NSEC_PER_SEC); + + /* Set this period into the unit timer period register */ + writel(period, eqep->mmio_base + QUPRD); + dev_info(&pdev->dev, "QUPRD:0x%08x\n", (u32) period); + + /* + * Enable the eQEP with basic position counting turned on + * PHEN - Quadrature position counter enable bit + * UTE - unit timer enable + * QCLM - latch QPOSLAT to QPOSCNT upon unit timer overflow + * IEL0 - Latch QPOSILAT on index signal. Rising or falling, IEL[1:0] = 0 is reserved + * SWI - Software initialization of position count register, i.e. set QPOSCNT <= QPOSINIT, + * but this bit was not being reset by hardware as advertised in TRM, + * (so omit & clear QPOSCNT manually elsewhere?) + */ + status = PHEN | UTE | QCLM | IEL0 | SWI; + writew(status, eqep->mmio_base + QEPCTL); + dev_info(&pdev->dev, "QEPCTL:0x%04x write\n", status); + dev_info(&pdev->dev, "QEPCTL:0x%04x read\n", readw(eqep->mmio_base + QEPCTL)); + + /* We default to absolute mode */ + eqep->op_mode = TIEQEP_MODE_ABSOLUTE; + + /* Enable the power management runtime */ + pm_runtime_enable(&pdev->dev); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(&pdev->dev); + + /* Initialize the notify work struture */ + INIT_WORK(&eqep->notify_work, notify_handler); + + /* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(&pdev->dev); + + /* Set the platform driver data to the data object we've been creating for the eQEP unit */ + platform_set_drvdata(pdev, eqep); + + /* Success! */ + dev_info(&pdev->dev, "irq:%d, clk_rate:%u\n", eqep->irq, eqep->clk_rate); + return 0; +} + +/* Remove an instance of the eQEP driver */ +static int eqep_remove(struct platform_device *pdev) +{ + /* Get the eQEP driver data from the platform device structure */ + struct eqep_chip *eqep = platform_get_drvdata(pdev); + + /* Cancel work */ + cancel_work_sync(&eqep->notify_work); + + /* Unmap from sysfs */ + sysfs_remove_group(&pdev->dev.kobj, &eqep_device_attr_group); + + /* Release important assets */ + free_irq(eqep->irq, pdev); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(&pdev->dev); + + /* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); + + /* Disable the runtime power management of this device */ + pm_runtime_disable(&pdev->dev); + + /* Return success */ + return 0; +} + +/* Power management suspend device */ +static int eqep_suspend(struct device *dev) +{ + /* Get the eqep driver information */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + u16 tmp; + + /* Shut down interrupts */ + eqep->prior_qeint = readw(eqep->mmio_base + QEINT); + tmp = eqep->prior_qeint & ~UTOF; + writew(tmp, eqep->mmio_base + QEINT); + + /* Get the existing state of QEPCTL */ + eqep->prior_qepctl = readw(eqep->mmio_base + QEPCTL); + + /* Disable eQEP controller */ + writew(eqep->prior_qepctl & ~PHEN, eqep->mmio_base + QEPCTL); + + /* Decrement the device usage count and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(dev); + + /* Return success */ + return 0; +} + +/* Power management wake device back up */ +static int eqep_resume(struct device *dev) +{ + /* Get the eqep driver information */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + /* Restore interrupt enabled register */ + writew(eqep->prior_qeint, eqep->mmio_base + QEINT); + + /* Restore prior qep control register */ + writew(eqep->prior_qepctl, eqep->mmio_base + QEPCTL); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + + /* Success */ + return 0; +} + +/* create pm functions object */ +static SIMPLE_DEV_PM_OPS(eqep_pm_ops, eqep_suspend, eqep_resume); + +/* Platform driver information */ +static struct platform_driver eqep_driver = { + .driver = { + .name = "eqep", + .owner = THIS_MODULE, + .pm = &eqep_pm_ops, + .of_match_table = eqep_of_match, + }, + .probe = eqep_probe, + .remove = eqep_remove, +}; + +/* Register this platform driver */ +module_platform_driver(eqep_driver); + +/* Module information */ +MODULE_DESCRIPTION("TI eQEP driver"); +MODULE_AUTHOR("Nathaniel R. Lewis"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c index 0a03fe103cf2d..2155a04b2cb84 100644 --- a/drivers/net/ethernet/ti/davinci_mdio.c +++ b/drivers/net/ethernet/ti/davinci_mdio.c @@ -108,6 +108,10 @@ struct davinci_mdio_data { u32 clk_div; }; +#if IS_ENABLED(CONFIG_OF) +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask); +#endif + static void davinci_mdio_init_clk(struct davinci_mdio_data *data) { u32 mdio_in, div, mdio_out_khz, access_time; @@ -175,6 +179,12 @@ static int davinci_mdio_reset(struct mii_bus *bus) /* restrict mdio bus to live phys only */ dev_info(data->dev, "detected phy mask %x\n", ~phy_mask); phy_mask = ~phy_mask; + + #if IS_ENABLED(CONFIG_OF) + if (of_machine_is_compatible("ti,am335x-bone")) + davinci_mdio_update_dt_from_phymask(phy_mask); + #endif + } else { /* desperately scan all phys */ dev_warn(data->dev, "no live phy, scanning all\n"); @@ -490,6 +500,93 @@ static int davinci_mdio_runtime_resume(struct device *dev) davinci_mdio_enable(data); return 0; } +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask) +{ + int i, len, skip; + u32 addr; + __be32 *old_phy_p, *phy_id_p; + struct property *phy_id_property = NULL; + struct device_node *node_p, *slave_p; + + addr = 0; + + for (i = 0; i < PHY_MAX_ADDR; i++) { + if ((phy_mask & (1 << i)) == 0) { + addr = (u32) i; + break; + } + } + + for_each_compatible_node(node_p, NULL, "ti,cpsw") { + for_each_node_by_name(slave_p, "slave") { + +#if IS_ENABLED(CONFIG_OF_OVERLAY) + skip = 1; + // Hack, the overlay fixup "slave" doesn't have phy-mode... + old_phy_p = (__be32 *) of_get_property(slave_p, "phy-mode", &len); + + if (len != (sizeof(__be32 *) * 1)) + { + skip = 0; + } + + if (skip) { +#endif + + old_phy_p = (__be32 *) of_get_property(slave_p, "phy_id", &len); + + if (len != (sizeof(__be32 *) * 2)) + goto err_out; + + if (old_phy_p) { + + phy_id_property = kzalloc(sizeof(*phy_id_property), GFP_KERNEL); + + if (! phy_id_property) + goto err_out; + + phy_id_property->length = len; + phy_id_property->name = kstrdup("phy_id", GFP_KERNEL); + phy_id_property->value = kzalloc(len, GFP_KERNEL); + + if (! phy_id_property->name) + goto err_out; + + if (! phy_id_property->value) + goto err_out; + + memcpy(phy_id_property->value, old_phy_p, len); + + phy_id_p = (__be32 *) phy_id_property->value + 1; + + *phy_id_p = cpu_to_be32(addr); + + of_update_property(slave_p, phy_id_property); + pr_info("davinci_mdio: dt: updated phy_id[%d] from phy_mask[%x]\n", addr, phy_mask); + + ++addr; + } +#if IS_ENABLED(CONFIG_OF_OVERLAY) + } +#endif + } + } + + return; + +err_out: + + if (phy_id_property) { + if (phy_id_property->name) + kfree(phy_id_property->name); + + if (phy_id_property->value) + kfree(phy_id_property->value); + + if (phy_id_property) + kfree(phy_id_property); + } +} #endif #ifdef CONFIG_PM_SLEEP diff --git a/drivers/net/ethernet/wiznet/Kconfig b/drivers/net/ethernet/wiznet/Kconfig index 1981e88c18dc4..0422775e16592 100644 --- a/drivers/net/ethernet/wiznet/Kconfig +++ b/drivers/net/ethernet/wiznet/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # # WIZnet devices configuration # diff --git a/drivers/net/ethernet/wiznet/Makefile b/drivers/net/ethernet/wiznet/Makefile index 1e05e1a842086..78104f0bf4153 100644 --- a/drivers/net/ethernet/wiznet/Makefile +++ b/drivers/net/ethernet/wiznet/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_WIZNET_W5100) += w5100.o obj-$(CONFIG_WIZNET_W5100_SPI) += w5100-spi.o obj-$(CONFIG_WIZNET_W5300) += w5300.o diff --git a/drivers/net/ethernet/wiznet/w5100-spi.c b/drivers/net/ethernet/wiznet/w5100-spi.c index 93a2d3c07303e..2b4126d2427d8 100644 --- a/drivers/net/ethernet/wiznet/w5100-spi.c +++ b/drivers/net/ethernet/wiznet/w5100-spi.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Ethernet driver for the WIZnet W5100/W5200/W5500 chip. * * Copyright (C) 2016 Akinobu Mita * - * Licensed under the GPL-2 or later. - * * Datasheet: * http://www.wiznet.co.kr/wp-content/uploads/wiznethome/Chip/W5100/Document/W5100_Datasheet_v1.2.6.pdf * http://wiznethome.cafe24.com/wp-content/uploads/wiznethome/Chip/W5200/Documents/W5200_DS_V140E.pdf @@ -16,6 +15,7 @@ #include #include #include +#include #include #include "w5100.h" @@ -410,14 +410,32 @@ static const struct w5100_ops w5500_ops = { .init = w5500_spi_init, }; +static const struct of_device_id w5100_of_match[] = { + { .compatible = "wiznet,w5100", .data = (const void*)W5100, }, + { .compatible = "wiznet,w5200", .data = (const void*)W5200, }, + { .compatible = "wiznet,w5500", .data = (const void*)W5500, }, + { }, +}; +MODULE_DEVICE_TABLE(of, w5100_of_match); + static int w5100_spi_probe(struct spi_device *spi) { - const struct spi_device_id *id = spi_get_device_id(spi); + const struct of_device_id *of_id; const struct w5100_ops *ops; + kernel_ulong_t driver_data; int priv_size; const void *mac = of_get_mac_address(spi->dev.of_node); - switch (id->driver_data) { + if (spi->dev.of_node) { + of_id = of_match_device(w5100_of_match, &spi->dev); + if (!of_id) + return -ENODEV; + driver_data = (kernel_ulong_t)of_id->data; + } else { + driver_data = spi_get_device_id(spi)->driver_data; + } + + switch (driver_data) { case W5100: ops = &w5100_spi_ops; priv_size = 0; @@ -454,6 +472,7 @@ static struct spi_driver w5100_spi_driver = { .driver = { .name = "w5100", .pm = &w5100_pm_ops, + .of_match_table = w5100_of_match, }, .probe = w5100_spi_probe, .remove = w5100_spi_remove, diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c index 2bdfb39215e9c..bede1ff289c59 100644 --- a/drivers/net/ethernet/wiznet/w5100.c +++ b/drivers/net/ethernet/wiznet/w5100.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Ethernet driver for the WIZnet W5100 chip. * * Copyright (C) 2006-2008 WIZnet Co.,Ltd. * Copyright (C) 2012 Mike Sinkovsky - * - * Licensed under the GPL-2 or later. */ #include @@ -219,7 +218,6 @@ static inline int __w5100_write_direct(struct net_device *ndev, u32 addr, static inline int w5100_write_direct(struct net_device *ndev, u32 addr, u8 data) { __w5100_write_direct(ndev, addr, data); - mmiowb(); return 0; } @@ -236,7 +234,6 @@ static int w5100_write16_direct(struct net_device *ndev, u32 addr, u16 data) { __w5100_write_direct(ndev, addr, data >> 8); __w5100_write_direct(ndev, addr + 1, data); - mmiowb(); return 0; } @@ -260,8 +257,6 @@ static int w5100_writebulk_direct(struct net_device *ndev, u32 addr, for (i = 0; i < len; i++, addr++) __w5100_write_direct(ndev, addr, *buf++); - mmiowb(); - return 0; } @@ -375,7 +370,6 @@ static int w5100_readbulk_indirect(struct net_device *ndev, u32 addr, u8 *buf, for (i = 0; i < len; i++) *buf++ = w5100_read_direct(ndev, W5100_IDM_DR); - mmiowb(); spin_unlock_irqrestore(&mmio_priv->reg_lock, flags); return 0; @@ -394,7 +388,6 @@ static int w5100_writebulk_indirect(struct net_device *ndev, u32 addr, for (i = 0; i < len; i++) __w5100_write_direct(ndev, W5100_IDM_DR, *buf++); - mmiowb(); spin_unlock_irqrestore(&mmio_priv->reg_lock, flags); return 0; @@ -835,7 +828,7 @@ static void w5100_tx_work(struct work_struct *work) w5100_tx_skb(priv->ndev, skb); } -static int w5100_start_tx(struct sk_buff *skb, struct net_device *ndev) +static netdev_tx_t w5100_start_tx(struct sk_buff *skb, struct net_device *ndev) { struct w5100_priv *priv = netdev_priv(ndev); @@ -1164,7 +1157,7 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops, INIT_WORK(&priv->setrx_work, w5100_setrx_work); INIT_WORK(&priv->restart_work, w5100_restart_work); - if (mac_addr) + if (!IS_ERR_OR_NULL(mac_addr)) memcpy(ndev->dev_addr, mac_addr, ETH_ALEN); else eth_hw_addr_random(ndev); diff --git a/drivers/net/ethernet/wiznet/w5100.h b/drivers/net/ethernet/wiznet/w5100.h index 17983a3b8d6c6..5d3d4b541fec4 100644 --- a/drivers/net/ethernet/wiznet/w5100.h +++ b/drivers/net/ethernet/wiznet/w5100.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Ethernet driver for the WIZnet W5100 chip. * * Copyright (C) 2006-2008 WIZnet Co.,Ltd. * Copyright (C) 2012 Mike Sinkovsky - * - * Licensed under the GPL-2 or later. */ enum { diff --git a/drivers/net/ethernet/wiznet/w5300.c b/drivers/net/ethernet/wiznet/w5300.c index 56ae573001e8e..6ba2747779ce5 100644 --- a/drivers/net/ethernet/wiznet/w5300.c +++ b/drivers/net/ethernet/wiznet/w5300.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Ethernet driver for the WIZnet W5300 chip. * * Copyright (C) 2008-2009 WIZnet Co.,Ltd. * Copyright (C) 2011 Taehun Kim gmail.com> * Copyright (C) 2012 Mike Sinkovsky - * - * Licensed under the GPL-2 or later. */ #include @@ -141,7 +140,6 @@ static u16 w5300_read_indirect(struct w5300_priv *priv, u16 addr) spin_lock_irqsave(&priv->reg_lock, flags); w5300_write_direct(priv, W5300_IDM_AR, addr); - mmiowb(); data = w5300_read_direct(priv, W5300_IDM_DR); spin_unlock_irqrestore(&priv->reg_lock, flags); @@ -154,9 +152,7 @@ static void w5300_write_indirect(struct w5300_priv *priv, u16 addr, u16 data) spin_lock_irqsave(&priv->reg_lock, flags); w5300_write_direct(priv, W5300_IDM_AR, addr); - mmiowb(); w5300_write_direct(priv, W5300_IDM_DR, data); - mmiowb(); spin_unlock_irqrestore(&priv->reg_lock, flags); } @@ -192,7 +188,6 @@ static int w5300_command(struct w5300_priv *priv, u16 cmd) unsigned long timeout = jiffies + msecs_to_jiffies(100); w5300_write(priv, W5300_S0_CR, cmd); - mmiowb(); while (w5300_read(priv, W5300_S0_CR) != 0) { if (time_after(jiffies, timeout)) @@ -241,18 +236,15 @@ static void w5300_write_macaddr(struct w5300_priv *priv) w5300_write(priv, W5300_SHARH, ndev->dev_addr[4] << 8 | ndev->dev_addr[5]); - mmiowb(); } static void w5300_hw_reset(struct w5300_priv *priv) { w5300_write_direct(priv, W5300_MR, MR_RST); - mmiowb(); mdelay(5); w5300_write_direct(priv, W5300_MR, priv->indirect ? MR_WDF(7) | MR_PB | MR_IND : MR_WDF(7) | MR_PB); - mmiowb(); w5300_write(priv, W5300_IMR, 0); w5300_write_macaddr(priv); @@ -264,24 +256,20 @@ static void w5300_hw_reset(struct w5300_priv *priv) w5300_write32(priv, W5300_TMSRL, 64 << 24); w5300_write32(priv, W5300_TMSRH, 0); w5300_write(priv, W5300_MTYPE, 0x00ff); - mmiowb(); } static void w5300_hw_start(struct w5300_priv *priv) { w5300_write(priv, W5300_S0_MR, priv->promisc ? S0_MR_MACRAW : S0_MR_MACRAW_MF); - mmiowb(); w5300_command(priv, S0_CR_OPEN); w5300_write(priv, W5300_S0_IMR, S0_IR_RECV | S0_IR_SENDOK); w5300_write(priv, W5300_IMR, IR_S0); - mmiowb(); } static void w5300_hw_close(struct w5300_priv *priv) { w5300_write(priv, W5300_IMR, 0); - mmiowb(); w5300_command(priv, S0_CR_CLOSE); } @@ -365,14 +353,13 @@ static void w5300_tx_timeout(struct net_device *ndev) netif_wake_queue(ndev); } -static int w5300_start_tx(struct sk_buff *skb, struct net_device *ndev) +static netdev_tx_t w5300_start_tx(struct sk_buff *skb, struct net_device *ndev) { struct w5300_priv *priv = netdev_priv(ndev); netif_stop_queue(ndev); w5300_write_frame(priv, skb->data, skb->len); - mmiowb(); ndev->stats.tx_packets++; ndev->stats.tx_bytes += skb->len; dev_kfree_skb(skb); @@ -419,7 +406,6 @@ static int w5300_napi_poll(struct napi_struct *napi, int budget) if (rx_count < budget) { napi_complete_done(napi, rx_count); w5300_write(priv, W5300_IMR, IR_S0); - mmiowb(); } return rx_count; @@ -434,7 +420,6 @@ static irqreturn_t w5300_interrupt(int irq, void *ndev_instance) if (!ir) return IRQ_NONE; w5300_write(priv, W5300_S0_IR, ir); - mmiowb(); if (ir & S0_IR_SENDOK) { netif_dbg(priv, tx_done, ndev, "tx done\n"); @@ -444,7 +429,6 @@ static irqreturn_t w5300_interrupt(int irq, void *ndev_instance) if (ir & S0_IR_RECV) { if (napi_schedule_prep(&priv->napi)) { w5300_write(priv, W5300_IMR, 0); - mmiowb(); __napi_schedule(&priv->napi); } } @@ -661,8 +645,7 @@ static int w5300_remove(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP static int w5300_suspend(struct device *dev) { - struct platform_device *pdev = to_platform_device(dev); - struct net_device *ndev = platform_get_drvdata(pdev); + struct net_device *ndev = dev_get_drvdata(dev); struct w5300_priv *priv = netdev_priv(ndev); if (netif_running(ndev)) { @@ -676,8 +659,7 @@ static int w5300_suspend(struct device *dev) static int w5300_resume(struct device *dev) { - struct platform_device *pdev = to_platform_device(dev); - struct net_device *ndev = platform_get_drvdata(pdev); + struct net_device *ndev = dev_get_drvdata(dev); struct w5300_priv *priv = netdev_priv(ndev); if (!netif_running(ndev)) { diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c index c1ab976cc8006..414ee5ef92f35 100644 --- a/drivers/net/phy/dp83867.c +++ b/drivers/net/phy/dp83867.c @@ -251,9 +251,11 @@ static int dp83867_config_init(struct phy_device *phydev) return ret; } - if ((phydev->interface >= PHY_INTERFACE_MODE_RGMII_ID) && - (phydev->interface <= PHY_INTERFACE_MODE_RGMII_RXID)) { + if ((phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) || + (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID) || + (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)) { val = phy_read_mmd(phydev, DP83867_DEVADDR, DP83867_RGMIICTL); + val &= ~(DP83867_RGMII_TX_CLK_DELAY_EN | DP83867_RGMII_RX_CLK_DELAY_EN); if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) val |= (DP83867_RGMII_TX_CLK_DELAY_EN | DP83867_RGMII_RX_CLK_DELAY_EN); diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 47d2ef2fb9b33..03798e50b1766 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -397,6 +397,7 @@ int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd) struct mii_ioctl_data *mii_data = if_mii(ifr); u16 val = mii_data->val_in; bool change_autoneg = false; + int ret; switch (cmd) { case SIOCGMIIPHY: @@ -404,12 +405,29 @@ int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd) /* fall through */ case SIOCGMIIREG: + if (mdio_phy_id_is_c45(mii_data->phy_id)) { + ret = phy_read_mmd(phydev, + mdio_phy_id_devad(mii_data->phy_id), + mii_data->reg_num); + if (ret < 0) + return ret; + mii_data->val_out = ret; + return 0; + } + mii_data->val_out = mdiobus_read(phydev->mdio.bus, mii_data->phy_id, mii_data->reg_num); return 0; case SIOCSMIIREG: + if (mdio_phy_id_is_c45(mii_data->phy_id)) { + ret = phy_write_mmd(phydev, + mdio_phy_id_devad(mii_data->phy_id), + mii_data->reg_num, + mii_data->val_in); + return ret; + } if (mii_data->phy_id == phydev->mdio.addr) { switch (mii_data->reg_num) { case MII_BMCR: diff --git a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig index 9d99eb42d9176..d8c6fd64d3eea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/Kconfig +++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig @@ -85,3 +85,14 @@ config BRCMDBG select WANT_DEV_COREDUMP ---help--- Selecting this enables additional code for debug purposes. + +config BRCMFMAC_PCIE_BARWIN_SZ + bool "Custom PCIE BAR window size support for FullMAC driver" + depends on BRCMFMAC + depends on PCI + default n + ---help--- + If you say Y here, the FMAC driver will use custom PCIE BAR + window size. Say Y to allow developers to use custom PCIE + BAR window size when HOST PCIE IP can support less then 4MB + BAR window. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 9f2d0b0cf6e5c..dece0d3ba182e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -98,6 +98,8 @@ struct brcmf_proto_bcdc_header { * plus any space that might be needed * for bus alignment padding. */ +#define ROUND_UP_MARGIN 2048 + struct brcmf_bcdc { u16 reqid; u8 bus_header[BUS_HEADER_LEN]; @@ -471,7 +473,7 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES; drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN + - sizeof(struct brcmf_proto_bcdc_dcmd); + sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN; return 0; fail: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index dd6e27513cc1b..ea439eed18a67 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -47,6 +47,7 @@ #include "sdio.h" #include "core.h" #include "common.h" +#include "cfg80211.h" #define SDIOH_API_ACCESS_RETRY_LIMIT 2 @@ -54,6 +55,8 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 +#define SDIO_4373_FUNC2_BLOCKSIZE 256 +#define SDIO_435X_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -1000,7 +1003,7 @@ static void brcmf_sdiod_freezer_detach(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ -static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) { sdiodev->state = BRCMF_SDIOD_DOWN; if (sdiodev->bus) { @@ -1035,9 +1038,10 @@ static void brcmf_sdiod_host_fixup(struct mmc_host *host) host->caps |= MMC_CAP_NONREMOVABLE; } -static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; + unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE; sdiodev->num_funcs = 2; @@ -1049,11 +1053,28 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) sdio_release_host(sdiodev->func[1]); goto out; } - ret = sdio_set_block_size(sdiodev->func[2], SDIO_FUNC2_BLOCKSIZE); + + switch (sdiodev->func[0]->device) { + case SDIO_DEVICE_ID_CYPRESS_4373: + f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; + break; + case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: + case SDIO_DEVICE_ID_BROADCOM_4354: + case SDIO_DEVICE_ID_BROADCOM_4356: + f2_blksz = SDIO_435X_FUNC2_BLOCKSIZE; + break; + default: + break; + } + + ret = sdio_set_block_size(sdiodev->func[2], f2_blksz); if (ret) { brcmf_err("Failed to set F2 blocksize\n"); sdio_release_host(sdiodev->func[1]); goto out; + } else { + brcmf_dbg(SDIO, "set F2 blocksize to %d\n", f2_blksz); } /* increase F2 timeout */ @@ -1106,7 +1127,10 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_89359), { /* end: all zeroes */ } }; MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids); @@ -1173,6 +1197,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, dev_set_drvdata(&func->dev, bus_if); dev_set_drvdata(&sdiodev->func[1]->dev, bus_if); sdiodev->dev = &sdiodev->func[1]->dev; + dev_set_drvdata(&sdiodev->func[2]->dev, bus_if); brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN); @@ -1189,6 +1214,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, fail: dev_set_drvdata(&func->dev, NULL); dev_set_drvdata(&sdiodev->func[1]->dev, NULL); + dev_set_drvdata(&sdiodev->func[2]->dev, NULL); kfree(sdiodev->func[0]); kfree(sdiodev); kfree(bus_if); @@ -1245,14 +1271,26 @@ static int brcmf_ops_sdio_suspend(struct device *dev) struct brcmf_bus *bus_if; struct brcmf_sdio_dev *sdiodev; mmc_pm_flag_t sdio_flags; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; func = container_of(dev, struct sdio_func, dev); + bus_if = dev_get_drvdata(dev); + config = bus_if->drvr->config; + brcmf_dbg(SDIO, "Enter: F%d\n", func->num); + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + if (func->num != SDIO_FUNC_1) return 0; - - bus_if = dev_get_drvdata(dev); sdiodev = bus_if->bus_priv.sdio; brcmf_sdiod_freezer_on(sdiodev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 163ddc49f9511..95efdb6076687 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -71,6 +71,7 @@ struct brcmf_bus_dcmd { * @wowl_config: specify if dongle is configured for wowl when going to suspend * @get_ramsize: obtain size of device memory. * @get_memdump: obtain device memory dump in provided buffer. + * @get_fwname: obtain firmware name. * * This structure provides an abstract interface towards the * bus specific driver. For control messages to common driver @@ -87,6 +88,9 @@ struct brcmf_bus_ops { void (*wowl_config)(struct device *dev, bool enabled); size_t (*get_ramsize)(struct device *dev); int (*get_memdump)(struct device *dev, void *data, size_t len); + int (*get_fwname)(struct device *dev, uint chip, uint chiprev, + unsigned char *fw_name); + int (*reset)(struct device *dev); }; @@ -157,6 +161,7 @@ struct brcmf_bus { const struct brcmf_bus_ops *ops; struct brcmf_bus_msgbuf *msgbuf; + bool allow_skborphan; }; /* @@ -224,6 +229,22 @@ int brcmf_bus_get_memdump(struct brcmf_bus *bus, void *data, size_t len) return bus->ops->get_memdump(bus->dev, data, len); } +static inline +int brcmf_bus_get_fwname(struct brcmf_bus *bus, uint chip, uint chiprev, + unsigned char *fw_name) +{ + return bus->ops->get_fwname(bus->dev, chip, chiprev, fw_name); +} + +static inline +int brcmf_bus_reset(struct brcmf_bus *bus) +{ + if (!bus->ops->reset) + return -EOPNOTSUPP; + + return bus->ops->reset(bus->dev); +} + /* * interface functions from common layer */ @@ -239,6 +260,10 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings); void brcmf_detach(struct device *dev); /* Indication from bus module that dongle should be reset */ void brcmf_dev_reset(struct device *dev); +/* Request from bus module to initiate a coredump */ +void brcmf_dev_coredump(struct device *dev); +/* Indication that firmware has halted or crashed */ +void brcmf_fw_crashed(struct device *dev); /* Configure the "global" bus state used by upper layers */ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); @@ -246,6 +271,7 @@ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); int brcmf_bus_started(struct device *dev); s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len); void brcmf_bus_add_txhdrlen(struct device *dev, uint len); +int brcmf_fwlog_attach(struct device *dev); #ifdef CONFIG_BRCMFMAC_SDIO void brcmf_sdio_exit(void); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index cd6c5ece9a5d3..90953c2dc5499 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -33,6 +33,7 @@ #include "p2p.h" #include "btcoex.h" #include "pno.h" +#include "fwsignal.h" #include "cfg80211.h" #include "feature.h" #include "fwil.h" @@ -65,6 +66,7 @@ #define RSN_AKM_PSK 2 /* Pre-shared Key */ #define RSN_AKM_SHA256_1X 5 /* SHA256, 802.1X */ #define RSN_AKM_SHA256_PSK 6 /* SHA256, Pre-shared Key */ +#define RSN_AKM_SAE 8 /* SAE */ #define RSN_CAP_LEN 2 /* Length of RSN capabilities */ #define RSN_CAP_PTK_REPLAY_CNTR_MASK (BIT(2) | BIT(3)) #define RSN_CAP_MFPR_MASK BIT(6) @@ -239,6 +241,9 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; +#define WLC_E_IF_ROLE_STA 0 /* Infra STA */ +#define WLC_E_IF_ROLE_AP 1 /* Access Point */ + static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -1367,7 +1372,32 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } -static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) +static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, + u16 pwd_len) +{ + struct brcmf_wsec_sae_pwd_le sae_pwd; + int err; + + if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { + brcmf_err("sae_password must be less than %d\n", + BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); + return -EINVAL; + } + + sae_pwd.key_len = cpu_to_le16(pwd_len); + memcpy(sae_pwd.key, pwd_data, pwd_len); + + err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, + sizeof(sae_pwd)); + if (err < 0) + brcmf_err("failed to set SAE password in firmware (len=%u)\n", + pwd_len); + + return err; +} + +static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, + bool locally_generated) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy); s32 err = 0; @@ -1384,7 +1414,7 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) if ((vif->wdev.iftype == NL80211_IFTYPE_STATION) || (vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT)) cfg80211_disconnected(vif->wdev.netdev, reason, NULL, 0, - true, GFP_KERNEL); + locally_generated, GFP_KERNEL); } clear_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state); clear_bit(BRCMF_SCAN_STATUS_SUPPRESS, &cfg->scan_status); @@ -1561,7 +1591,7 @@ brcmf_cfg80211_leave_ibss(struct wiphy *wiphy, struct net_device *ndev) return 0; } - brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING); + brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING, true); brcmf_net_setcarrier(ifp, false); brcmf_dbg(TRACE, "Exit\n"); @@ -1581,6 +1611,8 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2) val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; + else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) + val = WPA3_AUTH_SAE_PSK; else val = WPA_AUTH_DISABLED; brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); @@ -1611,6 +1643,10 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, val = 1; brcmf_dbg(CONN, "shared key\n"); break; + case NL80211_AUTHTYPE_SAE: + val = 3; + brcmf_dbg(CONN, "SAE authentication\n"); + break; default: val = 2; brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type); @@ -1709,6 +1745,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; s32 val; s32 err; + s32 okc_enable; const struct brcmf_tlv *rsn_ie; const u8 *ie; u32 ie_len; @@ -1718,6 +1755,8 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 count; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -1760,6 +1799,26 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) case WLAN_AKM_SUITE_PSK: val = WPA2_AUTH_PSK; break; + case WLAN_AKM_SUITE_FT_8021X: + val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT; + profile->is_ft = true; + if (sme->want_1x) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + break; + case WLAN_AKM_SUITE_FT_PSK: + val = WPA2_AUTH_PSK | WPA2_AUTH_FT; + profile->is_ft = true; + break; + default: + brcmf_err("invalid cipher group (%d)\n", + sme->crypto.cipher_group); + return -EINVAL; + } + } else if (val & WPA3_AUTH_SAE_PSK) { + switch (sme->crypto.akm_suites[0]) { + case WLAN_AKM_SUITE_SAE: + val = WPA3_AUTH_SAE_PSK; + break; default: brcmf_err("invalid cipher group (%d)\n", sme->crypto.cipher_group); @@ -1767,8 +1826,17 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { brcmf_dbg(INFO, "using 1X offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + brcmf_err("get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP)) goto skip_mfp_config; @@ -1834,7 +1902,8 @@ brcmf_set_sharedkey(struct net_device *ndev, brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n", sec->wpa_versions, sec->cipher_pairwise); - if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2)) + if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)) return 0; if (!(sec->cipher_pairwise & @@ -2039,7 +2108,13 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.psk) { + if (sme->crypto.sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + } + + if (sme->crypto.psk && + profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { err = -EINVAL; goto done; @@ -2057,12 +2132,23 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) { + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); - if (err) + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + brcmf_err("failed to clean up user-space RSNE\n"); goto done; + } + err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, + sme->crypto.sae_pwd_len); + if (!err && sme->crypto.psk) + err = brcmf_set_pmk(ifp, sme->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); } + if (err) + goto done; /* Join with specific BSSID and cached SSID * If SSID is zero join based on BSSID only @@ -2380,6 +2466,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, if (!ext_key) key->flags = BRCMF_PRIMARY_KEY; + if (params->seq && params->seq_len == 6) { + /* rx iv */ + u8 *ivptr; + + ivptr = (u8 *)params->seq; + key->rxiv.hi = (ivptr[5] << 24) | (ivptr[4] << 16) | + (ivptr[3] << 8) | ivptr[2]; + key->rxiv.lo = (ivptr[1] << 8) | ivptr[0]; + key->iv_initialized = true; + } + switch (params->cipher) { case WLAN_CIPHER_SUITE_WEP40: key->algo = CRYPTO_ALGO_WEP1; @@ -2817,7 +2914,7 @@ brcmf_cfg80211_set_power_mgmt(struct wiphy *wiphy, struct net_device *ndev, goto done; } - pm = enabled ? PM_FAST : PM_OFF; + pm = enabled ? ifp->drvr->settings->default_pm : PM_OFF; /* Do not enable the power save after assoc if it is a p2p interface */ if (ifp->vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT) { brcmf_dbg(INFO, "Do not enable power save for P2P clients\n"); @@ -2855,7 +2952,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) { brcmf_err("Bss info is larger than buffer. Discarding\n"); - return 0; + return -EINVAL; } if (!bi->ctl_ch) { @@ -3581,6 +3678,8 @@ brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, } netinfo = brcmf_get_netinfo_array(pfn_result); + if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) + netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; cfg->wowl.nd->n_channels = 1; @@ -3681,10 +3780,24 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + struct brcmf_cfg80211_info *config = drvr->config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMING; + if (cfg->wowl.active) { + /* wait for bus resumed */ + while (retry && bus_if->state != BRCMF_BUS_UP) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && bus_if->state != BRCMF_BUS_UP) + brcmf_err("timed out wait for bus resume\n"); + brcmf_report_wowl_wakeind(wiphy, ifp); brcmf_fil_iovar_int_set(ifp, "wowl_clear", 0); brcmf_config_wowl_pattern(ifp, "clr", NULL, 0, NULL, 0); @@ -3700,7 +3813,12 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) brcmf_notify_sched_scan_results); cfg->wowl.nd_enabled = false; } + + /* disable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, false); + } + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; return 0; } @@ -3758,6 +3876,9 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg, brcmf_fil_iovar_int_set(ifp, "wowl_activate", 1); brcmf_bus_wowl_config(cfg->pub->bus_if, true); cfg->wowl.active = true; + + /* enable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, true); } static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, @@ -3767,9 +3888,12 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_vif *vif; + struct brcmf_cfg80211_info *config = ifp->drvr->config; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDING; + /* if the primary net_device is not READY there is nothing * we can do but pray resume goes smoothly. */ @@ -3784,7 +3908,8 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status)) brcmf_abort_scanning(cfg); - if (wowl == NULL) { + if (!wowl || !test_bit(BRCMF_VIF_STATUS_CONNECTED, + &ifp->vif->sme_state)) { brcmf_bus_wowl_config(cfg->pub->bus_if, false); list_for_each_entry(vif, &cfg->vif_list, list) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) @@ -3793,7 +3918,7 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, * disassociate from AP to save power while system is * in suspended state */ - brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event * generated due to DISASSOC call to the fw to keep * the state fw and WPA_Supplicant state consistent @@ -3804,14 +3929,19 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, brcmf_set_mpc(ifp, 1); } else { - /* Configure WOWL paramaters */ - brcmf_configure_wowl(cfg, ifp, wowl); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) + /* Configure WOWL parameters */ + brcmf_configure_wowl(cfg, ifp, wowl); } exit: - brcmf_dbg(TRACE, "Exit\n"); + /* set cfg80211 pm state to cfg80211 suspended state */ + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDED; + /* clear any scanning activity */ cfg->scan_status = 0; + + brcmf_dbg(TRACE, "Exit\n"); return 0; } @@ -4118,6 +4248,10 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "RSN_AKM_MFP_1X\n"); wpa_auth |= WPA2_AUTH_1X_SHA256; break; + case RSN_AKM_SAE: + brcmf_dbg(TRACE, "RSN_AKM_SAE\n"); + wpa_auth |= WPA3_AUTH_SAE_PSK; + break; default: brcmf_err("Invalid key mgmt info\n"); } @@ -4135,11 +4269,12 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "MFP Required\n"); mfp = BRCMF_MFP_REQUIRED; /* Firmware only supports mfp required in - * combination with WPA2_AUTH_PSK_SHA256 or - * WPA2_AUTH_1X_SHA256. + * combination with WPA2_AUTH_PSK_SHA256, + * WPA2_AUTH_1X_SHA256, or WPA3_AUTH_SAE_PSK. */ if (!(wpa_auth & (WPA2_AUTH_PSK_SHA256 | - WPA2_AUTH_1X_SHA256))) { + WPA2_AUTH_1X_SHA256 | + WPA3_AUTH_SAE_PSK))) { err = -EINVAL; goto exit; } @@ -4344,6 +4479,11 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, mgmt_ie_len = &saved_ie->assoc_req_ie_len; mgmt_ie_buf_len = sizeof(saved_ie->assoc_req_ie); break; + case BRCMF_VNDR_IE_ASSOCRSP_FLAG: + mgmt_ie_buf = saved_ie->assoc_res_ie; + mgmt_ie_len = &saved_ie->assoc_res_ie_len; + mgmt_ie_buf_len = sizeof(saved_ie->assoc_res_ie); + break; default: err = -EPERM; brcmf_err("not suitable type\n"); @@ -4493,6 +4633,57 @@ brcmf_config_ap_mgmt_ie(struct brcmf_cfg80211_vif *vif, else brcmf_dbg(TRACE, "Applied Vndr IEs for Probe Resp\n"); + /* Set Assoc Response IEs to FW */ + err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_ASSOCRSP_FLAG, + beacon->assocresp_ies, + beacon->assocresp_ies_len); + if (err) + brcmf_err("Set Assoc Resp IE Failed\n"); + else + brcmf_dbg(TRACE, "Applied Vndr IEs for Assoc Resp\n"); + + return err; +} + +static s32 +brcmf_parse_configure_security(struct brcmf_if *ifp, + struct cfg80211_ap_settings *settings, + enum nl80211_iftype dev_role) +{ + const struct brcmf_tlv *rsn_ie; + const struct brcmf_vs_tlv *wpa_ie; + s32 err = 0; + + /* find the RSN_IE */ + rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, + settings->beacon.tail_len, WLAN_EID_RSN); + + /* find the WPA_IE */ + wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, + settings->beacon.tail_len); + + if (wpa_ie || rsn_ie) { + brcmf_dbg(TRACE, "WPA(2) IE is found\n"); + if (wpa_ie) { + /* WPA IE */ + err = brcmf_configure_wpaie(ifp, wpa_ie, false); + if (err < 0) + return err; + } else { + struct brcmf_vs_tlv *tmp_ie; + + tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; + + /* RSN IE */ + err = brcmf_configure_wpaie(ifp, tmp_ie, true); + if (err < 0) + return err; + } + } else { + brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); + brcmf_configure_opensecurity(ifp); + } + return err; } @@ -4503,12 +4694,12 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, s32 ie_offset; struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; + struct cfg80211_crypto_settings *crypto = &settings->crypto; const struct brcmf_tlv *ssid_ie; const struct brcmf_tlv *country_ie; struct brcmf_ssid_le ssid_le; s32 err = -EPERM; - const struct brcmf_tlv *rsn_ie; - const struct brcmf_vs_tlv *wpa_ie; struct brcmf_join_params join_params; enum nl80211_iftype dev_role; struct brcmf_fil_bss_enable_le bss_enable; @@ -4526,6 +4717,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + brcmf_dbg(TRACE, "mbss %s\n", mbss ? "enabled" : "disabled"); /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -4562,36 +4754,6 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, brcmf_configure_arp_nd_offload(ifp, false); } - /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, - settings->beacon.tail_len, WLAN_EID_RSN); - - /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, - settings->beacon.tail_len); - - if ((wpa_ie != NULL || rsn_ie != NULL)) { - brcmf_dbg(TRACE, "WPA(2) IE is found\n"); - if (wpa_ie != NULL) { - /* WPA IE */ - err = brcmf_configure_wpaie(ifp, wpa_ie, false); - if (err < 0) - goto exit; - } else { - struct brcmf_vs_tlv *tmp_ie; - - tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; - - /* RSN IE */ - err = brcmf_configure_wpaie(ifp, tmp_ie, true); - if (err < 0) - goto exit; - } - } else { - brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); - brcmf_configure_opensecurity(ifp); - } - /* Parameters shared by all radio interfaces */ if (!mbss) { if ((supports_11d) && (is_11d != ifp->vif->is_11d)) { @@ -4620,9 +4782,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, } } - if ((dev_role == NL80211_IFTYPE_AP) && - ((ifp->ifidx == 0) || - !brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + if ((dev_role == NL80211_IFTYPE_AP) && (ifp->ifidx == 0)) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); if (err < 0) { brcmf_err("BRCMF_C_DOWN error %d\n", err); @@ -4668,6 +4828,33 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, brcmf_err("BRCMF_C_UP error (%d)\n", err); goto exit; } + + if (crypto->psk) { + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_PSK); + err = brcmf_set_pmk(ifp, crypto->psk, + BRCMF_WSEC_MAX_PSK_LEN); + if (err < 0) + goto exit; + } + if (crypto->sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_SAE); + err = brcmf_set_sae_password(ifp, crypto->sae_pwd, + crypto->sae_pwd_len); + if (err < 0) + goto exit; + } + if (profile->use_fwauth == 0) + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_AP); + if (err < 0) { + brcmf_err("brcmf_parse_configure_security error\n"); + goto exit; + } + /* On DOWN the firmware removes the WEP keys, reconfigure * them if they were set. */ @@ -4684,12 +4871,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, goto exit; } - if (settings->hidden_ssid) { - err = brcmf_fil_iovar_int_set(ifp, "closednet", 1); - if (err) { - brcmf_err("closednet error (%d)\n", err); - goto exit; - } + err = brcmf_fil_iovar_int_set(ifp, "closednet", + settings->hidden_ssid); + if (err) { + brcmf_err("%s closednet error (%d)\n", + settings->hidden_ssid ? + "enabled" : "disabled", + err); + goto exit; } brcmf_dbg(TRACE, "AP mode configuration complete\n"); @@ -4700,6 +4889,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, chanspec, err); goto exit; } + + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_P2P_GO); + if (err < 0) { + brcmf_err("brcmf_parse_configure_security error\n"); + goto exit; + } + err = brcmf_fil_bsscfg_data_set(ifp, "ssid", &ssid_le, sizeof(ssid_le)); if (err < 0) { @@ -4728,6 +4925,9 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if ((err) && (!mbss)) { brcmf_set_mpc(ifp, 1); brcmf_configure_arp_nd_offload(ifp, true); + } else { + cfg->num_softap++; + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); } return err; } @@ -4735,9 +4935,12 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) { struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; s32 err; struct brcmf_fil_bss_enable_le bss_enable; struct brcmf_join_params join_params; + s32 apsta = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -4746,6 +4949,35 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) /* first to make sure they get processed by fw. */ msleep(400); + cfg->num_softap--; + + if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) { + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK)) + brcmf_set_pmk(ifp, NULL, 0); + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_SAE)) + brcmf_set_sae_password(ifp, NULL, 0); + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + } + + /* Clear bss configuration and SSID */ + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(0); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) + brcmf_err("bss_enable config failed %d\n", err); + + memset(&join_params, 0, sizeof(join_params)); + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, + &join_params, sizeof(join_params)); + if (err < 0) + brcmf_err("SET SSID error (%d)\n", err); + + if (cfg->num_softap) { + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); + return 0; + } + if (ifp->vif->mbss) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); return err; @@ -4755,17 +4987,18 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) if (ifp->bsscfgidx == 0) brcmf_fil_iovar_int_set(ifp, "closednet", 0); - memset(&join_params, 0, sizeof(join_params)); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, sizeof(join_params)); + err = brcmf_fil_iovar_int_get(ifp, "apsta", &apsta); if (err < 0) - brcmf_err("SET SSID error (%d)\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); - if (err < 0) - brcmf_err("BRCMF_C_DOWN error %d\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); - if (err < 0) - brcmf_err("setting AP mode failed %d\n", err); + brcmf_err("wl apsta failed (%d)\n", err); + + if (!apsta) { + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); + if (err < 0) + brcmf_err("BRCMF_C_DOWN error %d\n", err); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); + if (err < 0) + brcmf_err("setting AP mode failed %d\n", err); + } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) brcmf_fil_iovar_int_set(ifp, "mbss", 0); brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY, @@ -5229,6 +5462,7 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; + int ret; brcmf_dbg(TRACE, "enter\n"); @@ -5237,6 +5471,13 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) return -EINVAL; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + brcmf_err("okc_info_pmk iovar failed: ret=%d\n", ret); + } + return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -5253,6 +5494,26 @@ static int brcmf_cfg80211_del_pmk(struct wiphy *wiphy, struct net_device *dev, return brcmf_set_pmk(ifp, NULL, 0); } +static int +brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev, + struct bss_parameters *params) +{ + struct brcmf_if *ifp; + int ret = 0; + u32 ap_isolate; + + brcmf_dbg(TRACE, "Enter\n"); + ifp = netdev_priv(dev); + if (params->ap_isolate >= 0) { + ap_isolate = (u32)params->ap_isolate; + ret = brcmf_fil_iovar_int_set(ifp, "ap_isolate", ap_isolate); + if (ret < 0) + brcmf_err("ap_isolate iovar failed: ret=%d\n", ret); + } + + return ret; +} + static struct cfg80211_ops brcmf_cfg80211_ops = { .add_virtual_intf = brcmf_cfg80211_add_iface, .del_virtual_intf = brcmf_cfg80211_del_iface, @@ -5298,6 +5559,7 @@ static struct cfg80211_ops brcmf_cfg80211_ops = { .update_connect_params = brcmf_cfg80211_update_conn_params, .set_pmk = brcmf_cfg80211_set_pmk, .del_pmk = brcmf_cfg80211_del_pmk, + .change_bss = brcmf_cfg80211_change_bss, }; struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, @@ -5306,6 +5568,7 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif_walk; struct brcmf_cfg80211_vif *vif; bool mbss; + struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); brcmf_dbg(TRACE, "allocating virtual interface (size=%zu)\n", sizeof(*vif)); @@ -5318,7 +5581,8 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, brcmf_init_prof(&vif->profile); - if (type == NL80211_IFTYPE_AP) { + if (type == NL80211_IFTYPE_AP && + brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) { mbss = false; list_for_each_entry(vif_walk, &cfg->vif_list, list) { if (vif_walk->wdev.iftype == NL80211_IFTYPE_AP) { @@ -5347,8 +5611,10 @@ void brcmf_cfg80211_free_netdev(struct net_device *ndev) ifp = netdev_priv(ndev); vif = ifp->vif; - if (vif) + if (vif) { brcmf_free_vif(vif); + ifp->vif = NULL; + } } static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, @@ -5357,25 +5623,28 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, u32 event = e->event_code; u32 status = e->status; - if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_PSK && - event == BRCMF_E_PSK_SUP && - status == BRCMF_E_STATUS_FWSUP_COMPLETED) + if (event == BRCMF_E_PSK_SUP && + status == BRCMF_E_STATUS_FWSUP_COMPLETED) { set_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state); + if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_1X) + return true; + } + if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); - if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK) + if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK && + vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE) return true; set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); } if (test_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state) && - test_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state)) { - clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state); - clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); + test_and_clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, + &vif->sme_state)) return true; - } + return false; } @@ -5432,11 +5701,151 @@ static void brcmf_clear_assoc_ies(struct brcmf_cfg80211_info *cfg) conn_info->resp_ie_len = 0; } +u8 brcmf_map_prio_to_prec(void *config, u8 prio) +{ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (!cfg) + return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? + (prio ^ 2) : prio; + + /* For those AC(s) with ACM flag set to 1, convert its 4-level priority + * to an 8-level precedence which is the same as BE's */ + if (prio > PRIO_8021D_EE && + cfg->ac_priority[prio] == cfg->ac_priority[PRIO_8021D_BE]) + return cfg->ac_priority[prio] * 2; + + /* Conversion of 4-level priority to 8-level precedence */ + if (prio == PRIO_8021D_BE || prio == PRIO_8021D_BK || + prio == PRIO_8021D_CL || prio == PRIO_8021D_VO) + return cfg->ac_priority[prio] * 2; + else + return cfg->ac_priority[prio] * 2 + 1; +} + +u8 brcmf_map_prio_to_aci(void *config, u8 prio) +{ + /* Prio here refers to the 802.1d priority in range of 0 to 7. + * ACI here refers to the WLAN AC Index in range of 0 to 3. + * This function will return ACI corresponding to input prio. + */ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (cfg) + return cfg->ac_priority[prio]; + + return prio; +} + +static void brcmf_init_wmm_prio(u8 *priority) +{ + /* Initialize AC priority array to default + * 802.1d priority as per following table: + * 802.1d prio 0,3 maps to BE + * 802.1d prio 1,2 maps to BK + * 802.1d prio 4,5 maps to VI + * 802.1d prio 6,7 maps to VO + */ + priority[0] = BRCMF_FWS_FIFO_AC_BE; + priority[3] = BRCMF_FWS_FIFO_AC_BE; + priority[1] = BRCMF_FWS_FIFO_AC_BK; + priority[2] = BRCMF_FWS_FIFO_AC_BK; + priority[4] = BRCMF_FWS_FIFO_AC_VI; + priority[5] = BRCMF_FWS_FIFO_AC_VI; + priority[6] = BRCMF_FWS_FIFO_AC_VO; + priority[7] = BRCMF_FWS_FIFO_AC_VO; +} + +static void brcmf_wifi_prioritize_acparams(const + struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority) +{ + u8 aci; + u8 aifsn; + u8 ecwmin; + u8 ecwmax; + u8 acm; + u8 ranking_basis[EDCF_AC_COUNT]; + u8 aci_prio[EDCF_AC_COUNT]; /* AC_BE, AC_BK, AC_VI, AC_VO */ + u8 index; + + for (aci = 0; aci < EDCF_AC_COUNT; aci++, acp++) { + aifsn = acp->ACI & EDCF_AIFSN_MASK; + acm = (acp->ACI & EDCF_ACM_MASK) ? 1 : 0; + ecwmin = acp->ECW & EDCF_ECWMIN_MASK; + ecwmax = (acp->ECW & EDCF_ECWMAX_MASK) >> EDCF_ECWMAX_SHIFT; + brcmf_dbg(CONN, "ACI %d aifsn %d acm %d ecwmin %d ecwmax %d\n", + aci, aifsn, acm, ecwmin, ecwmax); + /* Default AC_VO will be the lowest ranking value */ + ranking_basis[aci] = aifsn + ecwmin + ecwmax; + /* Initialise priority starting at 0 (AC_BE) */ + aci_prio[aci] = 0; + + /* If ACM is set, STA can't use this AC as per 802.11. + * Change the ranking to BE + */ + if (aci != AC_BE && aci != AC_BK && acm == 1) + ranking_basis[aci] = ranking_basis[AC_BE]; + } + + /* Ranking method which works for AC priority + * swapping when values for cwmin, cwmax and aifsn are varied + * Compare each aci_prio against each other aci_prio + */ + for (aci = 0; aci < EDCF_AC_COUNT; aci++) { + for (index = 0; index < EDCF_AC_COUNT; index++) { + if (index != aci) { + /* Smaller ranking value has higher priority, + * so increment priority for each ACI which has + * a higher ranking value + */ + if (ranking_basis[aci] < ranking_basis[index]) + aci_prio[aci]++; + } + } + } + + /* By now, aci_prio[] will be in range of 0 to 3. + * Use ACI prio to get the new priority value for + * each 802.1d traffic type, in this range. + */ + if (!(aci_prio[AC_BE] == aci_prio[AC_BK] && + aci_prio[AC_BK] == aci_prio[AC_VI] && + aci_prio[AC_VI] == aci_prio[AC_VO])) { + + /* 802.1d 0,3 maps to BE */ + priority[0] = aci_prio[AC_BE]; + priority[3] = aci_prio[AC_BE]; + + /* 802.1d 1,2 maps to BK */ + priority[1] = aci_prio[AC_BK]; + priority[2] = aci_prio[AC_BK]; + + /* 802.1d 4,5 maps to VO */ + priority[4] = aci_prio[AC_VI]; + priority[5] = aci_prio[AC_VI]; + + /* 802.1d 6,7 maps to VO */ + priority[6] = aci_prio[AC_VO]; + priority[7] = aci_prio[AC_VO]; + + } else { + /* Initialize to default priority */ + brcmf_init_wmm_prio(priority); + } + + brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n", + priority[0], priority[1], priority[2], priority[3]); + + brcmf_dbg(CONN, "Adj prio VI 4->%d, VI 5->%d, VO 6->%d, VO 7->%d\n", + priority[4], priority[5], priority[6], priority[7]); +} + static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) { struct brcmf_cfg80211_assoc_ielen_le *assoc_info; struct brcmf_cfg80211_connect_info *conn_info = cfg_to_conn(cfg); + struct brcmf_cfg80211_edcf_acparam edcf_acparam_info[EDCF_AC_COUNT]; u32 req_len; u32 resp_len; s32 err = 0; @@ -5481,6 +5890,17 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, conn_info->resp_ie = kmemdup(cfg->extra_buf, conn_info->resp_ie_len, GFP_KERNEL); + + err = brcmf_fil_iovar_data_get(ifp, "wme_ac_sta", + edcf_acparam_info, + sizeof(edcf_acparam_info)); + if (err) { + brcmf_err("could not get wme_ac_sta (%d)\n", err); + return err; + } + + brcmf_wifi_prioritize_acparams(edcf_acparam_info, + cfg->ac_priority); } else { conn_info->resp_ie_len = 0; conn_info->resp_ie = NULL; @@ -5491,6 +5911,47 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool +brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -5551,6 +6012,11 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, roam_info.resp_ie = conn_info->resp_ie; roam_info.resp_ie_len = conn_info->resp_ie_len; + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft || profile->is_okc)) + roam_info.authorized = true; + cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); @@ -5588,10 +6054,22 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, conn_params.req_ie_len = conn_info->req_ie_len; conn_params.resp_ie = conn_info->resp_ie; conn_params.resp_ie_len = conn_info->resp_ie_len; + + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + brcmf_has_pmkid(conn_params.req_ie, conn_params.req_ie_len)) + conn_params.authorized = true; + cfg80211_connect_done(ndev, &conn_params, GFP_KERNEL); brcmf_dbg(CONN, "Report connect result - connection %s\n", completed ? "succeeded" : "failed"); } + + if (test_and_clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, + &ifp->vif->sme_state) && + profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { + cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); + brcmf_dbg(CONN, "Report port authorized\n"); + } brcmf_dbg(TRACE, "Exit\n"); return 0; } @@ -5673,7 +6151,11 @@ brcmf_notify_connect_status(struct brcmf_if *ifp, if (!brcmf_is_ibssmode(ifp->vif)) { brcmf_bss_connect_done(cfg, ndev, e, false); brcmf_link_down(ifp->vif, - brcmf_map_fw_linkdown_reason(e)); + brcmf_map_fw_linkdown_reason(e), + e->event_code & + (BRCMF_E_DEAUTH_IND | + BRCMF_E_DISASSOC_IND) + ? false : true); brcmf_init_prof(ndev_to_prof(ndev)); if (ndev != cfg_to_ndev(cfg)) complete(&cfg->vif_disabled); @@ -5736,6 +6218,9 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, struct brcmf_if_event *ifevent = (struct brcmf_if_event *)data; struct brcmf_cfg80211_vif_event *event = &cfg->vif_event; struct brcmf_cfg80211_vif *vif; + enum nl80211_iftype iftype = NL80211_IFTYPE_UNSPECIFIED; + bool vif_pend = false; + int err; brcmf_dbg(TRACE, "Enter: action %u flags %u ifidx %u bsscfgidx %u\n", ifevent->action, ifevent->flags, ifevent->ifidx, @@ -5748,9 +6233,28 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, switch (ifevent->action) { case BRCMF_E_IF_ADD: /* waiting process may have timed out */ - if (!cfg->vif_event.vif) { + if (!vif) { + /* handle IF_ADD event from firmware */ spin_unlock(&event->vif_event_lock); - return -EBADF; + vif_pend = true; + if (ifevent->role == WLC_E_IF_ROLE_STA) + iftype = NL80211_IFTYPE_STATION; + else if (ifevent->role == WLC_E_IF_ROLE_AP) + iftype = NL80211_IFTYPE_AP; + else + vif_pend = false; + + if (vif_pend) { + vif = brcmf_alloc_vif(cfg, iftype); + if (IS_ERR(vif)) { + brcmf_err("Role:%d failed to alloc vif\n", + ifevent->role); + return PTR_ERR(vif); + } + } else { + brcmf_err("Invalid Role:%d\n", ifevent->role); + return -EBADF; + } } ifp->vif = vif; @@ -5760,6 +6264,18 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, ifp->ndev->ieee80211_ptr = &vif->wdev; SET_NETDEV_DEV(ifp->ndev, wiphy_dev(cfg->wiphy)); } + + if (vif_pend) { + err = brcmf_net_attach(ifp, false); + if (err) { + brcmf_err("netdevice register failed with err:%d\n", + err); + brcmf_free_vif(vif); + free_netdev(ifp->ndev); + } + return err; + } + spin_unlock(&event->vif_event_lock); wake_up(&event->vif_wq); return 0; @@ -5886,6 +6402,7 @@ static s32 wl_init_priv(struct brcmf_cfg80211_info *cfg) mutex_init(&cfg->usr_sync); brcmf_init_escan(cfg); brcmf_init_conf(cfg->conf); + brcmf_init_wmm_prio(cfg->ac_priority); init_completion(&cfg->vif_disabled); return err; } @@ -6098,8 +6615,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, * for subsequent chanspecs. */ channel->flags = IEEE80211_CHAN_NO_HT40 | - IEEE80211_CHAN_NO_80MHZ | - IEEE80211_CHAN_NO_160MHZ; + IEEE80211_CHAN_NO_80MHZ; ch.bw = BRCMU_CHAN_BW_20; cfg->d11inf.encchspec(&ch); chaninfo = ch.chspec; @@ -6402,6 +6918,16 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { .tx = 0xffff, .rx = BIT(IEEE80211_STYPE_ACTION >> 4) | BIT(IEEE80211_STYPE_PROBE_REQ >> 4) + }, + [NL80211_IFTYPE_AP] = { + .tx = 0xffff, + .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) | + BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) | + BIT(IEEE80211_STYPE_PROBE_REQ >> 4) | + BIT(IEEE80211_STYPE_DISASSOC >> 4) | + BIT(IEEE80211_STYPE_AUTH >> 4) | + BIT(IEEE80211_STYPE_DEAUTH >> 4) | + BIT(IEEE80211_STYPE_ACTION >> 4) } }; @@ -6423,6 +6949,9 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #AP <= 1, channels = 1, 2 total * #AP <= 4, matching BI, channels = 1, 4 total * + * no p2p and rsdb: + * #STA <= 1, #AP <= 2, channels = 2, 3 total + * * p2p, no mchan, and mbss: * * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 1, 3 total @@ -6434,6 +6963,16 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 2, 3 total * #STA <= 1, #P2P-DEV <= 1, #AP <= 1, #P2P-CL <= 1, channels = 1, 4 total * #AP <= 4, matching BI, channels = 1, 4 total + * + * p2p, rsdb, and no mbss: + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total + * + * p2p, rsdb, mbss + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total + * #AP <= 4, matching BI, channels = 1, 4 total + * */ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) { @@ -6441,13 +6980,14 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_iface_limit *c0_limits = NULL; struct ieee80211_iface_limit *p2p_limits = NULL; struct ieee80211_iface_limit *mbss_limits = NULL; - bool mbss, p2p; + bool mbss, p2p, rsdb; int i, c, n_combos; mbss = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS); p2p = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_P2P); + rsdb = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB); - n_combos = 1 + !!p2p + !!mbss; + n_combos = 1 + !!(p2p && !rsdb) + !!mbss; combo = kcalloc(n_combos, sizeof(*combo), GFP_KERNEL); if (!combo) goto err; @@ -6458,16 +6998,36 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c = 0; i = 0; - c0_limits = kcalloc(p2p ? 3 : 2, sizeof(*c0_limits), GFP_KERNEL); + if (p2p && rsdb) + c0_limits = kcalloc(4, sizeof(*c0_limits), GFP_KERNEL); + else if (p2p) + c0_limits = kcalloc(3, sizeof(*c0_limits), GFP_KERNEL); + else + c0_limits = kcalloc(2, sizeof(*c0_limits), GFP_KERNEL); if (!c0_limits) goto err; - c0_limits[i].max = 1; - c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); - if (p2p) { + if (p2p && rsdb) { + combo[c].num_different_channels = 2; + wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO) | + BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 4; + } else if (p2p) { if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN)) combo[c].num_different_channels = 2; else combo[c].num_different_channels = 1; + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); @@ -6476,16 +7036,26 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO); + combo[c].max_interfaces = i; + } else if (rsdb) { + combo[c].num_different_channels = 2; + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 3; } else { combo[c].num_different_channels = 1; c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = i; } - combo[c].max_interfaces = i; combo[c].n_limits = i; combo[c].limits = c0_limits; - if (p2p) { + if (p2p && !rsdb) { c++; i = 0; p2p_limits = kcalloc(4, sizeof(*p2p_limits), GFP_KERNEL); @@ -6547,6 +7117,7 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) #ifdef CONFIG_PM struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct wiphy_wowlan_support *wowl; + struct cfg80211_wowlan *brcmf_wowlan_config = NULL; wowl = kmemdup(&brcmf_wowlan_support, sizeof(brcmf_wowlan_support), GFP_KERNEL); @@ -6569,12 +7140,34 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) } wiphy->wowlan = wowl; + + /* wowlan_config structure report for kernels */ + brcmf_wowlan_config = kzalloc(sizeof(*brcmf_wowlan_config), + GFP_KERNEL); + if (brcmf_wowlan_config) { + brcmf_wowlan_config->any = false; + brcmf_wowlan_config->disconnect = true; + brcmf_wowlan_config->eap_identity_req = true; + brcmf_wowlan_config->four_way_handshake = true; + brcmf_wowlan_config->rfkill_release = false; + brcmf_wowlan_config->patterns = NULL; + brcmf_wowlan_config->n_patterns = 0; + brcmf_wowlan_config->tcp = NULL; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK)) + brcmf_wowlan_config->gtk_rekey_failure = true; + else + brcmf_wowlan_config->gtk_rekey_failure = false; + } else { + brcmf_err("Can not allocate memory for brcm_wowlan_config\n"); + } + wiphy->wowlan_config = brcmf_wowlan_config; #endif } static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); const struct ieee80211_iface_combination *combo; struct ieee80211_supported_band *band; u16 max_interfaces = 0; @@ -6620,6 +7213,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->flags |= WIPHY_FLAG_NETNS_OK | WIPHY_FLAG_PS_ON_BY_DEFAULT | + WIPHY_FLAG_HAVE_AP_SME | WIPHY_FLAG_OFFCHAN_TX | WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_TDLS)) @@ -6631,6 +7225,16 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); + } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) { + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; @@ -6641,9 +7245,12 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) /* vendor commands/events support */ wiphy->vendor_commands = brcmf_vendor_cmds; wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; + wiphy->vendor_events = brcmf_vendor_events; + wiphy->n_vendor_events = BRCMF_VNDR_EVTS_LAST; + brcmf_fweh_register(cfg->pub, BRCMF_E_PHY_TEMP, + brcmf_wiphy_phy_temp_evt_handler); - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) - brcmf_wiphy_wowl_params(wiphy, ifp); + brcmf_wiphy_wowl_params(wiphy, ifp); err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist, sizeof(bandlist)); if (err) { @@ -6700,6 +7307,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) struct wireless_dev *wdev; struct brcmf_if *ifp; s32 power_mode; + s32 eap_restrict; s32 err = 0; if (cfg->dongle_up) @@ -6714,7 +7322,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) brcmf_dongle_scantime(ifp); - power_mode = cfg->pwr_save ? PM_FAST : PM_OFF; + power_mode = cfg->pwr_save ? ifp->drvr->settings->default_pm : PM_OFF; err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PM, power_mode); if (err) goto default_conf_out; @@ -6724,6 +7332,14 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) err = brcmf_dongle_roam(ifp); if (err) goto default_conf_out; + + eap_restrict = ifp->drvr->settings->eap_restrict; + if (eap_restrict) { + err = brcmf_fil_iovar_int_set(ifp, "eap_restrict", + eap_restrict); + if (err) + brcmf_info("eap_restrict error (%d)\n", err); + } err = brcmf_cfg80211_change_iface(wdev->wiphy, ndev, wdev->iftype, NULL); if (err) @@ -6731,6 +7347,13 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) brcmf_configure_arp_nd_offload(ifp, true); + if (ifp->drvr->settings->frameburst) { + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_FAKEFRAG, 1); + if (err) + brcmf_info("setting frameburst mode failed\n"); + brcmf_dbg(INFO, "frameburst mode enabled\n"); + } + cfg->dongle_up = true; default_conf_out: @@ -6754,7 +7377,7 @@ static s32 __brcmf_cfg80211_down(struct brcmf_if *ifp) * from AP to save power */ if (check_vif_up(ifp->vif)) { - brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event generated due to DISASSOC call to the fw to keep @@ -7012,6 +7635,8 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->ops = ops; cfg->pub = drvr; + cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; + cfg->num_softap = 0; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 7b2835e5e4344..20cad8303abe9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -34,6 +34,23 @@ #define WL_ROAM_TRIGGER_LEVEL -75 #define WL_ROAM_DELTA 20 +/* WME Access Category Indices (ACIs) */ +#define AC_BE 0 /* Best Effort */ +#define AC_BK 1 /* Background */ +#define AC_VI 2 /* Video */ +#define AC_VO 3 /* Voice */ +#define EDCF_AC_COUNT 4 +#define MAX_8021D_PRIO 8 + +#define EDCF_ACI_MASK 0x60 +#define EDCF_ACI_SHIFT 5 +#define EDCF_ACM_MASK 0x10 +#define EDCF_ECWMIN_MASK 0x0f +#define EDCF_ECWMAX_SHIFT 4 +#define EDCF_AIFSN_MASK 0x0f +#define EDCF_AIFSN_MAX 15 +#define EDCF_ECWMAX_MASK 0xf0 + /* Keep BRCMF_ESCAN_BUF_SIZE below 64K (65536). Allocing over 64K can be * problematic on some systems and should be avoided. */ @@ -86,6 +103,15 @@ #define BRCMF_VIF_EVENT_TIMEOUT msecs_to_jiffies(1500) +#define BRCMF_PM_WAIT_MAXRETRY 100 + +/* cfg80211 wowlan definitions */ +#define WL_WOWLAN_MAX_PATTERNS 8 +#define WL_WOWLAN_MIN_PATTERN_LEN 1 +#define WL_WOWLAN_MAX_PATTERN_LEN 255 +#define WL_WOWLAN_PKT_FILTER_ID_FIRST 201 +#define WL_WOWLAN_PKT_FILTER_ID_LAST (WL_WOWLAN_PKT_FILTER_ID_FIRST + \ + WL_WOWLAN_MAX_PATTERNS - 1) /** * enum brcmf_scan_status - scan engine status * @@ -118,7 +144,21 @@ struct brcmf_cfg80211_security { enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, - BRCMF_PROFILE_FWSUP_1X + BRCMF_PROFILE_FWSUP_1X, + BRCMF_PROFILE_FWSUP_SAE +}; + +/** + * enum brcmf_profile_fwauth - firmware authenticator profile + * + * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for SAE + */ +enum brcmf_profile_fwauth { + BRCMF_PROFILE_FWAUTH_NONE, + BRCMF_PROFILE_FWAUTH_PSK, + BRCMF_PROFILE_FWAUTH_SAE }; /** @@ -133,6 +173,9 @@ struct brcmf_cfg80211_profile { struct brcmf_cfg80211_security sec; struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; + u16 use_fwauth; + bool is_ft; + bool is_okc; }; /** @@ -156,25 +199,36 @@ enum brcmf_vif_status { BRCMF_VIF_STATUS_ASSOC_SUCCESS, }; +enum brcmf_cfg80211_pm_state { + BRCMF_CFG80211_PM_STATE_RESUMED, + BRCMF_CFG80211_PM_STATE_RESUMING, + BRCMF_CFG80211_PM_STATE_SUSPENDED, + BRCMF_CFG80211_PM_STATE_SUSPENDING, +}; + /** * struct vif_saved_ie - holds saved IEs for a virtual interface. * * @probe_req_ie: IE info for probe request. * @probe_res_ie: IE info for probe response. * @beacon_ie: IE info for beacon frame. + * @assoc_res_ie: IE info for association response frame. * @probe_req_ie_len: IE info length for probe request. * @probe_res_ie_len: IE info length for probe response. * @beacon_ie_len: IE info length for beacon frame. + * @assoc_res_ie_len: IE info length for association response frame. */ struct vif_saved_ie { u8 probe_req_ie[IE_MAX_LEN]; u8 probe_res_ie[IE_MAX_LEN]; u8 beacon_ie[IE_MAX_LEN]; u8 assoc_req_ie[IE_MAX_LEN]; + u8 assoc_res_ie[IE_MAX_LEN]; u32 probe_req_ie_len; u32 probe_res_ie_len; u32 beacon_ie_len; u32 assoc_req_ie_len; + u32 assoc_res_ie_len; }; /** @@ -214,6 +268,12 @@ struct brcmf_cfg80211_assoc_ielen_le { __le32 resp_len; }; +struct brcmf_cfg80211_edcf_acparam { + u8 ACI; + u8 ECW; + u16 TXOP; /* stored in network order (ls octet first) */ +}; + /* dongle escan state */ enum wl_escan_state { WL_ESCAN_STATE_IDLE, @@ -335,6 +395,9 @@ struct brcmf_cfg80211_info { struct brcmf_assoclist_le assoclist; struct brcmf_cfg80211_wowl wowl; struct brcmf_pno_info *pno; + u8 ac_priority[MAX_8021D_PRIO]; + u8 pm_state; + u8 num_softap; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index c5d1a1cbf6019..0f83946ab8fe5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -165,6 +165,7 @@ struct sbconfig { #define SRCI_LSS_MASK 0x00f00000 #define SRCI_LSS_SHIFT 20 #define SRCI_SRNB_MASK 0xf0 +#define SRCI_SRNB_MASK_EXT 0x100 #define SRCI_SRNB_SHIFT 4 #define SRCI_SRBSZ_MASK 0xf #define SRCI_SRBSZ_SHIFT 0 @@ -223,6 +224,24 @@ struct sbsocramregs { #define ARMCR4_BSZ_MASK 0x3f #define ARMCR4_BSZ_MULT 8192 +/* Minimum PMU resource mask for 43012C0 */ +#define CY_43012_PMU_MIN_RES_MASK 0xF8BFE77 + +/* PMU STATUS mask for 43012C0 */ +#define CY_43012_PMU_STATUS_MASK 0x1AC + +/* PMU CONTROL EXT mask for 43012C0 */ +#define CY_43012_PMU_CONTROL_EXT_MASK 0x11 + +/* PMU Watchdog Counter Tick value for 43012C0 */ +#define CY_43012_PMU_WATCHDOG_TICK_VAL 0x04 + +/* PMU Watchdog Counter Tick value for 4373 */ +#define CY_4373_PMU_WATCHDOG_TICK_VAL 0x04 + +/* Minimum PMU resource mask for 4373 */ +#define CY_4373_PMU_MIN_RES_MASK 0xFCAFF7F + struct brcmf_core_priv { struct brcmf_core pub; u32 wrapbase; @@ -443,11 +462,25 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; + struct brcmf_core *d11core2 = NULL; + struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; + /* special handle two D11 cores reset */ + if (core->pub.id == BCMA_CORE_80211) { + d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); + if (d11core2) { + brcmf_dbg(INFO, "found two d11 cores, reset both\n"); + d11priv2 = container_of(d11core2, struct brcmf_core_priv, + pub); + } + } + /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); + if (d11priv2) + brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -459,9 +492,30 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } + if (d11priv2) { + count = 0; + while (ci->ops->read32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL) & + BCMA_RESET_CTL_RESET) { + ci->ops->write32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL, + 0); + count++; + if (count > 50) + break; + usleep_range(40, 60); + } + } + ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); + + if (d11priv2) { + ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, + postreset | BCMA_IOCTL_CLK); + ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); + } } static char *brcmf_chip_name(uint chipid, char *buf, uint len) @@ -592,7 +646,13 @@ static void brcmf_chip_socram_ramsize(struct brcmf_core_priv *sr, u32 *ramsize, if (lss != 0) *ramsize += (1 << ((lss - 1) + SR_BSZ_BASE)); } else { - nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + /* length of SRAM Banks increased for corerev greater than 23 */ + if (sr->pub.rev >= 23) { + nb = (coreinfo & (SRCI_SRNB_MASK | SRCI_SRNB_MASK_EXT)) + >> SRCI_SRNB_SHIFT; + } else { + nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + } for (i = 0; i < nb; i++) { retent = brcmf_chip_socram_banksize(sr, i, &banksize); *ramsize += banksize; @@ -692,6 +752,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) return 0x200000; case CY_CC_4373_CHIP_ID: return 0x160000; + case CY_CC_89459_CHIP_ID: + return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000); default: brcmf_err("unknown chip: %s\n", ci->pub.name); break; @@ -778,7 +840,7 @@ static int brcmf_chip_dmp_get_regaddr(struct brcmf_chip_priv *ci, u32 *eromaddr, u32 *regbase, u32 *wrapbase) { u8 desc; - u32 val; + u32 val, szdesc; u8 mpnum = 0; u8 stype, sztype, wraptype; @@ -824,14 +886,15 @@ static int brcmf_chip_dmp_get_regaddr(struct brcmf_chip_priv *ci, u32 *eromaddr, /* next size descriptor can be skipped */ if (sztype == DMP_SLAVE_SIZE_DESC) { - val = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); + szdesc = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); /* skip upper size descriptor if present */ - if (val & DMP_DESC_ADDRSIZE_GT32) + if (szdesc & DMP_DESC_ADDRSIZE_GT32) brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); } - /* only look for 4K register regions */ - if (sztype != DMP_SLAVE_SIZE_4K) + /* look for 4K or 8K register regions */ + if (sztype != DMP_SLAVE_SIZE_4K && + sztype != DMP_SLAVE_SIZE_8K) continue; stype = (val & DMP_SLAVE_TYPE) >> DMP_SLAVE_TYPE_S; @@ -888,7 +951,8 @@ int brcmf_chip_dmp_erom_scan(struct brcmf_chip_priv *ci) /* need core with ports */ if (nmw + nsw == 0 && - id != BCMA_CORE_PMU) + id != BCMA_CORE_PMU && + id != BCMA_CORE_GCI) continue; /* try to obtain register address info */ @@ -1112,6 +1176,21 @@ void brcmf_chip_detach(struct brcmf_chip *pub) kfree(chip); } +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit) +{ + struct brcmf_chip_priv *chip; + struct brcmf_core_priv *core; + + chip = container_of(pub, struct brcmf_chip_priv, pub); + list_for_each_entry(core, &chip->cores, list) { + if (core->pub.id == BCMA_CORE_80211) { + if (unit-- == 0) + return &core->pub; + } + } + return NULL; +} + struct brcmf_core *brcmf_chip_get_core(struct brcmf_chip *pub, u16 coreid) { struct brcmf_chip_priv *chip; @@ -1154,6 +1233,14 @@ struct brcmf_core *brcmf_chip_get_pmu(struct brcmf_chip *pub) return cc; } +struct brcmf_core *brcmf_chip_get_gci(struct brcmf_chip *pub) +{ + struct brcmf_core *gci; + + gci = brcmf_chip_get_core(pub, BCMA_CORE_GCI); + return gci; +} + bool brcmf_chip_iscoreup(struct brcmf_core *pub) { struct brcmf_core_priv *core; @@ -1322,7 +1409,7 @@ bool brcmf_chip_set_active(struct brcmf_chip *pub, u32 rstvec) bool brcmf_chip_sr_capable(struct brcmf_chip *pub) { - u32 base, addr, reg, pmu_cc3_mask = ~0; + u32 base, addr, reg, sr_eng_en, pmu_cc3_mask = ~0; struct brcmf_chip_priv *chip; struct brcmf_core *pmu = brcmf_chip_get_pmu(pub); @@ -1338,6 +1425,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) switch (pub->chip) { case BRCM_CC_4354_CHIP_ID: case BRCM_CC_4356_CHIP_ID: + case BRCM_CC_4345_CHIP_ID: /* explicitly check SR engine enable bit */ pmu_cc3_mask = BIT(2); /* fall-through */ @@ -1354,6 +1442,24 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) addr = CORE_CC_REG(base, sr_control1); reg = chip->ops->read32(chip->ctx, addr); return reg != 0; + case CY_CC_4373_CHIP_ID: + /* explicitly check SR engine enable bit */ + sr_eng_en = BIT(0); + addr = CORE_CC_REG(base, sr_control0); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & sr_eng_en) != 0; + case BRCM_CC_4359_CHIP_ID: + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(pmu->base, retention_ctl); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & (PMU_RCTL_MACPHY_DISABLE_MASK | + PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; + case CY_CC_89459_CHIP_ID: + /* explicitly check SR engine enable bit */ + sr_eng_en = BIT(0); + addr = CORE_CC_REG(base, sr_control0); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & sr_eng_en) != 0; default: addr = CORE_CC_REG(pmu->base, pmucapabilities_ext); reg = chip->ops->read32(chip->ctx, addr); @@ -1366,3 +1472,151 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; } } + +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + /* SW scratch */ + addr = CORE_CC_REG(base, swscratch); + chip->ops->write32(chip->ctx, addr, 0); + + /* PMU status */ + addr = CORE_CC_REG(base, pmustatus); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_STATUS_MASK); + + /* PMU control ext */ + addr = CORE_CC_REG(base, pmucontrol_ext); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_CONTROL_EXT_MASK); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_gci(pub)->base; + + /* LHL Top Level Power Sequence Control */ + addr = CORE_GCI_REG(base, lhl_top_pwrseq_ctl_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Enable0 */ + addr = CORE_GCI_REG(base, gpio_int_en_port_adr[0]); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Status0 */ + addr = CORE_GCI_REG(base, gpio_int_st_port_adr[0]); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL ARM Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL MAC TimerInt0 */ + addr = CORE_GCI_REG(base, lhl_wl_mactim_int0_adr); + chip->ops->write32(chip->ctx, addr, 0x0); +} + +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + /* Watchdog res mask */ + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + /* PMU watchdog */ + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_WATCHDOG_TICK_VAL); + break; + case CY_CC_4373_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_WATCHDOG_TICK_VAL); + mdelay(100); + break; + default: + break; + } +} + diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index dd0ec3eba6a98..a7098d9301554 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -21,6 +21,9 @@ #define CORE_CC_REG(base, field) \ (base + offsetof(struct chipcregs, field)) +#define CORE_GCI_REG(base, field) \ + (base + offsetof(struct chipgciregs, field)) + /** * struct brcmf_chip - chip level information. * @@ -93,5 +96,10 @@ void brcmf_chip_resetcore(struct brcmf_core *core, u32 prereset, u32 reset, void brcmf_chip_set_passive(struct brcmf_chip *ci); bool brcmf_chip_set_active(struct brcmf_chip *ci, u32 rstvec); bool brcmf_chip_sr_capable(struct brcmf_chip *pub); +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub); +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub); +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub); +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub); +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit); #endif /* BRCMF_AXIDMP_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 7a2b49587b4d3..000a505d9eec2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include "core.h" @@ -28,6 +29,10 @@ #include "tracepoint.h" #include "common.h" #include "of.h" +#include "firmware.h" +#include "fweh.h" +#include +#include "defs.h" MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -73,6 +78,22 @@ static int brcmf_roamoff; module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR); MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine"); +static int brcmf_eap_restrict; +module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); +MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); + +static int brcmf_sdio_wq_highpri; +module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0); +MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority"); + +static int brcmf_frameburst; +module_param_named(frameburst, brcmf_frameburst, int, 0); +MODULE_PARM_DESC(frameburst, "Enable firmware frameburst feature"); + +static int brcmf_max_pm; +module_param_named(max_pm, brcmf_max_pm, int, 0); +MODULE_PARM_DESC(max_pm, "Use max power management mode by default"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -104,16 +125,144 @@ void brcmf_c_set_joinpref_default(struct brcmf_if *ifp) brcmf_err("Set join_pref error (%d)\n", err); } +static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, + struct brcmf_dload_data_le *dload_buf, + u32 len) +{ + s32 err; + + flag |= (DLOAD_HANDLER_VER << DLOAD_FLAG_VER_SHIFT); + dload_buf->flag = cpu_to_le16(flag); + dload_buf->dload_type = cpu_to_le16(DL_TYPE_CLM); + dload_buf->len = cpu_to_le32(len); + dload_buf->crc = cpu_to_le32(0); + len = sizeof(*dload_buf) + len - 1; + + err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, len); + + return err; +} + +static int brcmf_c_get_clm_name(struct brcmf_if *ifp, u8 *clm_name) +{ + struct brcmf_bus *bus = ifp->drvr->bus_if; + struct brcmf_rev_info *ri = &ifp->drvr->revinfo; + u8 fw_name[BRCMF_FW_NAME_LEN]; + u8 *ptr; + size_t len; + s32 err; + + memset(fw_name, 0, BRCMF_FW_NAME_LEN); + err = brcmf_bus_get_fwname(bus, ri->chipnum, ri->chiprev, fw_name); + if (err) { + brcmf_err("get firmware name failed (%d)\n", err); + goto done; + } + + /* generate CLM blob file name */ + ptr = strrchr(fw_name, '.'); + if (!ptr) { + err = -ENOENT; + goto done; + } + + len = ptr - fw_name + 1; + if (len + strlen(".clm_blob") > BRCMF_FW_NAME_LEN) { + err = -E2BIG; + } else { + strlcpy(clm_name, fw_name, len); + strlcat(clm_name, ".clm_blob", BRCMF_FW_NAME_LEN); + } +done: + return err; +} + +static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) +{ + struct device *dev = ifp->drvr->bus_if->dev; + struct brcmf_dload_data_le *chunk_buf; + const struct firmware *clm = NULL; + u8 clm_name[BRCMF_FW_NAME_LEN]; + u32 chunk_len; + u32 datalen; + u32 cumulative_len; + u16 dl_flag = DL_BEGIN; + u32 status; + s32 err; + + brcmf_dbg(TRACE, "Enter\n"); + + memset(clm_name, 0, BRCMF_FW_NAME_LEN); + err = brcmf_c_get_clm_name(ifp, clm_name); + if (err) { + brcmf_err("get CLM blob file name failed (%d)\n", err); + return err; + } + + err = request_firmware(&clm, clm_name, dev); + if (err) { + brcmf_info("no clm_blob available(err=%d), device may have limited channels available\n", + err); + return 0; + } + + chunk_buf = kzalloc(sizeof(*chunk_buf) + MAX_CHUNK_LEN - 1, GFP_KERNEL); + if (!chunk_buf) { + err = -ENOMEM; + goto done; + } + + datalen = clm->size; + cumulative_len = 0; + do { + if (datalen > MAX_CHUNK_LEN) { + chunk_len = MAX_CHUNK_LEN; + } else { + chunk_len = datalen; + dl_flag |= DL_END; + } + memcpy(chunk_buf->data, clm->data + cumulative_len, chunk_len); + + err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len); + + dl_flag &= ~DL_BEGIN; + + cumulative_len += chunk_len; + datalen -= chunk_len; + } while ((datalen > 0) && (err == 0)); + + if (err) { + brcmf_err("clmload (%zu byte file) failed (%d); ", + clm->size, err); + /* Retrieve clmload_status and print */ + err = brcmf_fil_iovar_int_get(ifp, "clmload_status", &status); + if (err) + brcmf_err("get clmload_status failed (%d)\n", err); + else + brcmf_dbg(INFO, "clmload_status=%d\n", status); + err = -EIO; + } + + kfree(chunk_buf); +done: + release_firmware(clm); + return err; +} + int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { s8 eventmask[BRCMF_EVENTING_MASK_LEN]; u8 buf[BRCMF_DCMD_SMLEN]; struct brcmf_rev_info_le revinfo; struct brcmf_rev_info *ri; + char *clmver; char *ptr; s32 err; + struct eventmsgs_ext *eventmask_msg = NULL; + u8 msglen; + struct brcmf_bus *bus = ifp->drvr->bus_if; - /* retreive mac address */ + /* retrieve mac addresses */ err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr, sizeof(ifp->mac_addr)); if (err < 0) { @@ -148,6 +297,13 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) } ri->result = err; + /* Do any CLM downloading */ + err = brcmf_c_process_clm_blob(ifp); + if (err < 0) { + brcmf_err("download CLM blob file failed, %d\n", err); + goto done; + } + /* query for 'ver' to get version info from firmware */ memset(buf, 0, sizeof(buf)); strcpy(buf, "ver"); @@ -167,6 +323,31 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) ptr = strrchr(buf, ' ') + 1; strlcpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver)); + /* Query for 'clmver' to get CLM version info from firmware */ + memset(buf, 0, sizeof(buf)); + err = brcmf_fil_iovar_data_get(ifp, "clmver", buf, sizeof(buf)); + if (err) { + brcmf_dbg(TRACE, "retrieving clmver failed, %d\n", err); + } else { + clmver = (char *)buf; + /* store CLM version for adding it to revinfo debugfs file */ + memcpy(ifp->drvr->clmver, clmver, sizeof(ifp->drvr->clmver)); + + /* Replace all newline/linefeed characters with space + * character + */ + ptr = clmver; + while ((ptr = strnchr(ptr, '\n', sizeof(buf))) != NULL) + *ptr = ' '; + + brcmf_dbg(INFO, "CLM version = %s\n", clmver); + } + + /* set apsta */ + err = brcmf_fil_iovar_int_set(ifp, "apsta", 1); + if (err) + brcmf_info("failed setting apsta, %d\n", err); + /* set mpc */ err = brcmf_fil_iovar_int_set(ifp, "mpc", 1); if (err) { @@ -191,6 +372,41 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) goto done; } + /* Enable event_msg_ext specific to 43012 chip */ + if (bus->chip == CY_CC_43012_CHIP_ID) { + /* Program event_msg_ext to support event larger than 128 */ + msglen = (roundup(BRCMF_E_LAST, NBBY) / NBBY) + + EVENTMSGS_EXT_STRUCT_SIZE; + /* Allocate buffer for eventmask_msg */ + eventmask_msg = kzalloc(msglen, GFP_KERNEL); + if (!eventmask_msg) { + err = -ENOMEM; + goto done; + } + + /* Read the current programmed event_msgs_ext */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->len = roundup(BRCMF_E_LAST, NBBY) / NBBY; + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmask_msg, + msglen); + + /* Enable ULP event */ + brcmf_dbg(EVENT, "enable event ULP\n"); + setbit(eventmask_msg->mask, BRCMF_E_ULP); + + /* Write updated Event mask */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->command = EVENTMSGS_SET_MASK; + eventmask_msg->len = (roundup(BRCMF_E_LAST, NBBY) / NBBY); + + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmask_msg, msglen); + if (err) { + brcmf_err("Set event_msgs_ext error (%d)\n", err); + goto done; + } + } /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); @@ -212,6 +428,12 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) /* Enable tx beamforming, errors can be ignored (not supported) */ (void)brcmf_fil_iovar_int_set(ifp, "txbf", 1); + /* add unicast packet filter */ + err = brcmf_pktfilter_add_remove(ifp->ndev, + BRCMF_UNICAST_FILTER_NUM, true); + if (err) + brcmf_info("Add unicast filter error (%d)\n", err); + /* do bus specific preinit here */ err = brcmf_bus_preinit(ifp->drvr->bus_if); done: @@ -282,11 +504,15 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, if (!settings) return NULL; - /* start by using the module paramaters */ + /* start by using the module parameters */ settings->p2p_enable = !!brcmf_p2p_enable; settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; + settings->eap_restrict = !!brcmf_eap_restrict; + settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; + settings->frameburst = !!brcmf_frameburst; + settings->default_pm = !!brcmf_max_pm ? PM_MAX : PM_FAST; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif @@ -368,7 +594,7 @@ static int __init brcmfmac_module_init(void) if (err == -ENODEV) brcmf_dbg(INFO, "No platform data available.\n"); - /* Initialize global module paramaters */ + /* Initialize global module parameters */ brcmf_mp_attach(); /* Continue the initialization by registering the different busses */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index a62f8e70b3207..c53d39493f7f5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -49,6 +49,10 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @feature_disable: Feature_disable bitmask. * @fcmode: FWS flow control. * @roamoff: Firmware roaming off? + * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds + * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. + * @frameburst: Firmware frame burst mode. + * @default_pm: default power management (PM) mode. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -58,6 +62,11 @@ struct brcmf_mp_device { unsigned int feature_disable; int fcmode; bool roamoff; + bool eap_restrict; + int sdio_dpc_prio; + bool sdio_wq_highpri; + bool frameburst; + int default_pm; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { @@ -75,4 +84,8 @@ void brcmf_release_module_param(struct brcmf_mp_device *module_param); /* Sets dongle media info (drv_version, mac address). */ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp); +u8 brcmf_map_prio_to_prec(void *cfg, u8 prio); + +u8 brcmf_map_prio_to_aci(void *cfg, u8 prio); + #endif /* BRCMFMAC_COMMON_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 5cc3a07dda9e6..c671996bd71bb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -344,7 +344,8 @@ void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event) } else { /* Process special event packets */ if (handle_event) - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, + BCMILCP_SUBTYPE_VENDOR_LONG); brcmf_netif_rx(ifp, skb); } @@ -361,7 +362,7 @@ void brcmf_rx_event(struct device *dev, struct sk_buff *skb) if (brcmf_rx_hdrpull(drvr, skb, &ifp)) return; - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, 0); brcmu_pkt_buf_free_skb(skb); } @@ -370,6 +371,11 @@ void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success) struct ethhdr *eh; u16 type; + if (!ifp) { + brcmu_pkt_buf_free_skb(txp); + return; + } + eh = (struct ethhdr *)(txp->data); type = ntohs(eh->h_proto); @@ -413,8 +419,6 @@ static int brcmf_netdev_stop(struct net_device *ndev) brcmf_cfg80211_down(ndev); - brcmf_fil_iovar_data_set(ifp, "arp_hostip_clear", NULL, 0); - brcmf_net_setcarrier(ifp, false); return 0; @@ -925,6 +929,14 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings) return ret; } +int brcmf_fwlog_attach(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; + + return brcmf_debug_fwlog_init(drvr); +} + static int brcmf_revinfo_read(struct seq_file *s, void *data) { struct brcmf_bus *bus_if = dev_get_drvdata(s->private); @@ -950,9 +962,42 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) seq_printf(s, "anarev: %u\n", ri->anarev); seq_printf(s, "nvramrev: %08x\n", ri->nvramrev); + seq_printf(s, "clmver: %s\n", bus_if->drvr->clmver); + return 0; } +static void brcmf_core_bus_reset(struct work_struct *work) +{ + struct brcmf_pub *drvr = container_of(work, struct brcmf_pub, + bus_reset); + + brcmf_bus_reset(drvr->bus_if); +} + +static ssize_t bus_reset_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct brcmf_pub *drvr = file->private_data; + u8 value; + + if (kstrtou8_from_user(user_buf, count, 0, &value)) + return -EINVAL; + + if (value != 1) + return -EINVAL; + + schedule_work(&drvr->bus_reset); + + return count; +} + +static const struct file_operations bus_reset_fops = { + .open = simple_open, + .llseek = no_llseek, + .write = bus_reset_write, +}; + int brcmf_bus_started(struct device *dev) { int ret = -1; @@ -1029,6 +1074,12 @@ int brcmf_bus_started(struct device *dev) #endif #endif /* CONFIG_INET */ + INIT_WORK(&drvr->bus_reset, brcmf_core_bus_reset); + + /* populate debugfs */ + debugfs_create_file("reset", 0600, brcmf_debugfs_get_devdir(drvr), drvr, + &bus_reset_fops); + return 0; fail: @@ -1070,6 +1121,26 @@ void brcmf_dev_reset(struct device *dev) brcmf_fil_cmd_int_set(drvr->iflist[0], BRCMF_C_TERMINATED, 1); } +void brcmf_dev_coredump(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + + if (brcmf_debug_create_memdump(bus_if, NULL, 0) < 0) + brcmf_dbg(TRACE, "failed to create coredump\n"); +} + +void brcmf_fw_crashed(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; + + brcmf_err("Firmware has halted or crashed\n"); + + brcmf_dev_coredump(dev); + + schedule_work(&drvr->bus_reset); +} + void brcmf_detach(struct device *dev) { s32 i; @@ -1132,8 +1203,10 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp) !brcmf_get_pend_8021x_cnt(ifp), MAX_WAIT_FOR_8021X_TX); - if (!err) + if (!err) { brcmf_err("Timed out waiting for no pending 802.1x packets\n"); + atomic_set(&ifp->pend_8021x_cnt, 0); + } return !err; } @@ -1196,3 +1269,178 @@ void __exit brcmf_core_exit(void) #endif } +int +brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, bool add) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pkt_filter_le *pkt_filter; + int filter_fixed_len = offsetof(struct brcmf_pkt_filter_le, u); + int pattern_fixed_len = offsetof(struct brcmf_pkt_filter_pattern_le, + mask_and_pattern); + u16 mask_and_pattern[MAX_PKTFILTER_PATTERN_SIZE]; + int buflen = 0; + int ret = 0; + + brcmf_dbg(INFO, "%s packet filter number %d\n", + (add ? "add" : "remove"), filter_num); + + pkt_filter = kzalloc(sizeof(*pkt_filter) + + (MAX_PKTFILTER_PATTERN_SIZE * 2), GFP_ATOMIC); + if (!pkt_filter) + return -ENOMEM; + + switch (filter_num) { + case BRCMF_UNICAST_FILTER_NUM: + pkt_filter->id = 100; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 1; + mask_and_pattern[0] = 0x0001; + break; + case BRCMF_BROADCAST_FILTER_NUM: + //filter_pattern = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF"; + pkt_filter->id = 101; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0xFFFF; + mask_and_pattern[4] = 0xFFFF; + mask_and_pattern[5] = 0xFFFF; + break; + case BRCMF_MULTICAST4_FILTER_NUM: + //filter_pattern = "102 0 0 0 0xFFFFFF 0x01005E"; + pkt_filter->id = 102; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 3; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x01FF; + mask_and_pattern[2] = 0x5E00; + break; + case BRCMF_MULTICAST6_FILTER_NUM: + //filter_pattern = "103 0 0 0 0xFFFF 0x3333"; + pkt_filter->id = 103; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x3333; + break; + case BRCMF_MDNS_FILTER_NUM: + //filter_pattern = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB"; + pkt_filter->id = 104; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0001; + mask_and_pattern[4] = 0x005E; + mask_and_pattern[5] = 0xFB00; + break; + case BRCMF_ARP_FILTER_NUM: + //filter_pattern = "105 0 0 12 0xFFFF 0x0806"; + pkt_filter->id = 105; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 12; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x0608; + break; + case BRCMF_BROADCAST_ARP_FILTER_NUM: + //filter_pattern = "106 0 0 0 + //0xFFFFFFFFFFFF0000000000000806 + //0xFFFFFFFFFFFF0000000000000806"; + pkt_filter->id = 106; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 14; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0000; + mask_and_pattern[4] = 0x0000; + mask_and_pattern[5] = 0x0000; + mask_and_pattern[6] = 0x0608; + mask_and_pattern[7] = 0xFFFF; + mask_and_pattern[8] = 0xFFFF; + mask_and_pattern[9] = 0xFFFF; + mask_and_pattern[10] = 0x0000; + mask_and_pattern[11] = 0x0000; + mask_and_pattern[12] = 0x0000; + mask_and_pattern[13] = 0x0608; + break; + default: + ret = -EINVAL; + goto failed; + } + memcpy(pkt_filter->u.pattern.mask_and_pattern, mask_and_pattern, + pkt_filter->u.pattern.size_bytes * 2); + buflen = filter_fixed_len + pattern_fixed_len + + pkt_filter->u.pattern.size_bytes * 2; + + if (add) { + /* Add filter */ + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_add", + pkt_filter, buflen); + if (ret) + goto failed; + drvr->pkt_filter[filter_num].id = pkt_filter->id; + drvr->pkt_filter[filter_num].enable = 0; + + } else { + /* Delete filter */ + ret = brcmf_fil_iovar_int_set(ifp, "pkt_filter_delete", + pkt_filter->id); + if (ret == -ENOENT) + ret = 0; + if (ret) + goto failed; + + drvr->pkt_filter[filter_num].id = 0; + drvr->pkt_filter[filter_num].enable = 0; + + } +failed: + if (ret) + brcmf_err("%s packet filter failed, ret=%d\n", + (add ? "add" : "remove"), ret); + + kfree(pkt_filter); + return ret; +} + +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + int ret = 0; + int idx = 0; + + for (idx = 0; idx < MAX_PKT_FILTER_COUNT; ++idx) { + if (drvr->pkt_filter[idx].id != 0) { + drvr->pkt_filter[idx].enable = enable; + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_enable", + &drvr->pkt_filter[idx], + sizeof(struct brcmf_pkt_filter_enable_le)); + if (ret) { + brcmf_err("%s packet filter id(%d) failed, ret=%d\n", + (enable ? "enable" : "disable"), + drvr->pkt_filter[idx].id, ret); + } + } + } + return ret; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index a4dd313140f37..9418d7169a950 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -23,6 +23,7 @@ #include #include "fweh.h" +#include "fwil_types.h" #define TOE_TX_CSUM_OL 0x00000001 #define TOE_RX_CSUM_OL 0x00000002 @@ -36,7 +37,7 @@ #define BRCMF_DCMD_MEDLEN 1536 #define BRCMF_DCMD_MAXLEN 8192 -/* IOCTL from host to device are limited in lenght. A device can only handle +/* IOCTL from host to device are limited in length. A device can only handle * ethernet frame size. This limitation is to be applied by protocol layer. */ #define BRCMF_TX_IOCTL_MAX_MSG_SIZE (ETH_FRAME_LEN+ETH_FCS_LEN) @@ -141,6 +142,12 @@ struct brcmf_pub { struct notifier_block inetaddr_notifier; struct notifier_block inet6addr_notifier; struct brcmf_mp_device *settings; + + struct work_struct bus_reset; + + u8 clmver[BRCMF_DCMD_SMLEN]; + struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; + }; /* forward declarations */ @@ -214,5 +221,7 @@ void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_setcarrier(struct brcmf_if *ifp, bool on); int __init brcmf_core_init(void); void __exit brcmf_core_exit(void); - +int brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, + bool add); +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable); #endif /* BRCMFMAC_CORE_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 2d3e5e263a326..e6bbbdffc33a5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -27,6 +27,82 @@ static struct dentry *root_folder; +static int +brcmf_debug_msgtrace_seqchk(u32 *prev, u32 cur) +{ + if ((cur == 0 && *prev == 0xFFFFFFFF) || ((cur - *prev) == 1)) { + goto done; + } else if (cur == *prev) { + brcmf_dbg(FWCON, "duplicate trace\n"); + return -1; + } else if (cur > *prev) { + brcmf_dbg(FWCON, "lost %d packets\n", cur - *prev); + } else { + brcmf_dbg(FWCON, "seq out of order, host %d, dongle %d\n", + *prev, cur); + } +done: + *prev = cur; + return 0; +} + +static int +brcmf_debug_msg_parser(void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + char *data, *s; + static u32 seqnum_prev; + + hdr = (struct msgtrace_hdr *)event_data; + data = (char *)event_data + MSGTRACE_HDRLEN; + + /* There are 2 bytes available at the end of data */ + data[ntohs(hdr->len)] = '\0'; + + if (ntohl(hdr->discarded_bytes) || ntohl(hdr->discarded_printf)) { + brcmf_dbg(FWCON, "Discarded_bytes %d discarded_printf %d\n", + ntohl(hdr->discarded_bytes), + ntohl(hdr->discarded_printf)); + } + + err = brcmf_debug_msgtrace_seqchk(&seqnum_prev, ntohl(hdr->seqnum)); + if (err) + return err; + + while (*data != '\0' && (s = strstr(data, "\n")) != NULL) { + *s = '\0'; + brcmf_dbg(FWCON, "[FWLOG] %s\n", data); + data = s + 1; + } + if (*data) + brcmf_dbg(FWCON, "[FWLOG] %s", data); + + return err; +} + +static int +brcmf_debug_trace_parser(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + + hdr = (struct msgtrace_hdr *)event_data; + if (hdr->version != MSGTRACE_VERSION) { + brcmf_dbg(FWCON, "trace version mismatch host %d dngl %d\n", + MSGTRACE_VERSION, hdr->version); + err = -EPROTO; + return err; + } + + if (hdr->trace_type == MSGTRACE_HDR_TYPE_MSG) + err = brcmf_debug_msg_parser(event_data); + + return err; +} + int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len) { @@ -42,7 +118,8 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, if (!dump) return -ENOMEM; - memcpy(dump, data, len); + if (data && len > 0) + memcpy(dump, data, len); err = brcmf_bus_get_memdump(bus, dump + len, ramsize); if (err) { vfree(dump); @@ -81,6 +158,12 @@ int brcmf_debug_attach(struct brcmf_pub *drvr) return PTR_ERR_OR_ZERO(drvr->dbgfs_dir); } +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return brcmf_fweh_register(drvr, BRCMF_E_TRACE, + brcmf_debug_trace_parser); +} + void brcmf_debug_detach(struct brcmf_pub *drvr) { brcmf_fweh_unregister(drvr, BRCMF_E_PSM_WATCHDOG); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 35919d9e8e139..4ad813643bce1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -40,6 +40,7 @@ #define BRCMF_MSGBUF_VAL 0x00040000 #define BRCMF_PCIE_VAL 0x00080000 #define BRCMF_FWCON_VAL 0x00100000 +#define BRCMF_ULP_VAL 0x00200000 /* set default print format */ #undef pr_fmt @@ -101,6 +102,10 @@ do { \ #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ +#define MSGTRACE_VERSION 1 +#define MSGTRACE_HDR_TYPE_MSG 0 +#define MSGTRACE_HDR_TYPE_LOG 1 + #define brcmf_dbg_hex_dump(test, data, len, fmt, ...) \ do { \ trace_brcmf_hexdump((void *)data, len); \ @@ -122,6 +127,7 @@ int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)); int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len); +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr); #else static inline void brcmf_debugfs_init(void) { @@ -136,6 +142,11 @@ static inline int brcmf_debug_attach(struct brcmf_pub *drvr) static inline void brcmf_debug_detach(struct brcmf_pub *drvr) { } + +static inline struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr) +{ + return ERR_PTR(-ENOENT); +} static inline int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)) @@ -148,6 +159,25 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, { return 0; } + +static inline +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return 0; +} #endif +/* Message trace header */ +struct msgtrace_hdr { + u8 version; + u8 trace_type; + u16 len; /* Len of the trace */ + u32 seqnum; /* Sequence number of message */ + /* Number of discarded bytes because of trace overflow */ + u32 discarded_bytes; + /* Number of discarded printf because of trace overflow */ + u32 discarded_printf; +}; + +#define MSGTRACE_HDRLEN sizeof(struct msgtrace_hdr) #endif /* BRCMFMAC_DEBUG_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 53ae302599899..a0f27611d7a29 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -48,6 +48,8 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MBSS, "mbss" }, { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, + { BRCMF_FEAT_SAE, "sae" }, + { BRCMF_FEAT_FWAUTH, "idauth" }, }; #ifdef DEBUG @@ -132,11 +134,16 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp, static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp) { - char caps[256]; + char caps[512]; enum brcmf_feat_id id; - int i; + int i, err; + + err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); + if (err) { + brcmf_err("could not get firmware cap (%d)\n", err); + return; + } - brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); brcmf_dbg(INFO, "[ %s]\n", caps); for (i = 0; i < ARRAY_SIZE(brcmf_fwcap_map); i++) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 1ab4f16171120..67ce8cb87b76c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -33,6 +33,8 @@ * MFP: 802.11w Management Frame Protection. * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. + * SAE: simultaneous authentication of equals + * FWAUTH: Firmware authenticator */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -48,7 +50,9 @@ BRCMF_FEAT_DEF(WOWL_ARP_ND) \ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ - BRCMF_FEAT_DEF(FWSUP) + BRCMF_FEAT_DEF(FWSUP) \ + BRCMF_FEAT_DEF(SAE) \ + BRCMF_FEAT_DEF(FWAUTH) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c index 091b52979e030..e5947beff98f0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c @@ -581,6 +581,9 @@ int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev, return -ENODEV; } + fw_name[0] = '\0'; + if (nvram_name) + nvram_name[0] = '\0'; /* check if firmware path is provided by module parameter */ if (brcmf_mp_global.firmware_path[0] != '\0') { strlcpy(fw_name, brcmf_mp_global.firmware_path, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index d0b738da24582..dd328faa5ffbc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -427,7 +427,6 @@ void brcmf_flowring_configure_addr_mode(struct brcmf_flowring *flow, int ifidx, flowid = flow->hash[i].flowid; if (flow->rings[flowid]->status != RING_OPEN) continue; - flow->rings[flowid]->status = RING_CLOSING; brcmf_msgbuf_delete_flowring(drvr, flowid); } } @@ -466,10 +465,8 @@ void brcmf_flowring_delete_peer(struct brcmf_flowring *flow, int ifidx, if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) && (hash[i].ifidx == ifidx)) { flowid = flow->hash[i].flowid; - if (flow->rings[flowid]->status == RING_OPEN) { - flow->rings[flowid]->status = RING_CLOSING; + if (flow->rings[flowid]->status == RING_OPEN) brcmf_msgbuf_delete_flowring(drvr, flowid); - } } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index 816f80ea925b1..9ed8e637a3839 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -101,7 +101,9 @@ struct brcmf_cfg80211_info; BRCMF_ENUM_DEF(FIFO_CREDIT_MAP, 74) \ BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \ BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \ - BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) + BRCMF_ENUM_DEF(PHY_TEMP, 111) \ + BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) \ + BRCMF_ENUM_DEF(ULP, 146) #define BRCMF_ENUM_DEF(id, val) \ BRCMF_E_##id = (val), @@ -113,7 +115,7 @@ enum brcmf_fweh_event_code { * minimum length check in device firmware so it is * hard-coded here. */ - BRCMF_E_LAST = 139 + BRCMF_E_LAST = 147 }; #undef BRCMF_ENUM_DEF @@ -211,7 +213,7 @@ enum brcmf_fweh_event_code { */ #define BRCM_OUI "\x00\x10\x18" #define BCMILCP_BCM_SUBTYPE_EVENT 1 - +#define BCMILCP_SUBTYPE_VENDOR_LONG 32769 /** * struct brcm_ethhdr - broadcom specific ether header. @@ -294,6 +296,28 @@ struct brcmf_if_event { u8 role; }; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 +#define EVENTMSGS_EXT_STRUCT_SIZE offsetof(struct eventmsgs_ext, mask[0]) + +/* len- for SET it would be mask size from the application to the firmware */ +/* for GET it would be actual firmware mask size */ +/* maxgetsize - is only used for GET. indicate max mask size that the */ +/* application can read from the firmware */ +struct eventmsgs_ext { + u8 ver; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[1]; +}; + typedef int (*brcmf_fweh_handler_t)(struct brcmf_if *ifp, const struct brcmf_event_msg *evtmsg, void *data); @@ -334,10 +358,10 @@ void brcmf_fweh_process_event(struct brcmf_pub *drvr, void brcmf_fweh_p2pdev_setup(struct brcmf_if *ifp, bool ongoing); static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, - struct sk_buff *skb) + struct sk_buff *skb, u16 stype) { struct brcmf_event *event_packet; - u16 usr_stype; + u16 subtype, usr_stype; /* only process events when protocol matches */ if (skb->protocol != cpu_to_be16(ETH_P_LINK_CTL)) @@ -346,8 +370,16 @@ static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, if ((skb->len + ETH_HLEN) < sizeof(*event_packet)) return; - /* check for BRCM oui match */ event_packet = (struct brcmf_event *)skb_mac_header(skb); + + /* check subtype if needed */ + if (unlikely(stype)) { + subtype = get_unaligned_be16(&event_packet->hdr.subtype); + if (subtype != stype) + return; + } + + /* check for BRCM oui match */ if (memcmp(BRCM_OUI, &event_packet->hdr.oui[0], sizeof(event_packet->hdr.oui))) return; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h index 63b1287e2e6d4..b6b183b184135 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h @@ -80,6 +80,7 @@ #define BRCMF_C_SCB_DEAUTHENTICATE_FOR_REASON 201 #define BRCMF_C_SET_ASSOC_PREFER 205 #define BRCMF_C_GET_VALID_CHANNELS 217 +#define BRCMF_C_SET_FAKEFRAG 219 #define BRCMF_C_GET_KEY_PRIMARY 235 #define BRCMF_C_SET_KEY_PRIMARY 236 #define BRCMF_C_SET_SCAN_PASSIVE_TIME 258 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index e0d22fedb2b45..73b11242812d4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -30,7 +30,7 @@ #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 #define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ -#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0002 +#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_WME 0x00000002 /* WMM association */ #define BRCMF_STA_AUTHE 0x00000008 /* Authenticated */ @@ -53,6 +53,8 @@ #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 + /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 @@ -125,9 +127,22 @@ /* Link Down indication in WoWL mode: */ #define BRCMF_WOWL_LINKDOWN (1 << 31) -#define BRCMF_WOWL_MAXPATTERNS 8 +#define BRCMF_WOWL_MAXPATTERNS 16 #define BRCMF_WOWL_MAXPATTERNSIZE 128 +enum { + BRCMF_UNICAST_FILTER_NUM = 0, + BRCMF_BROADCAST_FILTER_NUM, + BRCMF_MULTICAST4_FILTER_NUM, + BRCMF_MULTICAST6_FILTER_NUM, + BRCMF_MDNS_FILTER_NUM, + BRCMF_ARP_FILTER_NUM, + BRCMF_BROADCAST_ARP_FILTER_NUM, + MAX_PKT_FILTER_COUNT +}; + +#define MAX_PKTFILTER_PATTERN_SIZE 16 + #define BRCMF_COUNTRY_BUF_SZ 4 #define BRCMF_ANT_MAX 4 @@ -155,6 +170,21 @@ #define BRCMF_MFP_CAPABLE 1 #define BRCMF_MFP_REQUIRED 2 +/* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each + * ioctl. It is relatively small because firmware has small maximum size input + * playload restriction for ioctls. + */ +#define MAX_CHUNK_LEN 1400 + +#define DLOAD_HANDLER_VER 1 /* Downloader version */ +#define DLOAD_FLAG_VER_MASK 0xf000 /* Downloader version mask */ +#define DLOAD_FLAG_VER_SHIFT 12 /* Downloader version shift */ + +#define DL_BEGIN 0x0002 +#define DL_END 0x0004 + +#define DL_TYPE_CLM 2 + /* join preference types for join_pref iovar */ enum brcmf_join_pref_types { BRCMF_JOIN_PREF_RSSI = 1, @@ -491,6 +521,17 @@ struct brcmf_wsec_pmk_le { u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; }; +/** + * struct brcmf_wsec_sae_pwd_le - firmware SAE password material. + * + * @key_len: number of octets in key materials. + * @key: SAE password material. + */ +struct brcmf_wsec_sae_pwd_le { + __le16 key_len; + u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; +}; + /* Used to get specific STA parameters */ struct brcmf_scb_val_le { __le32 val; @@ -826,6 +867,22 @@ struct brcmf_pno_macaddr_le { u8 mac[ETH_ALEN]; }; +/** + * struct brcmf_dload_data_le - data passing to firmware for downloading + * @flag: flags related to download data. + * @dload_type: type of download data. + * @len: length in bytes of download data. + * @crc: crc of download data. + * @data: download data. + */ +struct brcmf_dload_data_le { + __le16 flag; + __le16 dload_type; + __le32 len; + __le32 crc; + u8 data[1]; +}; + /** * struct brcmf_pno_bssid_le - bssid configuration for PNO scan. * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index f59642b2c935a..cda807c46c18b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -322,28 +322,6 @@ struct brcmf_skbuff_cb { /* How long to defer borrowing in jiffies */ #define BRCMF_FWS_BORROW_DEFER_PERIOD (HZ / 10) -/** - * enum brcmf_fws_fifo - fifo indices used by dongle firmware. - * - * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. - * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. - * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. - * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. - * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. - * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). - * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). - * @BRCMF_FWS_FIFO_COUNT: number of fifos. - */ -enum brcmf_fws_fifo { - BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BE, - BRCMF_FWS_FIFO_AC_VI, - BRCMF_FWS_FIFO_AC_VO, - BRCMF_FWS_FIFO_BCMC, - BRCMF_FWS_FIFO_ATIM, - BRCMF_FWS_FIFO_COUNT -}; /** * enum brcmf_fws_txstatus - txstatus flag values. @@ -356,6 +334,10 @@ enum brcmf_fws_fifo { * firmware suppress the packet as device is already in PS mode. * @BRCMF_FWS_TXSTATUS_FW_TOSSED: * firmware tossed the packet. + * @BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK: + * firmware tossed the packet after retries. + * @BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED: + * firmware wrongly reported suppressed previously, now fixing to acked. * @BRCMF_FWS_TXSTATUS_HOST_TOSSED: * host tossed the packet. */ @@ -364,6 +346,8 @@ enum brcmf_fws_txstatus { BRCMF_FWS_TXSTATUS_CORE_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_TOSSED, + BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK, + BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED, BRCMF_FWS_TXSTATUS_HOST_TOSSED }; @@ -415,7 +399,8 @@ struct brcmf_fws_mac_descriptor { u8 traffic_lastreported_bmp; }; -#define BRCMF_FWS_HANGER_MAXITEMS 1024 +#define BRCMF_FWS_HANGER_MAXITEMS 3072 +#define BRCMF_BORROW_RATIO 3 /** * enum brcmf_fws_hanger_item_state - state of hanger item. @@ -511,7 +496,9 @@ struct brcmf_fws_info { struct work_struct fws_dequeue_work; u32 fifo_enqpkt[BRCMF_FWS_FIFO_COUNT]; int fifo_credit[BRCMF_FWS_FIFO_COUNT]; - int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1]; + int init_fifo_credit[BRCMF_FWS_FIFO_COUNT]; + int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1] + [BRCMF_FWS_FIFO_AC_VO + 1]; int deq_node_pos[BRCMF_FWS_FIFO_COUNT]; u32 fifo_credit_map; u32 fifo_delay_map; @@ -520,6 +507,7 @@ struct brcmf_fws_info { bool creditmap_received; u8 mode; bool avoid_queueing; + int fifo_init_credit[BRCMF_FWS_FIFO_COUNT]; }; /* @@ -582,15 +570,21 @@ static bool brcmf_fws_ifidx_match(struct sk_buff *skb, void *arg) static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, int ifidx) { + struct brcmf_fws_hanger_item *hi; bool (*matchfn)(struct sk_buff *, void *) = NULL; struct sk_buff *skb; int prec; + u32 hslot; if (ifidx != -1) matchfn = brcmf_fws_ifidx_match; for (prec = 0; prec < q->num_prec; prec++) { skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); while (skb) { + hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); + hi = &fws->hanger.items[hslot]; + WARN_ON(skb != hi->pkt); + hi->state = BRCMF_FWS_HANGER_ITEM_STATE_FREE; brcmu_pkt_buf_free_skb(skb); skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); } @@ -1215,11 +1209,11 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, fws->fifo_credit_map |= 1 << fifo; - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->credits_borrowed[0])) { + if (fifo > BRCMF_FWS_FIFO_AC_BK && + fifo <= BRCMF_FWS_FIFO_AC_VO) { for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0; lender_ac--) { - borrowed = &fws->credits_borrowed[lender_ac]; + borrowed = &fws->credits_borrowed[fifo][lender_ac]; if (*borrowed) { fws->fifo_credit_map |= (1 << lender_ac); fifo_credit = &fws->fifo_credit[lender_ac]; @@ -1236,7 +1230,13 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, } } - fws->fifo_credit[fifo] += credits; + if (credits) { + fws->fifo_credit[fifo] += credits; + } + + if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) + fws->fifo_credit[fifo] = fws->init_fifo_credit[fifo]; + } static void brcmf_fws_schedule_deq(struct brcmf_fws_info *fws) @@ -1451,9 +1451,10 @@ static int brcmf_fws_txstatus_suppressed(struct brcmf_fws_info *fws, int fifo, static int brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, - u32 genbit, u16 seq) + u32 genbit, u16 seq, u8 compcnt) { u32 fifo; + u8 cnt = 0; int ret; bool remove_from_hanger = true; struct sk_buff *skb; @@ -1464,60 +1465,75 @@ brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, brcmf_dbg(DATA, "flags %d\n", flags); if (flags == BRCMF_FWS_TXSTATUS_DISCARD) - fws->stats.txs_discard++; + fws->stats.txs_discard += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_CORE_SUPPRESS) { - fws->stats.txs_supp_core++; + fws->stats.txs_supp_core += compcnt; remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS) { - fws->stats.txs_supp_ps++; + fws->stats.txs_supp_ps += compcnt; remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_TOSSED) - fws->stats.txs_tossed++; + fws->stats.txs_tossed += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK) + fws->stats.txs_discard += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED) + fws->stats.txs_discard += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED) - fws->stats.txs_host_tossed++; + fws->stats.txs_host_tossed += compcnt; else brcmf_err("unexpected txstatus\n"); - ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, - remove_from_hanger); - if (ret != 0) { - brcmf_err("no packet in hanger slot: hslot=%d\n", hslot); - return ret; - } + while (cnt < compcnt) { + ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, + remove_from_hanger); + if (ret != 0) { + brcmf_err("no packet in hanger slot: hslot=%d\n", + hslot); + goto cont; + } - skcb = brcmf_skbcb(skb); - entry = skcb->mac; - if (WARN_ON(!entry)) { - brcmu_pkt_buf_free_skb(skb); - return -EINVAL; - } - entry->transit_count--; - if (entry->suppressed && entry->suppr_transit_count) - entry->suppr_transit_count--; + skcb = brcmf_skbcb(skb); + entry = skcb->mac; + if (WARN_ON(!entry)) { + brcmu_pkt_buf_free_skb(skb); + goto cont; + } + entry->transit_count--; + if (entry->suppressed && entry->suppr_transit_count) + entry->suppr_transit_count--; - brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name, flags, - skcb->htod, seq); + brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name, + flags, skcb->htod, seq); - /* pick up the implicit credit from this packet */ - fifo = brcmf_skb_htod_tag_get_field(skb, FIFO); - if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) || - (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) || - (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) { - brcmf_fws_return_credits(fws, fifo, 1); - brcmf_fws_schedule_deq(fws); - } - brcmf_fws_macdesc_return_req_credit(skb); + /* pick up the implicit credit from this packet */ + fifo = brcmf_skb_htod_tag_get_field(skb, FIFO); + if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) || + (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) || + (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) { + brcmf_fws_return_credits(fws, fifo, 1); + brcmf_fws_schedule_deq(fws); + } + brcmf_fws_macdesc_return_req_credit(skb); - ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp); - if (ret) { - brcmu_pkt_buf_free_skb(skb); - return -EINVAL; + ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp); + if (ret) { + brcmu_pkt_buf_free_skb(skb); + goto cont; + } + if (!remove_from_hanger) + ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb, + genbit, seq); + if (remove_from_hanger || ret) + brcmf_txfinalize(ifp, skb, true); + +cont: + hslot = (hslot + 1) & (BRCMF_FWS_TXSTAT_HSLOT_MASK >> + BRCMF_FWS_TXSTAT_HSLOT_SHIFT); + if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode)) + seq = (seq + 1) & BRCMF_SKB_HTOD_SEQ_NR_MASK; + + cnt++; } - if (!remove_from_hanger) - ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb, - genbit, seq); - if (remove_from_hanger || ret) - brcmf_txfinalize(ifp, skb, true); return 0; } @@ -1543,7 +1559,8 @@ static int brcmf_fws_fifocreditback_indicate(struct brcmf_fws_info *fws, return BRCMF_FWS_RET_OK_SCHEDULE; } -static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 *data) +static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 type, + u8 *data) { __le32 status_le; __le16 seq_le; @@ -1552,23 +1569,31 @@ static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 *data) u32 genbit; u8 flags; u16 seq; + u8 compcnt; + u8 compcnt_offset = BRCMF_FWS_TYPE_TXSTATUS_LEN; - fws->stats.txs_indicate++; memcpy(&status_le, data, sizeof(status_le)); status = le32_to_cpu(status_le); flags = brcmf_txstatus_get_field(status, FLAGS); hslot = brcmf_txstatus_get_field(status, HSLOT); genbit = brcmf_txstatus_get_field(status, GENERATION); if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode)) { - memcpy(&seq_le, &data[BRCMF_FWS_TYPE_PKTTAG_LEN], + memcpy(&seq_le, &data[BRCMF_FWS_TYPE_TXSTATUS_LEN], sizeof(seq_le)); seq = le16_to_cpu(seq_le); + compcnt_offset += BRCMF_FWS_TYPE_SEQ_LEN; } else { seq = 0; } + if (type == BRCMF_FWS_TYPE_COMP_TXSTATUS) + compcnt = data[compcnt_offset]; + else + compcnt = 1; + fws->stats.txs_indicate += compcnt; + brcmf_fws_lock(fws); - brcmf_fws_txs_process(fws, flags, hslot, genbit, seq); + brcmf_fws_txs_process(fws, flags, hslot, genbit, seq, compcnt); brcmf_fws_unlock(fws); return BRCMF_FWS_RET_OK_NOSCHEDULE; } @@ -1595,19 +1620,22 @@ static int brcmf_fws_notify_credit_map(struct brcmf_if *ifp, brcmf_err("event payload too small (%d)\n", e->datalen); return -EINVAL; } - if (fws->creditmap_received) - return 0; fws->creditmap_received = true; brcmf_dbg(TRACE, "enter: credits %pM\n", credits); brcmf_fws_lock(fws); for (i = 0; i < ARRAY_SIZE(fws->fifo_credit); i++) { - if (*credits) + fws->fifo_credit[i] += credits[i] - fws->init_fifo_credit[i]; + fws->init_fifo_credit[i] = credits[i]; + if (fws->fifo_credit[i] > 0) fws->fifo_credit_map |= 1 << i; else fws->fifo_credit_map &= ~(1 << i); - fws->fifo_credit[i] = *credits++; + if (fws->fifo_credit[i] < 0) + brcmf_err("fifo_credit[%d] value is negative(%d)\n", + i, fws->fifo_credit[i]); + fws->fifo_init_credit[i] = fws->fifo_credit[i]; } brcmf_fws_schedule_deq(fws); brcmf_fws_unlock(fws); @@ -1841,6 +1869,9 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) WARN_ON(siglen > skb->len); + if (siglen > skb->len) + siglen = skb->len; + if (!siglen) return; /* if flow control disabled, skip to packet data and leave */ @@ -1882,8 +1913,6 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) err = BRCMF_FWS_RET_OK_NOSCHEDULE; switch (type) { - case BRCMF_FWS_TYPE_COMP_TXSTATUS: - break; case BRCMF_FWS_TYPE_HOST_REORDER_RXPKTS: rd = (struct brcmf_skb_reorder_data *)skb->cb; rd->reorder = data; @@ -1906,7 +1935,8 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) err = brcmf_fws_request_indicate(fws, type, data); break; case BRCMF_FWS_TYPE_TXSTATUS: - brcmf_fws_txstatus_indicate(fws, data); + case BRCMF_FWS_TYPE_COMP_TXSTATUS: + brcmf_fws_txstatus_indicate(fws, type, data); break; case BRCMF_FWS_TYPE_FIFO_CREDITBACK: err = brcmf_fws_fifocreditback_indicate(fws, data); @@ -1995,7 +2025,7 @@ static void brcmf_fws_rollback_toq(struct brcmf_fws_info *fws, fws->stats.rollback_failed++; hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, - hslot, 0, 0); + hslot, 0, 0, 1); } else { fws->stats.rollback_success++; brcmf_fws_return_credits(fws, fifo, 1); @@ -2003,27 +2033,31 @@ static void brcmf_fws_rollback_toq(struct brcmf_fws_info *fws, } } -static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws) +static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws, + int highest_lender_ac, int borrower_ac, + bool borrow_all) { - int lender_ac; + int lender_ac, borrow_limit = 0; - if (time_after(fws->borrow_defer_timestamp, jiffies)) { - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); - return -ENAVAIL; - } + for (lender_ac = 0; lender_ac <= highest_lender_ac; lender_ac++) { + + if (!borrow_all) + borrow_limit = + fws->init_fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; + else + borrow_limit = 0; - for (lender_ac = 0; lender_ac <= BRCMF_FWS_FIFO_AC_VO; lender_ac++) { - if (fws->fifo_credit[lender_ac]) { - fws->credits_borrowed[lender_ac]++; + if (fws->fifo_credit[lender_ac] > borrow_limit) { + fws->credits_borrowed[borrower_ac][lender_ac]++; fws->fifo_credit[lender_ac]--; if (fws->fifo_credit[lender_ac] == 0) fws->fifo_credit_map &= ~(1 << lender_ac); - fws->fifo_credit_map |= (1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map |= (1 << borrower_ac); brcmf_dbg(DATA, "borrow credit from: %d\n", lender_ac); return 0; } } - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map &= ~(1 << borrower_ac); return -ENAVAIL; } @@ -2092,6 +2126,7 @@ static int brcmf_fws_assign_htod(struct brcmf_fws_info *fws, struct sk_buff *p, int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) { + struct brcmf_pub *drvr = ifp->drvr; struct brcmf_fws_info *fws = drvr_to_fws(ifp->drvr); struct brcmf_skbuff_cb *skcb = brcmf_skbcb(skb); struct ethhdr *eh = (struct ethhdr *)(skb->data); @@ -2105,8 +2140,10 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) skcb->if_flags = 0; skcb->state = BRCMF_FWS_SKBSTATE_NEW; brcmf_skb_if_flags_set_field(skb, INDEX, ifp->ifidx); + + /* mapping from 802.1d priority to firmware fifo index */ if (!multicast) - fifo = brcmf_fws_prio2fifo[skb->priority]; + fifo = brcmf_map_prio_to_aci(drvr->config, skb->priority); brcmf_fws_lock(fws); if (fifo != BRCMF_FWS_FIFO_AC_BE && fifo < BRCMF_FWS_FIFO_BCMC) @@ -2120,8 +2157,7 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) brcmf_fws_enq(fws, BRCMF_FWS_SKBSTATE_DELAYED, fifo, skb); brcmf_fws_schedule_deq(fws); } else { - brcmf_err("drop skb: no hanger slot\n"); - brcmf_txfinalize(ifp, skb, false); + brcmf_err("no hanger slot available\n"); rc = -ENOMEM; } brcmf_fws_unlock(fws); @@ -2167,12 +2203,43 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_lock(fws); ifp->fws_desc = NULL; + brcmf_fws_macdesc_cleanup(fws, entry, ifp->ifidx); brcmf_dbg(TRACE, "deleting %s\n", entry->name); brcmf_fws_macdesc_deinit(entry); brcmf_fws_cleanup(fws, ifp->ifidx); brcmf_fws_unlock(fws); } +static bool brcmf_fws_ismultistream(struct brcmf_fws_info *fws) +{ + bool ret = false; + u8 credit_usage = 0; + + /* Check only for BE, VI and VO traffic */ + u32 delay_map = fws->fifo_delay_map & + ((1 << BRCMF_FWS_FIFO_AC_BE) | + (1 << BRCMF_FWS_FIFO_AC_VI) | + (1 << BRCMF_FWS_FIFO_AC_VO)); + + if (hweight_long(delay_map) > 1) { + ret = true; + } else { + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_BE] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_BE]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VI] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VI]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VO] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VO]) + credit_usage++; + + if (credit_usage > 1) + ret = true; + } + return ret; +} + static void brcmf_fws_dequeue_worker(struct work_struct *worker) { struct brcmf_fws_info *fws; @@ -2186,6 +2253,11 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) fws = container_of(worker, struct brcmf_fws_info, fws_dequeue_work); drvr = fws->drvr; + if (brcmf_fws_ismultistream(fws)) + drvr->bus_if->allow_skborphan = false; + else + drvr->bus_if->allow_skborphan = true; + brcmf_fws_lock(fws); for (fifo = BRCMF_FWS_FIFO_BCMC; fifo >= 0 && !fws->bus_flow_blocked; fifo--) { @@ -2210,8 +2282,10 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) } continue; } - while ((fws->fifo_credit[fifo]) || ((!fws->bcmc_credit_check) && - (fifo == BRCMF_FWS_FIFO_BCMC))) { + + while ((fws->fifo_credit[fifo]) || + ((!fws->bcmc_credit_check) && + (fifo == BRCMF_FWS_FIFO_BCMC))) { skb = brcmf_fws_deq(fws, fifo); if (!skb) break; @@ -2221,10 +2295,14 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) if (fws->bus_flow_blocked) break; } - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->fifo_credit[fifo] == 0) && - (!fws->bus_flow_blocked)) { - while (brcmf_fws_borrow_credit(fws) == 0) { + + if (fifo >= BRCMF_FWS_FIFO_AC_BE && + fifo <= BRCMF_FWS_FIFO_AC_VO && + fws->fifo_credit[fifo] == 0 && + !fws->bus_flow_blocked) { + while (brcmf_fws_borrow_credit(fws, + fifo - 1, fifo, + true) == 0) { skb = brcmf_fws_deq(fws, fifo); if (!skb) { brcmf_fws_return_credits(fws, fifo, 1); @@ -2452,7 +2530,8 @@ void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb) } brcmf_fws_lock(fws); hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); - brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0); + brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0, + 1); brcmf_fws_unlock(fws); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index ba07bd9720020..02540e6fa79c5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -18,6 +18,29 @@ #ifndef FWSIGNAL_H_ #define FWSIGNAL_H_ +/** + * enum brcmf_fws_fifo - fifo indices used by dongle firmware. + * + * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. + * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. + * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. + * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. + * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. + * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). + * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). + * @BRCMF_FWS_FIFO_COUNT: number of fifos. + */ +enum brcmf_fws_fifo { + BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BE, + BRCMF_FWS_FIFO_AC_VI, + BRCMF_FWS_FIFO_AC_VO, + BRCMF_FWS_FIFO_BCMC, + BRCMF_FWS_FIFO_ATIM, + BRCMF_FWS_FIFO_COUNT +}; + struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr); void brcmf_fws_detach(struct brcmf_fws_info *fws); bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index d2c834c3b2fc9..dc232cde52901 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -63,7 +63,7 @@ #define BRCMF_IOCTL_REQ_PKTID 0xFFFE -#define BRCMF_MSGBUF_MAX_PKT_SIZE 2048 +#define BRCMF_MSGBUF_MAX_PKT_SIZE 8192 #define BRCMF_MSGBUF_RXBUFPOST_THRESHOLD 32 #define BRCMF_MSGBUF_MAX_IOCTLRESPBUF_POST 8 #define BRCMF_MSGBUF_MAX_EVENTBUF_POST 8 @@ -78,6 +78,7 @@ #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS 32 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS 48 +#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES 10 struct msgbuf_common_hdr { u8 msgtype; @@ -330,7 +331,7 @@ brcmf_msgbuf_alloc_pktid(struct device *dev, do { (*idx)++; if (*idx == pktids->array_size) - *idx = 0; + *idx = 1; if (array[*idx].allocated.counter == 0) if (atomic_cmpxchg(&array[*idx].allocated, 0, 1) == 0) break; @@ -793,8 +794,12 @@ static int brcmf_msgbuf_tx_queue_data(struct brcmf_pub *drvr, int ifidx, flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx); if (flowid == BRCMF_FLOWRING_INVALID_ID) { flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb); - if (flowid == BRCMF_FLOWRING_INVALID_ID) + if (flowid == BRCMF_FLOWRING_INVALID_ID) { return -ENOMEM; + } else { + brcmf_flowring_enqueue(flow, flowid, skb); + return 0; + } } queue_count = brcmf_flowring_enqueue(flow, flowid, skb); force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0); @@ -1112,7 +1117,7 @@ static void brcmf_msgbuf_process_event(struct brcmf_msgbuf *msgbuf, void *buf) skb->protocol = eth_type_trans(skb, ifp->ndev); - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, 0); exit: brcmu_pkt_buf_free_skb(skb); @@ -1324,9 +1329,27 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; + struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid]; + struct brcmf_flowring *flow = msgbuf->flow; void *ret_ptr; u8 ifidx; int err; + int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; + + /* make sure it is not in txflow */ + brcmf_commonring_lock(commonring_del); + flow->rings[flowid]->status = RING_CLOSING; + brcmf_commonring_unlock(commonring_del); + + /* wait for commonring txflow finished */ + while (retry && atomic_read(&commonring_del->outstanding_tx)) { + usleep_range(5000, 10000); + retry--; + } + if (!retry) { + brcmf_err("timed out waiting for txstatus\n"); + atomic_set(&commonring_del->outstanding_tx, 0); + } commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; brcmf_commonring_lock(commonring); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index f93ba6be1ef87..692235d252779 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -27,8 +27,10 @@ #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 #define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE 32 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE 24 -#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 16 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 32 +#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7 16 +#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 24 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 32 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 40 #define BRCMF_H2D_TXFLOWRING_ITEMSIZE 48 struct msgbuf_buf_addr { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index aee6e5937c41c..fb19ab792f818 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -30,14 +30,22 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, struct device_node *np = dev->of_node; int irq; u32 irqf; - u32 val; + u32 val32; + u16 val16; if (!np || bus_type != BRCMF_BUSTYPE_SDIO || !of_device_is_compatible(np, "brcm,bcm4329-fmac")) return; - if (of_property_read_u32(np, "brcm,drive-strength", &val) == 0) - sdio->drive_strength = val; + if (of_property_read_u32(np, "brcm,drive-strength", &val32) == 0) + sdio->drive_strength = val32; + + sdio->broken_sg_support = of_property_read_bool(np, + "brcm,broken_sg_support"); + if (of_property_read_u16(np, "brcm,sd_head_align", &val16) == 0) + sdio->sd_head_align = val16; + if (of_property_read_u16(np, "brcm,sd_sgentry_align", &val16) == 0) + sdio->sd_sgentry_align = val16; /* make sure there are interrupts defined in the node */ if (!of_find_property(np, "interrupts", NULL)) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 450f2216fac2d..40f940df23b12 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -28,6 +28,7 @@ #include "fwil_types.h" #include "p2p.h" #include "cfg80211.h" +#include "feature.h" /* parameters used for p2p escan */ #define P2PAPI_SCAN_NPROBES 1 @@ -70,12 +71,13 @@ #define P2P_AF_MIN_DWELL_TIME 100 #define P2P_AF_MED_DWELL_TIME 400 #define P2P_AF_LONG_DWELL_TIME 1000 -#define P2P_AF_TX_MAX_RETRY 1 +#define P2P_AF_TX_MAX_RETRY 5 #define P2P_AF_MAX_WAIT_TIME msecs_to_jiffies(2000) #define P2P_INVALID_CHANNEL -1 #define P2P_CHANNEL_SYNC_RETRY 5 -#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(1500) +#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(450) #define P2P_DEFAULT_SLEEP_TIME_VSDB 200 +#define P2P_AF_RETRY_DELAY_TIME 40 /* WiFi P2P Public Action Frame OUI Subtypes */ #define P2P_PAF_GON_REQ 0 /* Group Owner Negotiation Req */ @@ -103,6 +105,9 @@ #define P2PSD_ACTION_ID_GAS_CRESP 0x0d /* GAS Comback Response AF */ #define BRCMF_P2P_DISABLE_TIMEOUT msecs_to_jiffies(500) + +/* Mask for retry counter of custom dwell time */ +#define CUSTOM_RETRY_MASK 0xff000000 /** * struct brcmf_p2p_disc_st_le - set discovery state in firmware. * @@ -467,10 +472,21 @@ static int brcmf_p2p_set_firmware(struct brcmf_if *ifp, u8 *p2p_mac) */ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) { + struct brcmf_if *pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; bool random_addr = false; + bool local_admin = false; - if (!dev_addr || is_zero_ether_addr(dev_addr)) - random_addr = true; + if (!dev_addr || is_zero_ether_addr(dev_addr)) { + /* If the primary interface address is already locally + * administered, create a new random address. + */ + if (pri_ifp->mac_addr[0] & 0x02) { + random_addr = true; + } else { + dev_addr = pri_ifp->mac_addr; + local_admin = true; + } + } /* Generate the P2P Device Address obtaining a random ethernet * address with the locally administered bit set. @@ -480,13 +496,20 @@ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) else memcpy(p2p->dev_addr, dev_addr, ETH_ALEN); + if (local_admin) + p2p->dev_addr[0] |= 0x02; + /* Generate the P2P Interface Address. If the discovery and connection * BSSCFGs need to simultaneously co-exist, then this address must be * different from the P2P Device Address, but also locally administered. */ - memcpy(p2p->int_addr, p2p->dev_addr, ETH_ALEN); - p2p->int_addr[0] |= 0x02; - p2p->int_addr[4] ^= 0x80; + memcpy(p2p->conn_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn_int_addr[0] |= 0x02; + p2p->conn_int_addr[4] ^= 0x80; + + memcpy(p2p->conn2_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn2_int_addr[0] |= 0x02; + p2p->conn2_int_addr[4] ^= 0x90; } /** @@ -1139,7 +1162,6 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) { struct afx_hdl *afx_hdl = &p2p->afx_hdl; struct brcmf_cfg80211_vif *pri_vif; - unsigned long duration; s32 retry; brcmf_dbg(TRACE, "Enter\n"); @@ -1155,7 +1177,6 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) * pending action frame tx is cancelled. */ retry = 0; - duration = msecs_to_jiffies(P2P_AF_FRM_SCAN_MAX_WAIT); while ((retry < P2P_CHANNEL_SYNC_RETRY) && (afx_hdl->peer_chan == P2P_INVALID_CHANNEL)) { afx_hdl->is_listen = false; @@ -1163,7 +1184,8 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) retry); /* search peer on peer's listen channel */ schedule_work(&afx_hdl->afx_work); - wait_for_completion_timeout(&afx_hdl->act_frm_scan, duration); + wait_for_completion_timeout(&afx_hdl->act_frm_scan, + P2P_AF_FRM_SCAN_MAX_WAIT); if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, &p2p->status))) @@ -1176,7 +1198,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) afx_hdl->is_listen = true; schedule_work(&afx_hdl->afx_work); wait_for_completion_timeout(&afx_hdl->act_frm_scan, - duration); + P2P_AF_FRM_SCAN_MAX_WAIT); } if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, @@ -1254,6 +1276,30 @@ bool brcmf_p2p_scan_finding_common_channel(struct brcmf_cfg80211_info *cfg, return true; } +/** + * brcmf_p2p_abort_action_frame() - abort action frame. + * + * @cfg: common configuration struct. + * + */ +static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_p2p_info *p2p = &cfg->p2p; + struct brcmf_cfg80211_vif *vif; + s32 err; + s32 int_val = 1; + + brcmf_dbg(TRACE, "Enter\n"); + + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val, + sizeof(s32)); + if (err) + brcmf_err(" aborting action frame has failed (%d)\n", err); + + return err; +} + /** * brcmf_p2p_stop_wait_next_action_frame() - finish scan if af tx complete. * @@ -1265,6 +1311,7 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) { struct brcmf_p2p_info *p2p = &cfg->p2p; struct brcmf_if *ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; + s32 err; if (test_bit(BRCMF_P2P_STATUS_SENDING_ACT_FRAME, &p2p->status) && (test_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status) || @@ -1273,8 +1320,13 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) /* if channel is not zero, "actfame" uses off channel scan. * So abort scan for off channel completion. */ - if (p2p->af_sent_channel) - brcmf_notify_escan_complete(cfg, ifp, true, true); + if (p2p->af_sent_channel) { + /* abort actframe using actframe_abort or abort scan */ + err = brcmf_p2p_abort_action_frame(cfg); + if (err) + brcmf_notify_escan_complete( + cfg, ifp, true, true); + } } else if (test_bit(BRCMF_P2P_STATUS_WAITING_NEXT_AF_LISTEN, &p2p->status)) { brcmf_dbg(TRACE, "*** Wake UP ** abort listen for next af frame\n"); @@ -1463,10 +1515,12 @@ int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, return 0; if (e->event_code == BRCMF_E_ACTION_FRAME_COMPLETE) { - if (e->status == BRCMF_E_STATUS_SUCCESS) + if (e->status == BRCMF_E_STATUS_SUCCESS) { set_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); - else { + if (!p2p->wait_for_offchan_complete) + complete(&p2p->send_af_done); + } else { set_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); /* If there is no ack, we don't need to wait for * WLC_E_ACTION_FRAME_OFFCHAN_COMPLETE event @@ -1497,6 +1551,7 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, struct brcmf_fil_af_params_le *af_params) { struct brcmf_cfg80211_vif *vif; + struct brcmf_p2p_action_frame *p2p_act_frame; s32 err = 0; s32 timeout = 0; @@ -1506,7 +1561,13 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, clear_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); clear_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); - vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + /* check if it is a p2p_presence response */ + p2p_act_frame = (struct brcmf_p2p_action_frame *) af_params->action_frame.data; + if (p2p_act_frame->subtype == P2P_AF_PRESENCE_RSP) + vif = p2p->bss_idx[P2PAPI_BSSCFG_CONNECTION].vif; + else + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe", af_params, sizeof(*af_params)); if (err) { @@ -1517,6 +1578,17 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, p2p->af_sent_channel = le32_to_cpu(af_params->channel); p2p->af_tx_sent_jiffies = jiffies; + if (test_bit(BRCMF_P2P_STATUS_DISCOVER_LISTEN, &p2p->status) && + p2p->af_sent_channel == + ieee80211_frequency_to_channel(p2p->remain_on_channel.center_freq)) + p2p->wait_for_offchan_complete = false; + else + p2p->wait_for_offchan_complete = true; + + brcmf_dbg(TRACE, "Waiting for %s tx completion event\n", + (p2p->wait_for_offchan_complete) ? + "off-channel" : "on-channel"); + timeout = wait_for_completion_timeout(&p2p->send_af_done, P2P_AF_MAX_WAIT_TIME); @@ -1634,6 +1706,17 @@ static s32 brcmf_p2p_pub_af_tx(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_p2p_check_dwell_overflow(s32 requested_dwell, + unsigned long dwell_jiffies) +{ + if ((requested_dwell & CUSTOM_RETRY_MASK) && + (jiffies_to_msecs(jiffies - dwell_jiffies) > + (requested_dwell & ~CUSTOM_RETRY_MASK))) { + brcmf_err("Action frame TX retry time over dwell time!\n"); + return true; + } + return false; +} /** * brcmf_p2p_send_action_frame() - send action frame . * @@ -1657,6 +1740,10 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, s32 tx_retry; s32 extra_listen_time; uint delta_ms; + unsigned long dwell_jiffies = 0; + bool dwell_overflow = false; + + s32 requested_dwell = af_params->dwell_time; action_frame = &af_params->action_frame; action_frame_len = le16_to_cpu(action_frame->len); @@ -1768,12 +1855,21 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, /* update channel */ af_params->channel = cpu_to_le32(afx_hdl->peer_chan); } + dwell_jiffies = jiffies; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); tx_retry = 0; while (!p2p->block_gon_req_tx && - (ack == false) && (tx_retry < P2P_AF_TX_MAX_RETRY)) { + (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) && + !dwell_overflow) { + if (af_params->channel) + msleep(P2P_AF_RETRY_DELAY_TIME); + ack = !brcmf_p2p_tx_action_frame(p2p, af_params); tx_retry++; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); } if (ack == false) { brcmf_err("Failed to send Action Frame(retry %d)\n", tx_retry); @@ -1986,7 +2082,7 @@ int brcmf_p2p_ifchange(struct brcmf_cfg80211_info *cfg, if_request.type = cpu_to_le16((u16)if_type); if_request.chspec = cpu_to_le16(chanspec); - memcpy(if_request.addr, p2p->int_addr, sizeof(if_request.addr)); + memcpy(if_request.addr, p2p->conn_int_addr, sizeof(if_request.addr)); brcmf_cfg80211_arm_vif_event(cfg, vif); err = brcmf_fil_iovar_data_set(vif->ifp, "p2p_ifupd", &if_request, @@ -2132,6 +2228,27 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, return ERR_PTR(err); } +int brcmf_p2p_get_conn_idx(struct brcmf_cfg80211_info *cfg) +{ + int i; + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + + if (!ifp) + return -ENODEV; + + for (i = P2PAPI_BSSCFG_CONNECTION; i < P2PAPI_BSSCFG_MAX; i++) { + if (!cfg->p2p.bss_idx[i].vif) { + if (i == P2PAPI_BSSCFG_CONNECTION2 && + !(brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + brcmf_err("Multi p2p not supported"); + return -EIO; + } + return i; + } + } + return -EIO; +} + /** * brcmf_p2p_add_vif() - create a new P2P virtual interface. * @@ -2150,7 +2267,9 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct brcmf_cfg80211_vif *vif; enum brcmf_fil_p2p_if_types iftype; - int err; + int err = 0; + int connidx; + u8 *p2p_intf_addr; if (brcmf_cfg80211_vif_event_armed(cfg)) return ERR_PTR(-EBUSY); @@ -2176,9 +2295,21 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, return (struct wireless_dev *)vif; brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, cfg->p2p.int_addr, - iftype); + connidx = brcmf_p2p_get_conn_idx(cfg); + + if (connidx == P2PAPI_BSSCFG_CONNECTION) + p2p_intf_addr = cfg->p2p.conn_int_addr; + else if (connidx == P2PAPI_BSSCFG_CONNECTION2) + p2p_intf_addr = cfg->p2p.conn2_int_addr; + else + err = -EINVAL; + + if (!err) + err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, + p2p_intf_addr, iftype); + if (err) { + brcmf_err("request p2p interface failed\n"); brcmf_cfg80211_arm_vif_event(cfg, NULL); goto fail; } @@ -2210,7 +2341,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, goto fail; } - cfg->p2p.bss_idx[P2PAPI_BSSCFG_CONNECTION].vif = vif; + cfg->p2p.bss_idx[connidx].vif = vif; /* Disable firmware roaming for P2P interface */ brcmf_fil_iovar_int_set(ifp, "roam_off", 1); if (iftype == BRCMF_FIL_P2P_IF_GO) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index 0e8b34d2d85cb..fe83962eead30 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -25,13 +25,15 @@ struct brcmf_cfg80211_info; * * @P2PAPI_BSSCFG_PRIMARY: maps to driver's primary bsscfg. * @P2PAPI_BSSCFG_DEVICE: maps to driver's P2P device discovery bsscfg. - * @P2PAPI_BSSCFG_CONNECTION: maps to driver's P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION: maps to driver's 1st P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION2: maps to driver's 2nd P2P connection bsscfg. * @P2PAPI_BSSCFG_MAX: used for range checking. */ enum p2p_bss_type { P2PAPI_BSSCFG_PRIMARY, /* maps to driver's primary bsscfg */ P2PAPI_BSSCFG_DEVICE, /* maps to driver's P2P device discovery bsscfg */ - P2PAPI_BSSCFG_CONNECTION, /* maps to driver's P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION, /* driver's 1st P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION2, /* driver's 2nd P2P connection bsscfg */ P2PAPI_BSSCFG_MAX }; @@ -124,12 +126,14 @@ struct afx_hdl { * @gon_req_action: about to send go negotiation requets frame. * @block_gon_req_tx: drop tx go negotiation requets frame. * @p2pdev_dynamically: is p2p device if created by module param or supplicant. + * @wait_for_offchan_complete: wait for off-channel tx completion event. */ struct brcmf_p2p_info { struct brcmf_cfg80211_info *cfg; unsigned long status; u8 dev_addr[ETH_ALEN]; - u8 int_addr[ETH_ALEN]; + u8 conn_int_addr[ETH_ALEN]; + u8 conn2_int_addr[ETH_ALEN]; struct p2p_bss bss_idx[P2PAPI_BSSCFG_MAX]; struct timer_list listen_timer; u8 listen_channel; @@ -144,6 +148,7 @@ struct brcmf_p2p_info { bool gon_req_action; bool block_gon_req_tx; bool p2pdev_dynamically; + bool wait_for_offchan_complete; }; s32 brcmf_p2p_attach(struct brcmf_cfg80211_info *cfg, bool p2pdev_forced); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index e6e9b00b79d71..d6c82743a94de 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -39,6 +39,7 @@ #include "chip.h" #include "core.h" #include "common.h" +#include "cfg80211.h" enum brcmf_pcie_state { @@ -58,6 +59,7 @@ BRCMF_FW_NVRAM_DEF(4365C, "brcmfmac4365c-pcie.bin", "brcmfmac4365c-pcie.txt"); BRCMF_FW_NVRAM_DEF(4366B, "brcmfmac4366b-pcie.bin", "brcmfmac4366b-pcie.txt"); BRCMF_FW_NVRAM_DEF(4366C, "brcmfmac4366c-pcie.bin", "brcmfmac4366c-pcie.txt"); BRCMF_FW_NVRAM_DEF(4371, "brcmfmac4371-pcie.bin", "brcmfmac4371-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4355, "brcmfmac89459-pcie.bin", "brcmfmac89459-pcie.txt"); static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -76,9 +78,10 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), + BRCMF_FW_NVRAM_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355), }; -#define BRCMF_PCIE_FW_UP_TIMEOUT 2000 /* msec */ +#define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ #define BRCMF_PCIE_REG_MAP_SIZE (32 * 1024) @@ -133,9 +136,11 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_PCIE_MB_INT_D2H2_DB1 | \ BRCMF_PCIE_MB_INT_D2H3_DB0 | \ BRCMF_PCIE_MB_INT_D2H3_DB1) - -#define BRCMF_PCIE_MIN_SHARED_VERSION 5 -#define BRCMF_PCIE_MAX_SHARED_VERSION 6 +#define BRCMF_PCIE_SHARED_VERSION_5 5 +#define BRCMF_PCIE_SHARED_VERSION_6 6 +#define BRCMF_PCIE_SHARED_VERSION_7 7 +#define BRCMF_PCIE_MIN_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_5 +#define BRCMF_PCIE_MAX_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_7 #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 @@ -178,6 +183,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_D2H_DEV_D3_ACK 0x00000001 #define BRCMF_D2H_DEV_DS_ENTER_REQ 0x00000002 #define BRCMF_D2H_DEV_DS_EXIT_NOTE 0x00000004 +#define BRCMF_D2H_DEV_FWHALT 0x10000000 #define BRCMF_H2D_HOST_D3_INFORM 0x00000001 #define BRCMF_H2D_HOST_DS_ACK 0x00000002 @@ -265,6 +271,9 @@ struct brcmf_pciedev_info { void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value); struct brcmf_mp_device *settings; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + ulong bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ }; struct brcmf_pcie_ringbuf { @@ -315,6 +324,14 @@ static const u32 brcmf_ring_max_item[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM }; +static const u32 brcmf_ring_itemsize_pre_v7[BRCMF_NROF_COMMON_MSGRINGS] = { + BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, + BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, + BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE, + BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7, + BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 +}; + static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, @@ -323,6 +340,13 @@ static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE }; +static void brcmf_pcie_setup(struct device *dev, int ret, + const struct firmware *fw, + void *nvram, u32 nvram_len); + +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ +DEFINE_RAW_SPINLOCK(pcie_lock); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ static u32 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) @@ -347,8 +371,24 @@ static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + u8 value; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread8(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread8(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -356,8 +396,24 @@ static u16 brcmf_pcie_read_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u16 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread16(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread16(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -366,8 +422,22 @@ brcmf_pcie_write_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } iowrite16(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite16(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -394,8 +464,24 @@ static u32 brcmf_pcie_read_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -404,17 +490,47 @@ brcmf_pcie_write_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { void __iomem *address = devinfo->tcm + mem_offset; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } static u32 brcmf_pcie_read_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); - return (ioread32(addr)); + return value; +#else + return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -422,9 +538,23 @@ static void brcmf_pcie_write_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; - iowrite32(value, addr); + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -436,12 +566,30 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *src32; __le16 *src16; u8 *src8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)srcaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)srcaddr & 2) || (len & 2)) { src8 = (u8 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite8(*src8, address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; src8++; len--; @@ -450,7 +598,22 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; src16 = (__le16 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite16(le16_to_cpu(*src16), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; src16++; len--; @@ -460,12 +623,29 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; src32 = (__le32 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite32(le32_to_cpu(*src32), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; src32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -477,12 +657,30 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *dst32; __le16 *dst16; u8 *dst8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)dstaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)dstaddr & 2) || (len & 2)) { dst8 = (u8 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst8 = ioread8(address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; dst8++; len--; @@ -491,7 +689,22 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; dst16 = (__le16 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst16 = cpu_to_le16(ioread16(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; dst16++; len--; @@ -501,12 +714,29 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; dst32 = (__le32 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst32 = cpu_to_le32(ioread32(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; dst32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -549,6 +779,9 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) BRCMF_PCIE_CFGREG_MSI_ADDR_L, BRCMF_PCIE_CFGREG_MSI_ADDR_H, BRCMF_PCIE_CFGREG_MSI_DATA, +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + BCMA_PCI_BAR1_WIN, +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ BRCMF_PCIE_CFGREG_LINK_STATUS_CTRL2, BRCMF_PCIE_CFGREG_RBAR_CTRL, BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1, @@ -671,7 +904,6 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) brcmf_pcie_write_tcm32(devinfo, addr, htod_mb_data); pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1); - pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1); return 0; } @@ -705,6 +937,10 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) devinfo->mbdata_completed = true; wake_up(&devinfo->mbdata_resp_wait); } + if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_fw_crashed(&devinfo->pdev->dev); + } } @@ -998,8 +1234,14 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, struct brcmf_pcie_ringbuf *ring; u32 size; u32 addr; + const u32 *ring_itemsize_array; - size = brcmf_ring_max_item[ring_id] * brcmf_ring_itemsize[ring_id]; + if (devinfo->shared.version < BRCMF_PCIE_SHARED_VERSION_7) + ring_itemsize_array = brcmf_ring_itemsize_pre_v7; + else + ring_itemsize_array = brcmf_ring_itemsize; + + size = brcmf_ring_max_item[ring_id] * ring_itemsize_array[ring_id]; dma_buf = brcmf_pcie_init_dmabuffer_for_device(devinfo, size, tcm_ring_phys_addr + BRCMF_RING_MEM_BASE_ADDR_OFFSET, &dma_handle); @@ -1009,7 +1251,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, addr = tcm_ring_phys_addr + BRCMF_RING_MAX_ITEM_OFFSET; brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_max_item[ring_id]); addr = tcm_ring_phys_addr + BRCMF_RING_LEN_ITEMS_OFFSET; - brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_itemsize[ring_id]); + brcmf_pcie_write_tcm16(devinfo, addr, ring_itemsize_array[ring_id]); ring = kzalloc(sizeof(*ring), GFP_KERNEL); if (!ring) { @@ -1018,7 +1260,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, return NULL; } brcmf_commonring_config(&ring->commonring, brcmf_ring_max_item[ring_id], - brcmf_ring_itemsize[ring_id], dma_buf); + ring_itemsize_array[ring_id], dma_buf); ring->dma_handle = dma_handle; ring->devinfo = devinfo; brcmf_commonring_register_cb(&ring->commonring, @@ -1088,9 +1330,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) u16 max_flowrings; u16 max_submissionrings; u16 max_completionrings; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_dev_tomem(devinfo, devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_fromio(&ringinfo, devinfo->tcm + devinfo->shared.ring_info_addr, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ + if (devinfo->shared.version >= 6) { max_submissionrings = le16_to_cpu(ringinfo.max_submissionrings); max_flowrings = le16_to_cpu(ringinfo.max_flowrings); @@ -1159,8 +1406,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) ringinfo.d2h_r_idx_hostaddr.high_addr = cpu_to_le32(address >> 32); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_mem_todev(devinfo, + devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_toio(devinfo->tcm + devinfo->shared.ring_info_addr, &ringinfo, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ brcmf_dbg(PCIE, "Using host memory indices\n"); } @@ -1350,6 +1603,64 @@ static int brcmf_pcie_get_memdump(struct device *dev, void *data, size_t len) return 0; } +static int brcmf_pcie_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; + int ret = 0; + + if (devinfo->fw_name[0] != '\0') + strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN); + else + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_pcie_fwnames, + ARRAY_SIZE(brcmf_pcie_fwnames), + fw_name, NULL); + + return ret; +} + +static int brcmf_pcie_reset(struct device *dev) +{ + struct brcmf_bus *bus; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; + int err; + u16 domain_nr; + u16 bus_nr; + + domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1; + bus_nr = devinfo->pdev->bus->number; + brcmf_dbg(PCIE, "Enter %x:%x (%d/%d)\n", devinfo->pdev->vendor, + devinfo->pdev->device, domain_nr, bus_nr); + bus = dev_get_drvdata(dev); + + brcmf_detach(dev); + + brcmf_pcie_release_irq(devinfo); + brcmf_pcie_release_scratchbuffers(devinfo); + brcmf_pcie_release_ringbuffers(devinfo); + brcmf_pcie_reset_device(devinfo); + + err = brcmf_fw_map_chip_to_name(devinfo->ci->chip, + devinfo->ci->chiprev, + brcmf_pcie_fwnames, + ARRAY_SIZE(brcmf_pcie_fwnames), + devinfo->fw_name, devinfo->nvram_name); + if (err) + return err; + + err = brcmf_fw_get_firmwares_pcie(bus->dev, BRCMF_FW_REQUEST_NVRAM | + BRCMF_FW_REQ_NV_OPTIONAL, + devinfo->fw_name, devinfo->nvram_name, + brcmf_pcie_setup, domain_nr, bus_nr); + if (err == 0) + return 0; + return err; +} static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .txdata = brcmf_pcie_tx, @@ -1359,6 +1670,8 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .wowl_config = brcmf_pcie_wowl_config, .get_ramsize = brcmf_pcie_get_ramsize, .get_memdump = brcmf_pcie_get_memdump, + .get_fwname = brcmf_pcie_get_fwname, + .reset = brcmf_pcie_reset, }; @@ -1535,6 +1848,9 @@ static int brcmf_pcie_get_resource(struct brcmf_pciedev_info *devinfo) devinfo->regs = ioremap_nocache(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE); devinfo->tcm = ioremap_nocache(bar1_addr, bar1_size); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + devinfo->bar1_size = bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (!devinfo->regs || !devinfo->tcm) { brcmf_err("ioremap() failed (%p,%p)\n", devinfo->regs, @@ -1869,11 +2185,22 @@ static int brcmf_pcie_pm_enter_D3(struct device *dev) { struct brcmf_pciedev_info *devinfo; struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(PCIE, "Enter\n"); bus = dev_get_drvdata(dev); devinfo = bus->bus_priv.pcie->devinfo; + config = bus->drvr->config; + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); brcmf_bus_change_state(bus, BRCMF_BUS_DOWN); @@ -1953,9 +2280,12 @@ static const struct dev_pm_ops brcmf_pciedrvr_pm = { static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4354_RAW_DEVICE_ID), + BRCMF_PCIE_DEVICE_SUB(BRCM_PCIE_4355_DEVICE_ID,BRCM_PCIE_VENDOR_ID_BROADCOM,BRCM_PCIE_4355_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_RAW_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4358_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4359_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_DEVICE_ID), @@ -1970,6 +2300,8 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index ffa243e2e2d0f..55974a43796aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -496,6 +496,11 @@ int brcmf_pno_stop_sched_scan(struct brcmf_if *ifp, u64 reqid) brcmf_dbg(TRACE, "reqid=%llu\n", reqid); pi = ifp_to_pno(ifp); + + /* No PNO reqeuset */ + if (!pi->n_reqs) + return 0; + err = brcmf_pno_remove_request(pi, reqid); if (err) return err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 4c28b04ea6053..5c05a9d2f99fd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -45,10 +46,28 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "fwil.h" #define DCMD_RESP_TIMEOUT msecs_to_jiffies(2500) #define CTL_DONE_TIMEOUT msecs_to_jiffies(2500) - +#define ULP_HUDI_PROC_DONE_TIME msecs_to_jiffies(2500) + +#define DEFAULT_F2_WATERMARK 0x8 +#define CY_4373_F2_WATERMARK 0x40 +#define CY_4373_F1_MESBUSYCTRL (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_43012_F2_WATERMARK 0x60 +#define CY_43012_MES_WATERMARK 0x50 +#define CY_43012_MESBUSYCTRL (CY_43012_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) +#define CY_43455_F2_WATERMARK 0x60 +#define CY_43455_MES_WATERMARK 0x50 +#define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) +#define CY_4339_F2_WATERMARK 48 +#define CY_4339_MES_WATERMARK 80 +#define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_435X_F2_WATERMARK 0x40 +#define CY_435X_F1_MESBUSYCTRL (CY_435X_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -138,6 +157,8 @@ struct rte_console { /* 1: isolate internal sdio signals, put external pads in tri-state; requires * sdio bus power cycle to clear (rev 9) */ #define SBSDIO_DEVCTL_PADS_ISO 0x08 +/* 1: enable F2 Watermark */ +#define SBSDIO_DEVCTL_F2WM_ENAB 0x10 /* Force SD->SB reset mapping (rev 11) */ #define SBSDIO_DEVCTL_SB_RST_CTL 0x30 /* Determined by CoreControl bit */ @@ -260,10 +281,11 @@ struct rte_console { #define I_HMB_HOST_INT I_HMB_SW3 /* Miscellaneous Interrupt */ /* tohostmailboxdata */ -#define HMB_DATA_NAKHANDLED 1 /* retransmit NAK'd frame */ -#define HMB_DATA_DEVREADY 2 /* talk to host after enable */ -#define HMB_DATA_FC 4 /* per prio flowcontrol update flag */ -#define HMB_DATA_FWREADY 8 /* fw ready for protocol activity */ +#define HMB_DATA_NAKHANDLED 0x0001 /* retransmit NAK'd frame */ +#define HMB_DATA_DEVREADY 0x0002 /* talk to host after enable */ +#define HMB_DATA_FC 0x0004 /* per prio flowcontrol update flag */ +#define HMB_DATA_FWREADY 0x0008 /* fw ready for protocol activity */ +#define HMB_DATA_FWHALT 0x0010 /* firmware halted */ #define HMB_DATA_FCDATA_MASK 0xff000000 #define HMB_DATA_FCDATA_SHIFT 24 @@ -316,14 +338,13 @@ struct rte_console { #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US) #define BRCMF_SDIO_MAX_ACCESS_ERRORS 5 -/* - * Conversion of 802.1D priority to precedence level - */ -static uint prio2prec(u32 prio) -{ - return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? - (prio^2) : prio; -} +static void brcmf_sdio_firmware_callback(struct device *dev, int err, + const struct firmware *code, + void *nvram, u32 nvram_len); +static int brcmf_sdio_f2_ready(struct brcmf_sdio *bus); +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data); #ifdef DEBUG /* Device console log buffer state */ @@ -618,7 +639,9 @@ BRCMF_FW_NVRAM_DEF(43430A1, "brcmfmac43430-sdio.bin", "brcmfmac43430-sdio.txt"); BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt"); BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt"); BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-sdio.bin", "brcmfmac4359-sdio.txt"); BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43012, "brcmfmac43012-sdio.bin", "brcmfmac43012-sdio.txt"); static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143), @@ -638,9 +661,13 @@ static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), - BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373) + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), + BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373), + BRCMF_FW_NVRAM_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; +#define TXCTL_CREDITS 2 + static void pkt_align(struct sk_buff *p, int len, int align) { uint datalign; @@ -651,8 +678,17 @@ static void pkt_align(struct sk_buff *p, int len, int align) __skb_trim(p, len); } -/* To check if there's window offered */ +/* To check if there's window offered + * Reserve 3 credits for txctl + */ static bool data_ok(struct brcmf_sdio *bus) +{ + return (u8)(bus->tx_max - bus->tx_seq) > TXCTL_CREDITS && + ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; +} + +/* To check if there's window offered */ +static bool txctl_ok(struct brcmf_sdio *bus) { return (u8)(bus->tx_max - bus->tx_seq) != 0 && ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; @@ -699,6 +735,15 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err); + /* In case of 43012 chip, the chip could go down immediately after + * KSO bit is cleared. So the further reads of KSO register could + * fail. Thereby just bailing out immediately after clearing KSO + * bit, to avoid polling of KSO bit. + */ + if (!on && (bus->ci->chip == CY_CC_43012_CHIP_ID)) { + return err; + } + if (on) { /* device WAKEUP through KSO: * write bit 0 & read back until @@ -1076,7 +1121,7 @@ static void brcmf_sdio_get_console_addr(struct brcmf_sdio *bus) } #endif /* DEBUG */ -static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) +static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) { u32 intstatus = 0; u32 hmb_data; @@ -1094,6 +1139,12 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) offsetof(struct sdpcmd_regs, tosbmailbox)); bus->sdcnt.f1regdata += 2; + /* dongle indicates the firmware has halted/crashed */ + if (hmb_data & HMB_DATA_FWHALT) { + brcmf_dbg(SDIO, "mailbox indicates firmware halted\n"); + brcmf_fw_crashed(bus->sdiodev->dev); + } + /* Dongle recomposed rx frames, accept them again */ if (hmb_data & HMB_DATA_NAKHANDLED) { brcmf_dbg(SDIO, "Dongle reports NAK handled, expect rtx of %d\n", @@ -1151,9 +1202,13 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) HMB_DATA_NAKHANDLED | HMB_DATA_FC | HMB_DATA_FWREADY | + HMB_DATA_FWHALT | HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK)) brcmf_err("Unknown mailbox data content: 0x%02x\n", hmb_data); + /* Populate hmb_data if argument is passed for DS1 check later */ + if (hmbd) + *hmbd = hmb_data; return intstatus; } @@ -2313,6 +2368,7 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) &prec_out); if (pkt == NULL) break; + skb_orphan(pkt); __skb_queue_tail(&pktq, pkt); } spin_unlock_bh(&bus->txq_lock); @@ -2442,9 +2498,20 @@ static void brcmf_sdio_bus_stop(struct device *dev) /* Force backplane clocks to assure F2 interrupt propagates */ saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); - if (!err) - brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + if (!err) { + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | + SBSDIO_HT_AVAIL_REQ), + &err); + } else { + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), + &err); + } + } if (err) brcmf_err("Failed to force clock for F2: err %d\n", err); @@ -2521,6 +2588,175 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) return ret; } +/* This Function is used to retrieve important + * details from dongle related to ULP mode Mostly + * values/SHM details that will be vary depending + * on the firmware branches + */ +static void +brcmf_sdio_ulp_preinit(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + struct brcmf_if *ifp = bus_if->drvr->iflist[0]; + + brcmf_dbg(ULP, "Enter\n"); + + /* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */ + brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", + &sdiodev->fmac_ulp.ulp_shm_offset, + sizeof(sdiodev->fmac_ulp.ulp_shm_offset)); + + sdiodev->ulp = false; + + brcmf_dbg(ULP, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", + M_DS1_CTRL_SDIO(sdiodev->fmac_ulp), + M_WAKEEVENT_IND(sdiodev->fmac_ulp)); + brcmf_dbg(ULP, "m_ulp_wakeind [%x]\n", + M_ULP_WAKE_IND(sdiodev->fmac_ulp)); +} + +/* Reinitialize ARM because In DS1 mode ARM got off */ +static int +brcmf_sdio_ulp_reinit_fw(struct brcmf_sdio *bus) +{ + struct brcmf_sdio_dev *sdiodev = bus->sdiodev; + int err = 0; + + /* After firmware redownload tx/rx seq are reset accordingly + * these values are reset on FMAC side tx_max is initially set to 4, + * which later is updated by FW. + */ + bus->tx_seq = 0; + bus->rx_seq = 0; + bus->tx_max = 4; + + err = brcmf_fw_get_firmwares(sdiodev->dev, BRCMF_FW_REQUEST_NVRAM, + sdiodev->fw_name, sdiodev->nvram_name, + brcmf_sdio_firmware_callback); + if (err != 0) + brcmf_err("async firmware request failed: %d\n", err); + return err; +} + +/* Check if device is in DS1 mode and handshake with ULP UCODE */ +static bool +brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus, u32 hmb_data) +{ + int err = 0; + u32 value = 0; + u32 val32, ulp_wake_ind, wowl_wake_ind; + int reg_addr; + unsigned long timeout; + struct brcmf_ulp *fmac_ulp = &bus->sdiodev->fmac_ulp; + int i = 0; + + /* If any host mail box data is present, ignore DS1 exit sequence */ + if (hmb_data) + return false; + /* Skip if DS1 Exit is already in progress + * This can happen if firmware download is taking more time + */ + if (fmac_ulp->ulp_state == FMAC_ULP_TRIGGERED) + return false; + + value = brcmf_sdiod_regrb(bus->sdiodev, SDIO_CCCR_IOEx, &err); + + if (value == SDIO_FUNC_ENABLE_1) { + brcmf_dbg(ULP, "GOT THE INTERRUPT FROM UCODE\n"); + bus->sdiodev->ulp = true; + fmac_ulp->ulp_state = FMAC_ULP_TRIGGERED; + ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->fmac_ulp), &err); + wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->fmac_ulp), &err); + + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x state %s\n", + wowl_wake_ind, ulp_wake_ind, (fmac_ulp->ulp_state) ? + "DS1 Exit Triggered" : "IDLE State"); + + if (wowl_wake_ind || ulp_wake_ind) { + /* RX wake Don't do anything. + * Just bail out and re-download firmware. + */ + /* Print out PHY TX error block when bit 9 set */ + if ((ulp_wake_ind & C_DS1_PHY_TXERR) && + M_DS1_PHYTX_ERR_BLK(bus->sdiodev->fmac_ulp)) { + brcmf_err("Dump PHY TX Error SHM Locations\n"); + for (i = 0; i < PHYTX_ERR_BLK_SIZE; i++) { + pr_err("0x%x", D11SHM_RDW(bus->sdiodev, + (M_DS1_PHYTX_ERR_BLK + (bus->sdiodev->fmac_ulp) + + (i * 2)), &err)); + } + brcmf_err("\n"); + } + } else { + /* TX wake negotiate with MAC */ + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO(bus->sdiodev->fmac_ulp), + &err)); + val32 = D11SHM_RD(bus->sdiodev, M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); + D11SHM_WR(bus->sdiodev, M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), val32, + (C_DS1_CTRL_SDIO_DS1_EXIT | + C_DS1_CTRL_REQ_VALID), + &err); + val32 = D11REG_RD(bus->sdiodev, + D11_MACCONTROL_REG, &err); + val32 = val32 | D11_MACCONTROL_REG_WAKE; + D11REG_WR(bus->sdiodev, + D11_MACCONTROL_REG, val32, &err); + + /* Poll for PROC_DONE to be set by ucode */ + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); + /* Wait here (polling) for C_DS1_CTRL_PROC_DONE */ + timeout = jiffies + ULP_HUDI_PROC_DONE_TIME; + while (!(value & C_DS1_CTRL_PROC_DONE)) { + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO + (bus->sdiodev->fmac_ulp), + &err); + if (time_after(jiffies, timeout)) + break; + usleep_range(1000, 2000); + } + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err)); + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); + if (!(value & C_DS1_CTRL_PROC_DONE)) { + brcmf_err("%s: timeout Failed to enter DS1 Exit state!\n", + __func__); + return false; + } + } + ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->fmac_ulp), &err); + wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->fmac_ulp), &err); + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", + wowl_wake_ind, ulp_wake_ind); + reg_addr = CORE_CC_REG( + brcmf_chip_get_pmu(bus->ci)->base, min_res_mask); + brcmf_sdiod_regwl(bus->sdiodev, reg_addr, + DEFAULT_43012_MIN_RES_MASK, &err); + if (err) + brcmf_err("min_res_mask failed\n"); + + return true; + } + + return false; +} + static void brcmf_sdio_dpc(struct brcmf_sdio *bus) { u32 newstatus = 0; @@ -2591,8 +2827,11 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) /* Handle host mailbox indication */ if (intstatus & I_HMB_HOST_INT) { + u32 hmb_data = 0; intstatus &= ~I_HMB_HOST_INT; - intstatus |= brcmf_sdio_hostmail(bus); + intstatus |= brcmf_sdio_hostmail(bus, &hmb_data); + if (brcmf_sdio_ulp_pre_redownload_check(bus, hmb_data)) + brcmf_sdio_ulp_reinit_fw(bus); } sdio_release_host(bus->sdiodev->func[1]); @@ -2637,14 +2876,18 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_sdio_clrintr(bus); if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && - data_ok(bus)) { + txctl_ok(bus) && brcmf_sdio_f2_ready(bus)) { sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { - err = brcmf_sdio_tx_ctrlframe(bus, bus->ctrl_frame_buf, + err = brcmf_sdio_tx_ctrlframe(bus, + bus->ctrl_frame_buf, bus->ctrl_frame_len); bus->ctrl_frame_err = err; - wmb(); + wmb(); /* drain writebuffer */ bus->ctrl_frame_stat = false; + if (err) + brcmf_err("sdio ctrlframe tx failed err=%d\n", + err); } sdio_release_host(bus->sdiodev->func[1]); brcmf_sdio_wait_event_wakeup(bus); @@ -2747,7 +2990,13 @@ static int brcmf_sdio_bus_txdata(struct device *dev, struct sk_buff *pkt) skb_push(pkt, bus->tx_hdrlen); /* precondition: IS_ALIGNED((unsigned long)(pkt->data), 2) */ - prec = prio2prec((pkt->priority & PRIOMASK)); + /* In WLAN, priority is always set by the AP using WMM parameters + * and this need not always follow the standard 802.1d priority. + * Based on AP WMM config, map from 802.1d priority to corresponding + * precedence level. + */ + prec = brcmf_map_prio_to_prec(bus_if->drvr->config, + (pkt->priority & PRIOMASK)); /* Check for existing queue, current flow-control, pending event, or pending clock */ @@ -3338,34 +3587,52 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) { int err = 0; u8 val; + u8 wakeupctrl; + u8 cardcap; + u8 chipclkcsr; brcmf_dbg(TRACE, "Enter\n"); + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + wakeupctrl = SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT; + chipclkcsr = SBSDIO_HT_AVAIL_REQ; + } else { + wakeupctrl = SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; + chipclkcsr = SBSDIO_FORCE_HT; + } + + if (bus->ci->chip == CY_CC_43012_CHIP_ID || + bus->ci->chip == CY_CC_4373_CHIP_ID || + bus->ci->chip == BRCM_CC_4339_CHIP_ID || + bus->ci->chip == BRCM_CC_4354_CHIP_ID || + bus->ci->chip == BRCM_CC_4356_CHIP_ID || + bus->ci->chip == BRCM_CC_4345_CHIP_ID) { + cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; + } else { + cardcap = (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | + SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT); + } + val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err); if (err) { brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } - - val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; + val |= 1 << wakeupctrl; brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } - - /* Add CMD14 Support */ brcmf_sdiod_regwb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP, - (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | - SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT), + cardcap, &err); if (err) { brcmf_err("error writing SDIO_CCCR_BRCM_CARDCAP\n"); return; } - brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - SBSDIO_FORCE_HT, &err); + chipclkcsr, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_CHIPCLKCSR\n"); return; @@ -3439,6 +3706,10 @@ static int brcmf_sdio_bus_preinit(struct device *dev) if (err < 0) goto done; + /* initialize SHM address from firmware for DS1 */ + if (!bus->sdiodev->ulp) + brcmf_sdio_ulp_preinit(dev); + bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN; if (sdiodev->sg_support) { bus->txglom = false; @@ -3608,7 +3879,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) if (bus->idlecount > bus->idletime) { brcmf_dbg(SDIO, "idle\n"); sdio_claim_host(bus->sdiodev->func[1]); - brcmf_sdio_wd_timer(bus, false); +#ifdef DEBUG + if (!BRCMF_FWCON_ON() || + bus->console_interval == 0) +#endif + brcmf_sdio_wd_timer(bus, false); bus->idlecount = 0; brcmf_sdio_bus_sleep(bus, true, false); sdio_release_host(bus->sdiodev->func[1]); @@ -3979,6 +4254,54 @@ brcmf_sdio_watchdog(unsigned long data) } } +static int brcmf_sdio_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + int ret = 0; + + if (sdiodev->fw_name[0] != '\0') + strlcpy(fw_name, sdiodev->fw_name, BRCMF_FW_NAME_LEN); + else + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_sdio_fwnames, + ARRAY_SIZE(brcmf_sdio_fwnames), + fw_name, NULL); + + return ret; +} + +static int brcmf_sdio_bus_reset(struct device *dev) +{ + int ret = 0; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + + brcmf_dbg(SDIO, "Enter\n"); + + /* start by unregistering irqs */ + brcmf_sdiod_intr_unregister(sdiodev); + + brcmf_sdiod_remove(sdiodev); + + /* reset the adapter */ + sdio_claim_host(sdiodev->func[1]); + mmc_hw_reset(sdiodev->func[1]->card->host); + sdio_release_host(sdiodev->func[1]); + + brcmf_bus_change_state(sdiodev->bus_if, BRCMF_BUS_DOWN); + + ret = brcmf_sdiod_probe(sdiodev); + if (ret) { + brcmf_err("Failed to probe after sdio device reset: ret %d\n", + ret); + brcmf_sdiod_remove(sdiodev); + } + + return ret; +} + static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .stop = brcmf_sdio_bus_stop, .preinit = brcmf_sdio_bus_preinit, @@ -3989,6 +4312,8 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .wowl_config = brcmf_sdio_wowl_config, .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, + .get_fwname = brcmf_sdio_get_fwname, + .reset = brcmf_sdio_bus_reset, }; static void brcmf_sdio_firmware_callback(struct device *dev, int err, @@ -3999,8 +4324,9 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, struct brcmf_sdio_dev *sdiodev; struct brcmf_sdio *bus; u8 saveclk; + u8 devctl; - brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err); + brcmf_dbg(ULP, "Enter: dev=%s, err=%d\n", dev_name(dev), err); bus_if = dev_get_drvdata(dev); sdiodev = bus_if->bus_priv.sdio; if (err) @@ -4032,8 +4358,14 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, /* Force clocks on backplane to be sure F2 interrupt propagates */ saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (!err) { - brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_HT_AVAIL_REQ), + &err); + } else { + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), &err); + } } if (err) { brcmf_err("Failed to force clock for F2: err %d\n", err); @@ -4054,8 +4386,78 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, bus->hostintmask = HOSTINTMASK; w_sdreg32(bus, bus->hostintmask, offsetof(struct sdpcmd_regs, hostintmask)); - - brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, 8, &err); + switch (sdiodev->func[0]->device) { + case SDIO_DEVICE_ID_CYPRESS_4373: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4373_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, CY_4373_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, CY_4373_F1_MESBUSYCTRL, &err); + break; + case SDIO_DEVICE_ID_CYPRESS_43012: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43012\n", + CY_43012_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, + SBSDIO_WATERMARK, + CY_43012_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, + SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, + SBSDIO_DEVICE_CTL, devctl, &err); + /* Register for ULP events */ + brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, + brcmf_ulp_event_notify); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43012_MESBUSYCTRL, &err); + break; + case SDIO_DEVICE_ID_BROADCOM_43455: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43455\n", + CY_43455_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, + CY_43455_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43455_MESBUSYCTRL, &err); + break; + case SDIO_DEVICE_ID_BROADCOM_4339: + brcmf_sdiod_regwb(sdiodev, + SBSDIO_WATERMARK, + CY_4339_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, + SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, + SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err); + break; + case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: + case SDIO_DEVICE_ID_BROADCOM_4354: + case SDIO_DEVICE_ID_BROADCOM_4356: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_435X_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, + CY_435X_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_435X_F1_MESBUSYCTRL, &err); + break; + default: + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); + break; + } } else { /* Disable F2 again */ sdio_disable_func(sdiodev->func[SDIO_FUNC_2]); @@ -4085,11 +4487,24 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, sdio_release_host(sdiodev->func[1]); - err = brcmf_bus_started(dev); - if (err != 0) { - brcmf_err("dongle is not responding\n"); - goto fail; + /* Waking up from deep sleep don't requirerd to reint the sdio bus + * as all sdiod core registers will get restored by Firmware using + * FCBS engine. + */ + if (!bus->sdiodev->ulp) { + err = brcmf_bus_started(dev); + if (err != 0) { + brcmf_err("dongle is not responding\n"); + goto fail; + } + } else { + /* For ULP, after firmware redownload complete + * set ULP state to IDLE + */ + if (bus->sdiodev->fmac_ulp.ulp_state == FMAC_ULP_TRIGGERED) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_IDLE; } + return; release: @@ -4121,9 +4536,21 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) bus->txminmax = BRCMF_TXMINMAX; bus->tx_seq = SDPCM_SEQ_WRAP - 1; + /* attempt to attach to the dongle */ + if (!(brcmf_sdio_probe_attach(bus))) { + brcmf_err("brcmf_sdio_probe_attach failed\n"); + goto fail; + } + /* single-threaded workqueue */ - wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, - dev_name(&sdiodev->func[1]->dev)); + if (sdiodev->settings->sdio_wq_highpri) { + wq = alloc_workqueue("brcmf_wq/%s", + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, + 1, dev_name(&sdiodev->func[1]->dev)); + } else { + wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, + dev_name(&sdiodev->func[1]->dev)); + } if (!wq) { brcmf_err("insufficient memory to create txworkqueue\n"); goto fail; @@ -4132,12 +4559,6 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) INIT_WORK(&bus->datawork, brcmf_sdio_dataworker); bus->brcmf_wq = wq; - /* attempt to attach to the dongle */ - if (!(brcmf_sdio_probe_attach(bus))) { - brcmf_err("brcmf_sdio_probe_attach failed\n"); - goto fail; - } - spin_lock_init(&bus->rxctl_lock); spin_lock_init(&bus->txq_lock); init_waitqueue_head(&bus->ctrl_wait); @@ -4271,7 +4692,17 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) * necessary cores. */ msleep(20); - brcmf_chip_set_passive(bus->ci); + if (bus->sdiodev->fmac_ulp.ulp_state == + FMAC_ULP_ENTRY_RECV) { + brcmf_chip_ulp_reset_lhl_regs(bus->ci); + brcmf_chip_reset_pmu_regs(bus->ci); + } else { + brcmf_chip_set_passive(bus->ci); + } + /* Reset the PMU, backplane and all the + * cores by using the PMUWatchdogCounter. + */ + brcmf_chip_reset_watchdog(bus->ci); brcmf_sdio_clkctl(bus, CLK_NONE, false); sdio_release_host(bus->sdiodev->func[1]); } @@ -4327,3 +4758,41 @@ int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep) return ret; } +/* Check F2 Ready bit before sending data to Firmware */ +static int +brcmf_sdio_f2_ready(struct brcmf_sdio *bus) +{ + int ret = -1; + int iordy_status = 0; + + sdio_claim_host(bus->sdiodev->func[1]); + /* Read the status of IOR2 */ + iordy_status = brcmf_sdiod_regrb(bus->sdiodev, + SDIO_CCCR_IORx, + NULL); + + sdio_release_host(bus->sdiodev->func[1]); + ret = iordy_status & SDIO_FUNC_ENABLE_2; + return ret; +} + +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data) +{ + int err = 0; + struct brcmf_bus *bus_if = ifp->drvr->bus_if; + struct brcmf_sdio_dev *sdiodev; + struct brcmf_sdio *bus; + struct brcmf_ulp_event *ulp_event = (struct brcmf_ulp_event *)data; + + sdiodev = bus_if->bus_priv.sdio; + bus = sdiodev->bus; + + brcmf_dbg(ULP, "Chip went to DS1 state : action %d\n", + ulp_event->ulp_dongle_action); + if (ulp_event->ulp_dongle_action == FMAC_ULP_ENTRY) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_ENTRY_RECV; + + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index f3da32fc6360b..f73a0058187d6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -107,6 +107,9 @@ #define SBSDIO_FUNC1_RFRAMEBCHI 0x1001C /* MesBusyCtl (rev 11) */ #define SBSDIO_FUNC1_MESBUSYCTRL 0x1001D +/* Enable busy capability for MES access */ +#define SBSDIO_MESBUSYCTRL_ENAB 0x80 + /* Sdio Core Rev 12 */ #define SBSDIO_FUNC1_WAKEUPCTRL 0x1001E #define SBSDIO_FUNC1_WCTRL_ALPWAIT_MASK 0x1 @@ -177,6 +180,35 @@ struct brcmf_sdreg { struct brcmf_sdio; struct brcmf_sdiod_freezer; +/* ULP SHM Offsets info */ +struct ulp_shm_info { + u32 m_ulp_ctrl_sdio; + u32 m_ulp_wakeevt_ind; + u32 m_ulp_wakeind; + u32 m_ulp_phytxblk; +}; + +/* FMAC ULP state machine */ +#define FMAC_ULP_IDLE (0) +#define FMAC_ULP_ENTRY_RECV (1) +#define FMAC_ULP_TRIGGERED (2) + +/* BRCMF_E_ULP event data */ +#define FMAC_ULP_EVENT_VERSION 1 +#define FMAC_ULP_DISABLE_CONSOLE 1 /* Disable console */ +#define FMAC_ULP_UCODE_DOWNLOAD 2 /* Download ULP ucode file */ +#define FMAC_ULP_ENTRY 3 /* Inform ulp entry to Host */ + +struct brcmf_ulp { + uint ulp_state; + struct ulp_shm_info ulp_shm_offset; +}; + +struct brcmf_ulp_event { + u16 version; + u16 ulp_dongle_action; +}; + struct brcmf_sdio_dev { struct sdio_func *func[SDIO_MAX_FUNCS]; u8 num_funcs; /* Supported funcs on client */ @@ -201,6 +233,8 @@ struct brcmf_sdio_dev { bool wowl_enabled; enum brcmf_sdiod_state state; struct brcmf_sdiod_freezer *freezer; + struct brcmf_ulp fmac_ulp; + bool ulp; }; /* sdio core registers */ @@ -367,6 +401,9 @@ static inline void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); + struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus); @@ -376,4 +413,83 @@ void brcmf_sdio_wowl_config(struct device *dev, bool enabled); int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep); void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); +/* SHM offsets */ +#define M_DS1_CTRL_SDIO(ptr) ((ptr).ulp_shm_offset.m_ulp_ctrl_sdio) +#define M_WAKEEVENT_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeevt_ind) +#define M_ULP_WAKE_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeind) +#define M_DS1_PHYTX_ERR_BLK(ptr) ((ptr).ulp_shm_offset.m_ulp_phytxblk) + +#define D11_BASE_ADDR 0x18001000 +#define D11_AXI_BASE_ADDR 0xE8000000 +#define D11_SHM_BASE_ADDR (D11_AXI_BASE_ADDR + 0x4000) + +#define D11REG_ADDR(offset) (D11_BASE_ADDR + (offset)) +#define D11IHR_ADDR(offset) (D11_AXI_BASE_ADDR + 0x400 + (2 * (offset))) +#define D11SHM_ADDR(offset) (D11_SHM_BASE_ADDR + (offset)) + +/* MacControl register */ +#define D11_MACCONTROL_REG D11REG_ADDR(0x120) +#define D11_MACCONTROL_REG_WAKE 0x4000000 + +/* HUDI Sequence SHM bits */ +#define C_DS1_CTRL_SDIO_DS1_SLEEP 0x1 +#define C_DS1_CTRL_SDIO_MAC_ON 0x2 +#define C_DS1_CTRL_SDIO_RADIO_PHY_ON 0x4 +#define C_DS1_CTRL_SDIO_DS1_EXIT 0x8 +#define C_DS1_CTRL_PROC_DONE 0x100 +#define C_DS1_CTRL_REQ_VALID 0x200 + +/* M_ULP_WAKEIND bits */ +#define C_WATCHDOG_EXPIRY BIT(0) +#define C_FCBS_ERROR BIT(1) +#define C_RETX_FAILURE BIT(2) +#define C_HOST_WAKEUP BIT(3) +#define C_INVALID_FCBS_BLOCK BIT(4) +#define C_HUDI_DS1_EXIT BIT(5) +#define C_LOB_SLEEP BIT(6) +#define C_DS1_PHY_TXERR BIT(9) +#define C_DS1_WAKE_TIMER BIT(10) + +#define PHYTX_ERR_BLK_SIZE 18 +#define D11SHM_FIRST2BYTE_MASK 0xFFFF0000 +#define D11SHM_SECOND2BYTE_MASK 0x0000FFFF +#define D11SHM_2BYTE_SHIFT 16 + +#define D11SHM_RD(sdh, offset, ret) \ + brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) + +/* SHM Read is motified based on SHM 4 byte alignment as SHM size is 2 bytes and + * 2 byte is currently not working on FMAC + * If SHM address is not 4 byte aligned, then right shift by 16 + * otherwise, mask the first two MSB bytes + * Suppose data in address 7260 is 0x440002 and it is 4 byte aligned + * Correct SHM value is 0x2 for this SHM offset and next SHM value is 0x44 + */ +#define D11SHM_RDW(sdh, offset, ret) \ + ((offset % 4) ? \ + (brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \ + >> D11SHM_2BYTE_SHIFT) : \ + (brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \ + & D11SHM_SECOND2BYTE_MASK)) + +/* SHM is of size 2 bytes, 4 bytes write will overwrite other SHM's + * First read 4 bytes and then clear the required two bytes based on + * 4 byte alignment, then update the required value and write the + * 4 byte value now + */ +#define D11SHM_WR(sdh, offset, val, mask, ret) \ + do { \ + if ((offset) % 4) \ + val = (val & D11SHM_SECOND2BYTE_MASK) | \ + ((mask) << D11SHM_2BYTE_SHIFT); \ + else \ + val = (mask) | (val & D11SHM_FIRST2BYTE_MASK); \ + brcmf_sdiod_regwl(sdh, D11SHM_ADDR(offset), val, ret); \ + } while (0) +#define D11REG_WR(sdh, addr, val, ret) \ + brcmf_sdiod_regwl(sdh, addr, val, ret) + +#define D11REG_RD(sdh, addr, ret) \ + brcmf_sdiod_regrl(sdh, addr, ret) + #endif /* BRCMFMAC_SDIO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 11ffaa01599eb..6e893d613b552 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -30,6 +30,7 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "cfg80211.h" #define IOCTL_RESP_TIMEOUT msecs_to_jiffies(2000) @@ -103,7 +104,6 @@ static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = { */ #define DL_IMAGE_TOOBIG 7 /* firmware image too big */ - struct trx_header_le { __le32 magic; /* "HDR0" */ __le32 len; /* Length of file including header */ @@ -160,7 +160,7 @@ struct brcmf_usbdev_info { struct usb_device *usbdev; struct device *dev; - struct mutex dev_init_lock; + struct completion fw_downloaded; int ctl_in_pipe, ctl_out_pipe; struct urb *ctl_urb; /* URB for control endpoint */ @@ -175,7 +175,6 @@ struct brcmf_usbdev_info { struct urb *bulk_urb; /* used for FW download */ - bool wowl_enabled; struct brcmf_mp_device *settings; }; @@ -323,27 +322,44 @@ static int brcmf_usb_tx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_send_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("Txctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); + clear_bit(0, &devinfo->ctl_op); err = -EIO; + goto fail; } + clear_bit(0, &devinfo->ctl_op); + +fail: + usb_autopm_put_interface(intf); +out: return err; } @@ -352,32 +368,48 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_recv_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); err = devinfo->ctl_urb_status; - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("rxctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); + clear_bit(0, &devinfo->ctl_op); err = -EIO; + goto fail; } + clear_bit(0, &devinfo->ctl_op); + +fail: + usb_autopm_put_interface(intf); if (!err) return devinfo->ctl_urb_actual_length; - else - return err; +out: + return err; } static struct brcmf_usbreq *brcmf_usb_deq(struct brcmf_usbdev_info *devinfo, @@ -515,10 +547,12 @@ static void brcmf_usb_rx_complete(struct urb *urb) return; } - if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP) { + if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP || + devinfo->bus_pub.state == BRCMFMAC_USB_STATE_SLEEP) { skb_put(skb, urb->actual_length); brcmf_rx_frame(devinfo->dev, skb, true); brcmf_usb_rx_refill(devinfo, req); + usb_mark_last_busy(urb->dev); } else { brcmu_pkt_buf_free_skb(skb); brcmf_usb_enq(devinfo, &devinfo->rx_freeq, req, NULL); @@ -604,6 +638,11 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) struct brcmf_usbreq *req; int ret; unsigned long flags; + struct usb_interface *intf = to_usb_interface(dev); + + ret = usb_autopm_get_interface(intf); + if (ret) + goto out; brcmf_dbg(USB, "Enter, skb=%p\n", skb); if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { @@ -619,6 +658,8 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } + if (devinfo->bus_pub.bus->allow_skborphan) + skb_orphan(skb); req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, @@ -642,9 +683,10 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) devinfo->tx_flowblock = true; } spin_unlock_irqrestore(&devinfo->tx_flowblock_lock, flags); - return 0; fail: + usb_autopm_put_interface(intf); +out: return ret; } @@ -1002,20 +1044,32 @@ static int brcmf_usb_fw_download(struct brcmf_usbdev_info *devinfo) { int err; + struct usb_interface *intf; brcmf_dbg(USB, "Enter\n"); - if (devinfo == NULL) - return -ENODEV; + if (!devinfo) { + err = -ENODEV; + goto out; + } if (!devinfo->image) { brcmf_err("No firmware!\n"); - return -ENOENT; + err = -ENOENT; + goto out; } + intf = to_usb_interface(devinfo->dev); + err = usb_autopm_get_interface(intf); + if (err) + goto out; + err = brcmf_usb_dlstart(devinfo, (u8 *)devinfo->image, devinfo->image_len); if (err == 0) err = brcmf_usb_dlrun(devinfo); + + usb_autopm_put_interface(intf); +out: return err; } @@ -1116,16 +1170,21 @@ struct brcmf_usbdev *brcmf_usb_attach(struct brcmf_usbdev_info *devinfo, return NULL; } -static void brcmf_usb_wowl_config(struct device *dev, bool enabled) +static int brcmf_usb_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) { struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + int ret = 0; - brcmf_dbg(USB, "Configuring WOWL, enabled=%d\n", enabled); - devinfo->wowl_enabled = enabled; - if (enabled) - device_set_wakeup_enable(devinfo->dev, true); + if (devinfo->fw_name[0] != '\0') + strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN); else - device_set_wakeup_enable(devinfo->dev, false); + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_usb_fwnames, + ARRAY_SIZE(brcmf_usb_fwnames), + fw_name, NULL); + + return ret; } static const struct brcmf_bus_ops brcmf_usb_bus_ops = { @@ -1133,7 +1192,7 @@ static const struct brcmf_bus_ops brcmf_usb_bus_ops = { .stop = brcmf_usb_down, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, - .wowl_config = brcmf_usb_wowl_config, + .get_fwname = brcmf_usb_get_fwname, }; static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo) @@ -1147,6 +1206,12 @@ static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo) return ret; } + if (BRCMF_FWCON_ON()) { + ret = brcmf_fwlog_attach(devinfo->dev); + if (ret) + goto fail; + } + ret = brcmf_usb_up(devinfo->dev); if (ret) goto fail; @@ -1192,11 +1257,11 @@ static void brcmf_usb_probe_phase2(struct device *dev, int ret, if (ret) goto error; - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); return; error: brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), ret); - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); device_release_driver(dev); } @@ -1225,6 +1290,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) bus->ops = &brcmf_usb_bus_ops; bus->proto_type = BRCMF_PROTO_BCDC; bus->always_use_fws_queue = true; + bus->allow_skborphan = true; #ifdef CONFIG_PM bus->wowl_supported = true; #endif @@ -1242,7 +1308,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) if (ret) goto fail; /* we are done */ - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); return 0; } bus->chip = bus_pub->devid; @@ -1303,14 +1369,14 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) devinfo->usbdev = usb; devinfo->dev = &usb->dev; - /* Take an init lock, to protect for disconnect while still loading. - * Necessary because of the asynchronous firmware load construction - */ - mutex_init(&devinfo->dev_init_lock); - mutex_lock(&devinfo->dev_init_lock); + + /* Initialize fw downloaded completion and mark it as not complete */ + init_completion(&devinfo->fw_downloaded); usb_set_intfdata(intf, devinfo); + intf->needs_remote_wakeup = 1; + /* Check that the device supports only one configuration */ if (usb->descriptor.bNumConfigurations != 1) { brcmf_err("Number of configurations: %d not supported\n", @@ -1385,7 +1451,7 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) return 0; fail: - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); kfree(devinfo); usb_set_intfdata(intf, NULL); return ret; @@ -1400,7 +1466,7 @@ brcmf_usb_disconnect(struct usb_interface *intf) devinfo = (struct brcmf_usbdev_info *)usb_get_intfdata(intf); if (devinfo) { - mutex_lock(&devinfo->dev_init_lock); + wait_for_completion(&devinfo->fw_downloaded); /* Make sure that devinfo still exists. Firmware probe routines * may have released the device and cleared the intfdata. */ @@ -1421,13 +1487,25 @@ static int brcmf_usb_suspend(struct usb_interface *intf, pm_message_t state) { struct usb_device *usb = interface_to_usbdev(intf); struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); + struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(USB, "Enter\n"); + + bus = devinfo->bus_pub.bus; + config = bus->drvr->config; + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP; - if (devinfo->wowl_enabled) - brcmf_cancel_all_urbs(devinfo); - else - brcmf_detach(&usb->dev); + brcmf_cancel_all_urbs(devinfo); + device_set_wakeup_enable(devinfo->dev, true); return 0; } @@ -1440,11 +1518,9 @@ static int brcmf_usb_resume(struct usb_interface *intf) struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); brcmf_dbg(USB, "Enter\n"); - if (!devinfo->wowl_enabled) - return brcmf_usb_bus_setup(devinfo); - devinfo->bus_pub.state = BRCMFMAC_USB_STATE_UP; brcmf_usb_rx_fill_all(devinfo); + device_set_wakeup_enable(devinfo->dev, false); return 0; } @@ -1492,6 +1568,7 @@ static struct usb_driver brcmf_usbdrvr = { .suspend = brcmf_usb_suspend, .resume = brcmf_usb_resume, .reset_resume = brcmf_usb_reset_resume, + .supports_autosuspend = true, .disable_hub_initiated_lpm = 1, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 8eff2753abade..10003e808b4eb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -74,14 +74,27 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + if (cmdhdr->cmd == BRCMF_C_SET_AP) { + if (*(int *)(dcmd_buf) == 1) { + ifp->vif->wdev.iftype = NL80211_IFTYPE_AP; + brcmf_net_setcarrier(ifp, true); + } else { + ifp->vif->wdev.iftype = NL80211_IFTYPE_STATION; + } + } + if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - if (ret != 0) + + if (ret != 0) { + brcmf_dbg(INFO, "error(%d), return -EPERM\n", ret); + ret = -EPERM; goto exit; + } wr_pointer = dcmd_buf; while (ret_len > 0) { @@ -114,6 +127,56 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +s32 +brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) + +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct wiphy *wiphy = cfg_to_wiphy(cfg); + struct sk_buff *skb; + struct nlattr *phy_temp_data; + u32 version, temp, tempdelta; + struct brcmf_phy_temp_evt *phy_temp_evt; + + phy_temp_evt = (struct brcmf_phy_temp_evt *)data; + + version = le32_to_cpu(phy_temp_evt->version); + temp = le32_to_cpu(phy_temp_evt->temp); + tempdelta = le32_to_cpu(phy_temp_evt->tempdelta); + + skb = cfg80211_vendor_event_alloc(wiphy, NULL, + sizeof(*phy_temp_evt), + BRCMF_VNDR_EVTS_PHY_TEMP, + GFP_KERNEL); + + if (!skb) { + brcmf_dbg(EVENT, "NO MEM: can't allocate skb for vendor PHY_TEMP_EVENT\n"); + return -ENOMEM; + } + + phy_temp_data = nla_nest_start(skb, NL80211_ATTR_VENDOR_EVENTS); + if (!phy_temp_data) { + nla_nest_cancel(skb, phy_temp_data); + kfree_skb(skb); + brcmf_dbg(EVENT, "skb could not nest vendor attributes\n"); + return -EMSGSIZE; + } + + if (nla_put_u32(skb, BRCMF_NLATTR_VERS, version) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMP, temp) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMPDELTA, tempdelta)) { + kfree_skb(skb); + brcmf_dbg(EVENT, "NO ROOM in skb for vendor PHY_TEMP_EVENT\n"); + return -EMSGSIZE; + } + + nla_nest_end(skb, phy_temp_data); + + cfg80211_vendor_event(skb, GFP_KERNEL); + return 0; +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -125,3 +188,10 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, }; + +const struct nl80211_vendor_cmd_info brcmf_vendor_events[] = { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_EVTS_PHY_TEMP, + }, +}; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 061b7bfa2e1cf..f4495e9f893ce 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -25,6 +25,11 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_LAST }; +enum brcmf_vndr_evts { + BRCMF_VNDR_EVTS_PHY_TEMP, + BRCMF_VNDR_EVTS_LAST +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * @@ -36,11 +41,21 @@ enum brcmf_nlattrs { BRCMF_NLATTR_LEN, BRCMF_NLATTR_DATA, + BRCMF_NLATTR_VERS, + BRCMF_NLATTR_PHY_TEMP, + BRCMF_NLATTR_PHY_TEMPDELTA, __BRCMF_NLATTR_AFTER_LAST, BRCMF_NLATTR_MAX = __BRCMF_NLATTR_AFTER_LAST - 1 }; +/* structure of event sent up by firmware: is this the right place for it? */ +struct brcmf_phy_temp_evt { + __le32 version; + __le32 temp; + __le32 tempdelta; +} __packed; + /** * struct brcmf_vndr_dcmd_hdr - message header for cfg80211 vendor command dcmd * support @@ -60,5 +75,9 @@ struct brcmf_vndr_dcmd_hdr { }; extern const struct wiphy_vendor_command brcmf_vendor_cmds[]; +extern const struct nl80211_vendor_cmd_info brcmf_vendor_events[]; +s32 brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data); #endif /* _vendor_h_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c index eb5db94f57453..d8b79cb72b58d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c @@ -77,8 +77,6 @@ static u16 d11ac_bw(enum brcmu_chan_bw bw) return BRCMU_CHSPEC_D11AC_BW_40; case BRCMU_CHAN_BW_80: return BRCMU_CHSPEC_D11AC_BW_80; - case BRCMU_CHAN_BW_160: - return BRCMU_CHSPEC_D11AC_BW_160; default: WARN_ON(1); } @@ -192,41 +190,8 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) break; } break; - case BRCMU_CHSPEC_D11AC_BW_160: - ch->bw = BRCMU_CHAN_BW_160; - ch->sb = brcmu_maskget16(ch->chspec, BRCMU_CHSPEC_D11AC_SB_MASK, - BRCMU_CHSPEC_D11AC_SB_SHIFT); - switch (ch->sb) { - case BRCMU_CHAN_SB_LLL: - ch->control_ch_num -= CH_70MHZ_APART; - break; - case BRCMU_CHAN_SB_LLU: - ch->control_ch_num -= CH_50MHZ_APART; - break; - case BRCMU_CHAN_SB_LUL: - ch->control_ch_num -= CH_30MHZ_APART; - break; - case BRCMU_CHAN_SB_LUU: - ch->control_ch_num -= CH_10MHZ_APART; - break; - case BRCMU_CHAN_SB_ULL: - ch->control_ch_num += CH_10MHZ_APART; - break; - case BRCMU_CHAN_SB_ULU: - ch->control_ch_num += CH_30MHZ_APART; - break; - case BRCMU_CHAN_SB_UUL: - ch->control_ch_num += CH_50MHZ_APART; - break; - case BRCMU_CHAN_SB_UUU: - ch->control_ch_num += CH_70MHZ_APART; - break; - default: - WARN_ON_ONCE(1); - break; - } - break; case BRCMU_CHSPEC_D11AC_BW_8080: + case BRCMU_CHSPEC_D11AC_BW_160: default: WARN_ON_ONCE(1); break; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 57544a3a3ce4a..459b251868069 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -59,6 +59,8 @@ #define BRCM_CC_4366_CHIP_ID 0x4366 #define BRCM_CC_4371_CHIP_ID 0x4371 #define CY_CC_4373_CHIP_ID 0x4373 +#define CY_CC_43012_CHIP_ID 43012 +#define CY_CC_89459_CHIP_ID 0x4355 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e @@ -73,9 +75,12 @@ /* PCIE Device IDs */ #define BRCM_PCIE_4350_DEVICE_ID 0x43a3 #define BRCM_PCIE_4354_DEVICE_ID 0x43df +#define BRCM_PCIE_4354_RAW_DEVICE_ID 0x4354 +#define BRCM_PCIE_4355_DEVICE_ID 0x4355 #define BRCM_PCIE_4356_DEVICE_ID 0x43ec #define BRCM_PCIE_43567_DEVICE_ID 0x43d3 #define BRCM_PCIE_43570_DEVICE_ID 0x43d9 +#define BRCM_PCIE_43570_RAW_DEVICE_ID 0xaa31 #define BRCM_PCIE_4358_DEVICE_ID 0x43e9 #define BRCM_PCIE_4359_DEVICE_ID 0x43ef #define BRCM_PCIE_43602_DEVICE_ID 0x43ba @@ -89,7 +94,8 @@ #define BRCM_PCIE_4366_2G_DEVICE_ID 0x43c4 #define BRCM_PCIE_4366_5G_DEVICE_ID 0x43c5 #define BRCM_PCIE_4371_DEVICE_ID 0x440d - +#define CY_PCIE_89459_DEVICE_ID 0x4415 +#define CY_PCIE_89459_RAW_DEVICE_ID 0x4355 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 75b2a0438cfa7..cfe592fc4b040 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -29,8 +29,6 @@ #define CH_UPPER_SB 0x01 #define CH_LOWER_SB 0x02 #define CH_EWA_VALID 0x04 -#define CH_70MHZ_APART 14 -#define CH_50MHZ_APART 10 #define CH_30MHZ_APART 6 #define CH_20MHZ_APART 4 #define CH_10MHZ_APART 2 @@ -239,8 +237,11 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_RESERVED4 0x0400 #define WPA2_AUTH_RESERVED5 0x0800 #define WPA2_AUTH_1X_SHA256 0x1000 /* 1X with SHA256 key derivation */ +#define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ +#define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ + #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index e1fd499930a03..f5265966bf181 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -225,8 +225,197 @@ struct chipcregs { u32 PAD[3]; u32 retention_grpidx; /* 0x680 */ u32 retention_grpctl; /* 0x684 */ - u32 PAD[94]; - u16 sromotp[768]; + u32 mac_res_req_timer; /* 0x688 */ + u32 mac_res_req_mask; /* 0x68c */ + u32 PAD[18]; + u32 pmucontrol_ext; /* 0x6d8 */ + u32 slowclkperiod; /* 0x6dc */ + u32 PAD[8]; + u32 pmuintmask0; /* 0x700 */ + u32 pmuintmask1; /* 0x704 */ + u32 PAD[14]; + u32 pmuintstatus; /* 0x740 */ + u32 extwakeupstatus; /* 0x744 */ + u32 watchdog_res_mask; /* 0x748 */ + u32 swscratch; /* 0x750 */ + u32 PAD[3]; + u32 extwakemask[2]; /* 0x760-0x764 */ + u32 PAD[2]; + u32 extwakereqmask[2]; /* 0x770-0x774 */ + u32 PAD[2]; + u32 pmuintctrl0; /* 0x780 */ + u32 pmuintctrl1; /* 0x784 */ + u32 PAD[2]; + u32 extwakectrl[2]; /* 0x790 */ +}; + +#define CHIPGCIREGOFFS(field) offsetof(struct chipgciregs, field) + +struct chipgciregs { + u32 gci_corecaps0; /* 0x000 */ + u32 gci_corecaps1; /* 0x004 */ + u32 gci_corecaps2; /* 0x008 */ + u32 gci_corectrl; /* 0x00c */ + u32 gci_corestat; /* 0x010 */ + u32 gci_intstat; /* 0x014 */ + u32 gci_intmask; /* 0x018 */ + u32 gci_wakemask; /* 0x01c */ + u32 gci_levelintstat; /* 0x020 */ + u32 gci_eventintstat; /* 0x024 */ + u32 gci_wakelevelintstat; /* 0x028 */ + u32 gci_wakeeventintstat; /* 0x02c */ + u32 semaphoreintstatus; /* 0x030 */ + u32 semaphoreintmask; /* 0x034 */ + u32 semaphorerequest; /* 0x038 */ + u32 semaphorereserve; /* 0x03c */ + u32 gci_indirect_addr; /* 0x040 */ + u32 gci_gpioctl; /* 0x044 */ + u32 gci_gpiostatus; /* 0x048 */ + u32 gci_gpiomask; /* 0x04c */ + u32 eventsummary; /* 0x050 */ + u32 gci_miscctl; /* 0x054 */ + u32 gci_gpiointmask; /* 0x058 */ + u32 gci_gpiowakemask; /* 0x05c */ + u32 gci_input[32]; /* 0x060 */ + u32 gci_event[32]; /* 0x0e0 */ + u32 gci_output[4]; /* 0x160 */ + u32 gci_control_0; /* 0x170 */ + u32 gci_control_1; /* 0x174 */ + u32 gci_intpolreg; /* 0x178 */ + u32 gci_levelintmask; /* 0x17c */ + u32 gci_eventintmask; /* 0x180 */ + u32 wakelevelintmask; /* 0x184 */ + u32 wakeeventintmask; /* 0x188 */ + u32 hwmask; /* 0x18c */ + u32 PAD; + u32 gci_inbandeventintmask; /* 0x194 */ + u32 PAD; + u32 gci_inbandeventstatus; /* 0x19c */ + u32 gci_seciauxtx; /* 0x1a0 */ + u32 gci_seciauxrx; /* 0x1a4 */ + u32 gci_secitx_datatag; /* 0x1a8 */ + u32 gci_secirx_datatag; /* 0x1ac */ + u32 gci_secitx_datamask; /* 0x1b0 */ + u32 gci_seciusef0tx_reg; /* 0x1b4 */ + u32 gci_secif0tx_offset; /* 0x1b8 */ + u32 gci_secif0rx_offset; /* 0x1bc */ + u32 gci_secif1tx_offset; /* 0x1c0 */ + u32 gci_rxfifo_common_ctrl; /* 0x1c4 */ + u32 gci_rxfifoctrl; /* 0x1c8 */ + u32 gci_hw_sema_status; /* 0x1cc */ + u32 gci_seciuartescval; /* 0x1d0 */ + u32 gic_seciuartautobaudctr; /* 0x1d4 */ + u32 gci_secififolevel; /* 0x1d8 */ + u32 gci_seciuartdata; /* 0x1dc */ + u32 gci_secibauddiv; /* 0x1e0 */ + u32 gci_secifcr; /* 0x1e4 */ + u32 gci_secilcr; /* 0x1e8 */ + u32 gci_secimcr; /* 0x1ec */ + u32 gci_secilsr; /* 0x1f0 */ + u32 gci_secimsr; /* 0x1f4 */ + u32 gci_baudadj; /* 0x1f8 */ + u32 gci_inbandintmask; /* 0x1fc */ + u32 gci_chipctrl; /* 0x200 */ + u32 gci_chipsts; /* 0x204 */ + u32 gci_gpioout; /* 0x208 */ + u32 gci_gpioout_read; /* 0x20C */ + u32 gci_mpwaketx; /* 0x210 */ + u32 gci_mpwakedetect; /* 0x214 */ + u32 gci_seciin_ctrl; /* 0x218 */ + u32 gci_seciout_ctrl; /* 0x21C */ + u32 gci_seciin_auxfifo_en; /* 0x220 */ + u32 gci_seciout_txen_txbr; /* 0x224 */ + u32 gci_seciin_rxbrstatus; /* 0x228 */ + u32 gci_seciin_rxerrstatus; /* 0x22C */ + u32 gci_seciin_fcstatus; /* 0x230 */ + u32 gci_seciout_txstatus; /* 0x234 */ + u32 gci_seciout_txbrstatus; /* 0x238 */ + u32 wlan_mem_info; /* 0x23C */ + u32 wlan_bankxinfo; /* 0x240 */ + u32 bt_smem_select; /* 0x244 */ + u32 bt_smem_stby; /* 0x248 */ + u32 bt_smem_status; /* 0x24C */ + u32 wlan_bankxactivepda; /* 0x250 */ + u32 wlan_bankxsleeppda; /* 0x254 */ + u32 wlan_bankxkill; /* 0x258 */ + u32 PAD[41]; + u32 gci_chipid; /* 0x300 */ + u32 PAD[3]; + u32 otpstatus; /* 0x310 */ + u32 otpcontrol; /* 0x314 */ + u32 otpprog; /* 0x318 */ + u32 otplayout; /* 0x31c */ + u32 otplayoutextension; /* 0x320 */ + u32 otpcontrol1; /* 0x324 */ + u32 otpprogdata; /* 0x328 */ + u32 PAD[52]; + u32 otpECCstatus; /* 0x3FC */ + u32 PAD[512]; + u32 lhl_core_capab_adr; /* 0xC00 */ + u32 lhl_main_ctl_adr; /* 0xC04 */ + u32 lhl_pmu_ctl_adr; /* 0xC08 */ + u32 lhl_extlpo_ctl_adr; /* 0xC0C */ + u32 lpo_ctl_adr; /* 0xC10 */ + u32 lhl_lpo2_ctl_adr; /* 0xC14 */ + u32 lhl_osc32k_ctl_adr; /* 0xC18 */ + u32 lhl_clk_status_adr; /* 0xC1C */ + u32 lhl_clk_det_ctl_adr; /* 0xC20 */ + u32 lhl_clk_sel_adr; /* 0xC24 */ + u32 hidoff_cnt_adr[2]; /* 0xC28-0xC2C */ + u32 lhl_autoclk_ctl_adr; /* 0xC30 */ + u32 PAD; + u32 lhl_hibtim_adr; /* 0xC38 */ + u32 lhl_wl_ilp_val_adr; /* 0xC3C */ + u32 lhl_wl_armtim0_intrp_adr; /* 0xC40 */ + u32 lhl_wl_armtim0_st_adr; /* 0xC44 */ + u32 lhl_wl_armtim0_adr; /* 0xC48 */ + u32 PAD[9]; + u32 lhl_wl_mactim0_intrp_adr; /* 0xC70 */ + u32 lhl_wl_mactim0_st_adr; /* 0xC74 */ + u32 lhl_wl_mactim_int0_adr; /* 0xC78 */ + u32 lhl_wl_mactim_frac0_adr; /* 0xC7C */ + u32 lhl_wl_mactim1_intrp_adr; /* 0xC80 */ + u32 lhl_wl_mactim1_st_adr; /* 0xC84 */ + u32 lhl_wl_mactim_int1_adr; /* 0xC88 */ + u32 lhl_wl_mactim_frac1_adr; /* 0xC8C */ + u32 PAD[8]; + u32 gpio_int_en_port_adr[4]; /* 0xCB0-0xCBC */ + u32 gpio_int_st_port_adr[4]; /* 0xCC0-0xCCC */ + u32 gpio_ctrl_iocfg_p_adr[64]; /* 0xCD0-0xDCC */ + u32 gpio_gctrl_iocfg_p0_p39_adr; /* 0xDD0 */ + u32 gpio_gdsctrl_iocfg_p0_p25_p30_p39_adr; /* 0xDD4 */ + u32 gpio_gdsctrl_iocfg_p26_p29_adr; /* 0xDD8 */ + u32 PAD[8]; + u32 lhl_gpio_din0_adr; /* 0xDFC */ + u32 lhl_gpio_din1_adr; /* 0xE00 */ + u32 lhl_wkup_status_adr; /* 0xE04 */ + u32 lhl_ctl_adr; /* 0xE08 */ + u32 lhl_adc_ctl_adr; /* 0xE0C */ + u32 lhl_qdxyz_in_dly_adr; /* 0xE10 */ + u32 lhl_optctl_adr; /* 0xE14 */ + u32 lhl_optct2_adr; /* 0xE18 */ + u32 lhl_scanp_cntr_init_val_adr; /* 0xE1C */ + u32 lhl_opt_togg_val_adr[6]; /* 0xE20-0xE34 */ + u32 lhl_optx_smp_val_adr; /* 0xE38 */ + u32 lhl_opty_smp_val_adr; /* 0xE3C */ + u32 lhl_optz_smp_val_adr; /* 0xE40 */ + u32 lhl_hidoff_keepstate_adr[3]; /* 0xE44-0xE4C */ + u32 lhl_bt_slmboot_ctl0_adr[4]; /* 0xE50-0xE5C */ + u32 lhl_wl_fw_ctl; /* 0xE60 */ + u32 lhl_wl_hw_ctl_adr[2]; /* 0xE64-0xE68 */ + u32 lhl_bt_hw_ctl_adr; /* 0xE6C */ + u32 lhl_top_pwrseq_en_adr; /* 0xE70 */ + u32 lhl_top_pwrdn_ctl_adr; /* 0xE74 */ + u32 lhl_top_pwrup_ctl_adr; /* 0xE78 */ + u32 lhl_top_pwrseq_ctl_adr; /* 0xE7C */ + u32 lhl_top_pwrdn2_ctl_adr; /* 0xE80 */ + u32 lhl_top_pwrup2_ctl_adr; /* 0xE84 */ + u32 wpt_regon_intrp_cfg_adr; /* 0xE88 */ + u32 bt_regon_intrp_cfg_adr; /* 0xE8C */ + u32 wl_regon_intrp_cfg_adr; /* 0xE90 */ + u32 regon_intrp_st_adr; /* 0xE94 */ + u32 regon_intrp_en_adr; /* 0xE98 */ + }; /* chipid */ @@ -298,6 +487,8 @@ struct chipcregs { * Maximum delay for the PMU state transition in us. * This is an upper bound intended for spinwaits etc. */ -#define PMU_MAX_TRANSITION_DLY 15000 +#define PMU_MAX_TRANSITION_DLY 15000 + +#define DEFAULT_43012_MIN_RES_MASK 0x0f8bfe77 #endif /* _SBCHIPC_H */ diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig index ba7b034b2b91c..7d4169e05b1d9 100644 --- a/drivers/of/Kconfig +++ b/drivers/of/Kconfig @@ -112,4 +112,11 @@ config OF_OVERLAY config OF_NUMA bool +config OF_CONFIGFS + bool "Device Tree Overlay ConfigFS interface" + select CONFIGFS_FS + depends on OF_OVERLAY + help + Enable a simple user-space driven DT overlay interface. + endif # OF diff --git a/drivers/of/Makefile b/drivers/of/Makefile index 8f19d342eed8e..7afce596322d2 100644 --- a/drivers/of/Makefile +++ b/drivers/of/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-y = base.o device.o platform.o property.o +obj-$(CONFIG_OF_CONFIGFS) += configfs.o obj-$(CONFIG_OF_DYNAMIC) += dynamic.o obj-$(CONFIG_OF_FLATTREE) += fdt.o obj-$(CONFIG_OF_EARLY_FLATTREE) += fdt_address.o diff --git a/drivers/of/base.c b/drivers/of/base.c index ce8a6e0c9b6a9..b41324e82349f 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "of_private.h" @@ -45,6 +46,18 @@ static const char *of_stdout_options; struct kset *of_kset; +const struct rhashtable_params of_phandle_ht_params = { + .key_offset = offsetof(struct device_node, phandle), /* base offset */ + .key_len = sizeof(phandle), + .head_offset = offsetof(struct device_node, ht_node), + .automatic_shrinking = true, +}; + +struct rhashtable *of_phandle_ht; + +/* default is false */ +bool of_phandle_ht_is_disabled; + /* * Used to protect the of_aliases, to hold off addition of nodes to sysfs. * This mutex must be held whenever modifications are being made to the @@ -162,13 +175,19 @@ int __of_add_property_sysfs(struct device_node *np, struct property *pp) return rc; } -int __of_attach_node_sysfs(struct device_node *np) +int __of_attach_node_post(struct device_node *np) { const char *name; struct kobject *parent; struct property *pp; int rc; + if (of_phandle_ht_available()) { + rc = of_phandle_ht_insert(np); + WARN(rc, "insert to phandle hash fail @%s\n", + of_node_full_name(np)); + } + if (!IS_ENABLED(CONFIG_SYSFS)) return 0; @@ -200,6 +219,18 @@ int __of_attach_node_sysfs(struct device_node *np) void __init of_core_init(void) { struct device_node *np; + int ret; + + of_phandle_ht = kzalloc(sizeof(*of_phandle_ht), GFP_KERNEL); + if (!of_phandle_ht) { + pr_warn("devicetree: Failed to allocate hashtable\n"); + return; + } + ret = rhashtable_init(of_phandle_ht, &of_phandle_ht_params); + if (ret) { + pr_warn("devicetree: Failed to initialize hashtable\n"); + return; + } /* Create the kset, and register existing nodes */ mutex_lock(&of_mutex); @@ -210,12 +241,16 @@ void __init of_core_init(void) return; } for_each_of_allnodes(np) - __of_attach_node_sysfs(np); + __of_attach_node_post(np); mutex_unlock(&of_mutex); /* Symlink in /proc as required by userspace ABI */ if (of_root) proc_symlink("device-tree", NULL, "/sys/firmware/devicetree/base"); + + ret = of_overlay_init(); + if (ret != 0) + pr_warn("of_init: of_overlay_init failed!\n"); } static struct property *__of_find_property(const struct device_node *np, @@ -1133,9 +1168,14 @@ struct device_node *of_find_node_by_phandle(phandle handle) return NULL; raw_spin_lock_irqsave(&devtree_lock, flags); - for_each_of_allnodes(np) - if (np->phandle == handle) - break; + /* when we're ready use the hash table (and not disabled) */ + if (of_phandle_ht_available() && !of_phandle_ht_is_disabled) + np = of_phandle_ht_lookup(handle); + else { /* fallback */ + for_each_of_allnodes(np) + if (np->phandle == handle) + break; + } of_node_get(np); raw_spin_unlock_irqrestore(&devtree_lock, flags); return np; diff --git a/drivers/of/configfs.c b/drivers/of/configfs.c new file mode 100644 index 0000000000000..c7e999c9d56f7 --- /dev/null +++ b/drivers/of/configfs.c @@ -0,0 +1,307 @@ +/* + * Configfs entries for device-tree + * + * Copyright (C) 2013 - Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "of_private.h" + +struct cfs_overlay_item { + struct config_item item; + + char path[PATH_MAX]; + + const struct firmware *fw; + struct device_node *overlay; + int ov_id; + + void *dtbo; + int dtbo_size; +}; + +static int create_overlay(struct cfs_overlay_item *overlay, void *blob) +{ + int err; + + /* unflatten the tree */ + of_fdt_unflatten_tree(blob, NULL, &overlay->overlay); + if (overlay->overlay == NULL) { + pr_err("%s: failed to unflatten tree\n", __func__); + err = -EINVAL; + goto out_err; + } + pr_debug("%s: unflattened OK\n", __func__); + + /* mark it as detached */ + of_node_set_flag(overlay->overlay, OF_DETACHED); + + /* perform resolution */ + err = of_resolve_phandles(overlay->overlay); + if (err != 0) { + pr_err("%s: Failed to resolve tree\n", __func__); + goto out_err; + } + pr_debug("%s: resolved OK\n", __func__); + + err = of_overlay_create(overlay->overlay); + if (err < 0) { + pr_err("%s: Failed to create overlay (err=%d)\n", + __func__, err); + goto out_err; + } + overlay->ov_id = err; + +out_err: + return err; +} + +static inline struct cfs_overlay_item *to_cfs_overlay_item( + struct config_item *item) +{ + return item ? container_of(item, struct cfs_overlay_item, item) : NULL; +} + +static ssize_t cfs_overlay_item_path_show(struct config_item *item, char *page) +{ + return sprintf(page, "%s\n", to_cfs_overlay_item(item)->path); +} + +static ssize_t cfs_overlay_item_path_store(struct config_item *item, + const char *page, size_t count) +{ + struct cfs_overlay_item *overlay = to_cfs_overlay_item(item); + const char *p = page; + char *s; + int err; + + /* if it's set do not allow changes */ + if (overlay->path[0] != '\0' || overlay->dtbo_size > 0) + return -EPERM; + + /* copy to path buffer (and make sure it's always zero terminated */ + count = snprintf(overlay->path, sizeof(overlay->path) - 1, "%s", p); + overlay->path[sizeof(overlay->path) - 1] = '\0'; + + /* strip trailing newlines */ + s = overlay->path + strlen(overlay->path); + while (s > overlay->path && *--s == '\n') + *s = '\0'; + + pr_debug("%s: path is '%s'\n", __func__, overlay->path); + + err = request_firmware(&overlay->fw, overlay->path, NULL); + if (err != 0) + goto out_err; + + err = create_overlay(overlay, (void *)overlay->fw->data); + if (err < 0) + goto out_err; + + return count; + +out_err: + + release_firmware(overlay->fw); + overlay->fw = NULL; + + overlay->path[0] = '\0'; + return err; +} + +static ssize_t cfs_overlay_item_status_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%s\n", to_cfs_overlay_item(item)->ov_id >= 0 ? + "applied" : "unapplied"); +} + +CONFIGFS_ATTR(cfs_overlay_item_, path); +CONFIGFS_ATTR_RO(cfs_overlay_item_, status); + +static struct configfs_attribute *cfs_overlay_attrs[] = { + &cfs_overlay_item_attr_path, + &cfs_overlay_item_attr_status, + NULL, +}; + +ssize_t cfs_overlay_item_dtbo_read(struct config_item *item, void *buf, + size_t max_count) +{ + struct cfs_overlay_item *overlay = to_cfs_overlay_item(item); + + pr_debug("%s: buf=%p max_count=%u\n", __func__, + buf, max_count); + + if (overlay->dtbo == NULL) + return 0; + + /* copy if buffer provided */ + if (buf != NULL) { + /* the buffer must be large enough */ + if (overlay->dtbo_size > max_count) + return -ENOSPC; + + memcpy(buf, overlay->dtbo, overlay->dtbo_size); + } + + return overlay->dtbo_size; +} + +ssize_t cfs_overlay_item_dtbo_write(struct config_item *item, const void *buf, + size_t count) +{ + struct cfs_overlay_item *overlay = to_cfs_overlay_item(item); + int err; + + /* if it's set do not allow changes */ + if (overlay->path[0] != '\0' || overlay->dtbo_size > 0) + return -EPERM; + + /* copy the contents */ + overlay->dtbo = kmemdup(buf, count, GFP_KERNEL); + if (overlay->dtbo == NULL) + return -ENOMEM; + + overlay->dtbo_size = count; + + err = create_overlay(overlay, overlay->dtbo); + if (err < 0) + goto out_err; + + return count; + +out_err: + kfree(overlay->dtbo); + overlay->dtbo = NULL; + overlay->dtbo_size = 0; + + return err; +} + +CONFIGFS_BIN_ATTR(cfs_overlay_item_, dtbo, NULL, SZ_1M); + +static struct configfs_bin_attribute *cfs_overlay_bin_attrs[] = { + &cfs_overlay_item_attr_dtbo, + NULL, +}; + +static void cfs_overlay_release(struct config_item *item) +{ + struct cfs_overlay_item *overlay = to_cfs_overlay_item(item); + + if (overlay->ov_id >= 0) + of_overlay_destroy(overlay->ov_id); + if (overlay->fw) + release_firmware(overlay->fw); + /* kfree with NULL is safe */ + kfree(overlay->dtbo); + kfree(overlay); +} + +static struct configfs_item_operations cfs_overlay_item_ops = { + .release = cfs_overlay_release, +}; + +static struct config_item_type cfs_overlay_type = { + .ct_item_ops = &cfs_overlay_item_ops, + .ct_attrs = cfs_overlay_attrs, + .ct_bin_attrs = cfs_overlay_bin_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct config_item *cfs_overlay_group_make_item( + struct config_group *group, const char *name) +{ + struct cfs_overlay_item *overlay; + + overlay = kzalloc(sizeof(*overlay), GFP_KERNEL); + if (!overlay) + return ERR_PTR(-ENOMEM); + overlay->ov_id = -1; + + config_item_init_type_name(&overlay->item, name, &cfs_overlay_type); + return &overlay->item; +} + +static void cfs_overlay_group_drop_item(struct config_group *group, + struct config_item *item) +{ + struct cfs_overlay_item *overlay = to_cfs_overlay_item(item); + + config_item_put(&overlay->item); +} + +static struct configfs_group_operations overlays_ops = { + .make_item = cfs_overlay_group_make_item, + .drop_item = cfs_overlay_group_drop_item, +}; + +static struct config_item_type overlays_type = { + .ct_group_ops = &overlays_ops, + .ct_owner = THIS_MODULE, +}; + +static struct configfs_group_operations of_cfs_ops = { + /* empty - we don't allow anything to be created */ +}; + +static struct config_item_type of_cfs_type = { + .ct_group_ops = &of_cfs_ops, + .ct_owner = THIS_MODULE, +}; + +struct config_group of_cfs_overlay_group; + +static struct configfs_subsystem of_cfs_subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "device-tree", + .ci_type = &of_cfs_type, + }, + }, + .su_mutex = __MUTEX_INITIALIZER(of_cfs_subsys.su_mutex), +}; + +static int __init of_cfs_init(void) +{ + int ret; + + pr_info("%s\n", __func__); + + config_group_init(&of_cfs_subsys.su_group); + config_group_init_type_name(&of_cfs_overlay_group, "overlays", + &overlays_type); + configfs_add_default_group(&of_cfs_overlay_group, + &of_cfs_subsys.su_group); + + ret = configfs_register_subsystem(&of_cfs_subsys); + if (ret != 0) { + pr_err("%s: failed to register subsys\n", __func__); + goto out; + } + pr_info("%s: OK\n", __func__); +out: + return ret; +} +late_initcall(of_cfs_init); diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index 301b6db2b48db..15d22758234a9 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "of_private.h" @@ -43,9 +44,16 @@ void of_node_put(struct device_node *node) } EXPORT_SYMBOL(of_node_put); -void __of_detach_node_sysfs(struct device_node *np) +void __of_detach_node_post(struct device_node *np) { struct property *pp; + int rc; + + if (of_phandle_ht_available()) { + rc = of_phandle_ht_remove(np); + WARN(rc, "remove from phandle hash fail @%s\n", + of_node_full_name(np)); + } if (!IS_ENABLED(CONFIG_SYSFS)) return; @@ -253,7 +261,7 @@ int of_attach_node(struct device_node *np) __of_attach_node(np); raw_spin_unlock_irqrestore(&devtree_lock, flags); - __of_attach_node_sysfs(np); + __of_attach_node_post(np); mutex_unlock(&of_mutex); of_reconfig_notify(OF_RECONFIG_ATTACH_NODE, &rd); @@ -306,7 +314,7 @@ int of_detach_node(struct device_node *np) __of_detach_node(np); raw_spin_unlock_irqrestore(&devtree_lock, flags); - __of_detach_node_sysfs(np); + __of_detach_node_post(np); mutex_unlock(&of_mutex); of_reconfig_notify(OF_RECONFIG_DETACH_NODE, &rd); @@ -396,8 +404,9 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) } /** - * __of_node_dup() - Duplicate or create an empty device node dynamically. - * @fmt: Format string (plus vargs) for new full name of the device node + * __of_node_dupv() - Duplicate or create an empty device node dynamically. + * @fmt: Format string for new full name of the device node + * @vargs: va_list containing the arugments for the node full name * * Create an device tree node, either by duplicating an empty node or by allocating * an empty one suitable for further modification. The node data are @@ -405,17 +414,15 @@ struct property *__of_prop_dup(const struct property *prop, gfp_t allocflags) * OF_DETACHED bits set. Returns the newly allocated node or NULL on out of * memory error. */ -struct device_node *__of_node_dup(const struct device_node *np, const char *fmt, ...) +struct device_node *__of_node_dupv(const struct device_node *np, + const char *fmt, va_list vargs) { - va_list vargs; struct device_node *node; node = kzalloc(sizeof(*node), GFP_KERNEL); if (!node) return NULL; - va_start(vargs, fmt); node->full_name = kvasprintf(GFP_KERNEL, fmt, vargs); - va_end(vargs); if (!node->full_name) { kfree(node); return NULL; @@ -447,6 +454,24 @@ struct device_node *__of_node_dup(const struct device_node *np, const char *fmt, return NULL; } +/** + * __of_node_dup() - Duplicate or create an empty device node dynamically. + * @fmt: Format string (plus vargs) for new full name of the device node + * + * See: __of_node_dupv() + */ +struct device_node *__of_node_dup(const struct device_node *np, + const char *fmt, ...) +{ + va_list vargs; + struct device_node *node; + + va_start(vargs, fmt); + node = __of_node_dupv(np, fmt, vargs); + va_end(vargs); + return node; +} + static void __of_changeset_entry_destroy(struct of_changeset_entry *ce) { of_node_put(ce->np); @@ -613,10 +638,10 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce) switch (ce->action) { case OF_RECONFIG_ATTACH_NODE: - __of_attach_node_sysfs(ce->np); + __of_attach_node_post(ce->np); break; case OF_RECONFIG_DETACH_NODE: - __of_detach_node_sysfs(ce->np); + __of_detach_node_post(ce->np); break; case OF_RECONFIG_ADD_PROPERTY: /* ignore duplicate names */ @@ -812,3 +837,295 @@ int of_changeset_action(struct of_changeset *ocs, unsigned long action, return 0; } EXPORT_SYMBOL_GPL(of_changeset_action); + +/* changeset helpers */ + +/** + * of_changeset_create_device_node - Create an empty device node + * + * @ocs: changeset pointer + * @parent: parent device node + * @fmt: format string for the node's full_name + * @args: argument list for the format string + * + * Create an empty device node, marking it as detached and allocated. + * + * Returns a device node on success, an error encoded pointer otherwise + */ +struct device_node *of_changeset_create_device_nodev( + struct of_changeset *ocs, struct device_node *parent, + const char *fmt, va_list vargs) +{ + struct device_node *node; + + node = __of_node_dupv(NULL, fmt, vargs); + if (!node) + return ERR_PTR(-ENOMEM); + + node->parent = parent; + return node; +} +EXPORT_SYMBOL_GPL(of_changeset_create_device_nodev); + +/** + * of_changeset_create_device_node - Create an empty device node + * + * @ocs: changeset pointer + * @parent: parent device node + * @fmt: Format string for the node's full_name + * ... Arguments + * + * Create an empty device node, marking it as detached and allocated. + * + * Returns a device node on success, an error encoded pointer otherwise + */ +__printf(3, 4) struct device_node * +of_changeset_create_device_node(struct of_changeset *ocs, + struct device_node *parent, const char *fmt, ...) +{ + va_list vargs; + struct device_node *node; + + va_start(vargs, fmt); + node = of_changeset_create_device_nodev(ocs, parent, fmt, vargs); + va_end(vargs); + return node; +} +EXPORT_SYMBOL_GPL(of_changeset_create_device_node); + +/** + * __of_changeset_add_property_copy - Create/update a new property copying + * name & value + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @value: pointer to the value data + * @length: length of the value in bytes + * @update: True on update operation + * + * Adds/updates a property to the changeset by making copies of the name & value + * entries. The @update parameter controls whether an add or update takes place. + * + * Returns zero on success, a negative error value otherwise. + */ +int __of_changeset_add_update_property_copy(struct of_changeset *ocs, + struct device_node *np, const char *name, const void *value, + int length, bool update) +{ + struct property *prop; + char *new_name; + void *new_value; + int ret = -ENOMEM; + + prop = kzalloc(sizeof(*prop), GFP_KERNEL); + if (!prop) + return -ENOMEM; + + new_name = kstrdup(name, GFP_KERNEL); + if (!new_name) + goto out_err; + + /* + * NOTE: There is no check for zero length value. + * In case of a boolean property, this will allocate a value + * of zero bytes. We do this to work around the use + * of of_get_property() calls on boolean values. + */ + new_value = kmemdup(value, length, GFP_KERNEL); + if (!new_value) + goto out_err; + + of_property_set_flag(prop, OF_DYNAMIC); + + prop->name = new_name; + prop->value = new_value; + prop->length = length; + + if (!update) + ret = of_changeset_add_property(ocs, np, prop); + else + ret = of_changeset_update_property(ocs, np, prop); + + if (!ret) + return 0; + +out_err: + kfree(prop->value); + kfree(prop->name); + kfree(prop); + return ret; +} +EXPORT_SYMBOL_GPL(__of_changeset_add_update_property_copy); + +/** + * of_changeset_add_property_stringf - Create a new formatted string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @fmt: format of string property + * ... arguments of the format string + * + * Adds a string property to the changeset by making copies of the name + * and the formatted value. + * + * Returns zero on success, a negative error value otherwise. + */ +__printf(4, 5) int of_changeset_add_property_stringf( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *fmt, ...) +{ + va_list vargs; + int ret; + + va_start(vargs, fmt); + ret = __of_changeset_add_update_property_stringv(ocs, np, name, fmt, + vargs, false); + va_end(vargs); + return ret; +} +EXPORT_SYMBOL_GPL(of_changeset_add_property_stringf); + +/** + * of_changeset_update_property_stringf - Update formatted string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @fmt: format of string property + * ... arguments of the format string + * + * Updates a string property to the changeset by making copies of the name + * and the formatted value. + * + * Returns zero on success, a negative error value otherwise. + */ +int of_changeset_update_property_stringf( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *fmt, ...) +{ + va_list vargs; + int ret; + + va_start(vargs, fmt); + ret = __of_changeset_add_update_property_stringv(ocs, np, name, fmt, + vargs, true); + va_end(vargs); + return ret; +} +EXPORT_SYMBOL_GPL(of_changeset_update_property_stringf); + +/** + * __of_changeset_add_update_property_string_list - Create/update a string + * list property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @strs: pointer to the string list + * @count: string count + * @update: True on update operation + * + * Adds a string list property to the changeset. + * + * Returns zero on success, a negative error value otherwise. + */ +int __of_changeset_add_update_property_string_list( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char **strs, int count, bool update) +{ + int total = 0, i, ret; + char *value, *s; + + for (i = 0; i < count; i++) { + /* check if it's NULL */ + if (!strs[i]) + return -EINVAL; + total += strlen(strs[i]) + 1; + } + + value = kmalloc(total, GFP_KERNEL); + if (!value) + return -ENOMEM; + + for (i = 0, s = value; i < count; i++) { + /* no need to check for NULL, check above */ + strcpy(s, strs[i]); + s += strlen(strs[i]) + 1; + } + + ret = __of_changeset_add_update_property_copy(ocs, np, name, value, + total, update); + + kfree(value); + + return ret; +} +EXPORT_SYMBOL_GPL(__of_changeset_add_update_property_string_list); + +static struct device_node * +__of_changeset_node_move_one(struct of_changeset *ocs, + struct device_node *np, struct device_node *new_parent) +{ + struct device_node *np2; + const char *unitname; + int err; + + err = of_changeset_detach_node(ocs, np); + if (err) + return ERR_PTR(err); + + unitname = strrchr(np->full_name, '/'); + if (!unitname) + unitname = np->full_name; + + np2 = __of_node_dup(np, "%s/%s", + new_parent->full_name, unitname); + if (!np2) + return ERR_PTR(-ENOMEM); + np2->parent = new_parent; + + err = of_changeset_attach_node(ocs, np2); + if (err) + return ERR_PTR(err); + + return np2; +} + +/** + * of_changeset_node_move_to - Moves a subtree to a new place in + * the tree + * + * @ocs: changeset pointer + * @np: device node pointer to be moved + * @to: device node of the new parent + * + * Moves a subtree to a new place in the tree. + * Note that a move is a safe operation because the phandles + * remain valid. + * + * Returns zero on success, a negative error value otherwise. + */ +int of_changeset_node_move(struct of_changeset *ocs, + struct device_node *np, struct device_node *new_parent) +{ + struct device_node *npc, *nppc; + + /* move the root first */ + nppc = __of_changeset_node_move_one(ocs, np, new_parent); + if (IS_ERR(nppc)) + return PTR_ERR(nppc); + + /* move the subtrees next */ + for_each_child_of_node(np, npc) { + nppc = __of_changeset_node_move_one(ocs, npc, nppc); + if (IS_ERR(nppc)) { + of_node_put(npc); + return PTR_ERR(nppc); + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(of_changeset_node_move); diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 3ae12ffbf5472..7c2b2aeda44a9 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -93,9 +93,9 @@ extern int __of_update_property(struct device_node *np, extern void __of_update_property_sysfs(struct device_node *np, struct property *newprop, struct property *oldprop); -extern int __of_attach_node_sysfs(struct device_node *np); +extern int __of_attach_node_post(struct device_node *np); extern void __of_detach_node(struct device_node *np); -extern void __of_detach_node_sysfs(struct device_node *np); +extern void __of_detach_node_post(struct device_node *np); extern void __of_sysfs_remove_bin_file(struct device_node *np, struct property *prop); @@ -109,4 +109,46 @@ extern void __of_sysfs_remove_bin_file(struct device_node *np, #define for_each_transaction_entry_reverse(_oft, _te) \ list_for_each_entry_reverse(_te, &(_oft)->te_list, node) +#if defined(CONFIG_OF_OVERLAY) +extern int of_overlay_init(void); +#else +static inline int of_overlay_init(void) +{ + return 0; +} +#endif + +extern const struct rhashtable_params of_phandle_ht_params; +extern struct rhashtable *of_phandle_ht; + +/* for unittest use */ +extern bool of_phandle_ht_is_disabled; + +static inline bool of_phandle_ht_available(void) +{ + return of_phandle_ht != NULL; +} + +static inline int of_phandle_ht_insert(struct device_node *np) +{ + if (!np || !np->phandle) + return 0; + return rhashtable_insert_fast(of_phandle_ht, + &np->ht_node, of_phandle_ht_params); +} + +static inline int of_phandle_ht_remove(struct device_node *np) +{ + if (!np || !np->phandle) + return 0; + return rhashtable_remove_fast(of_phandle_ht, + &np->ht_node, of_phandle_ht_params); +} + +static inline struct device_node *of_phandle_ht_lookup(phandle handle) +{ + return rhashtable_lookup_fast(of_phandle_ht, + &handle, of_phandle_ht_params); +} + #endif /* _LINUX_OF_PRIVATE_H */ diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index 8ecfee31ab6d3..a63f624d2e00a 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -21,11 +21,28 @@ #include #include #include +#include +#include #include "of_private.h" +/* fwd. decl */ +struct of_overlay; +struct of_overlay_info; + +/* an attribute for each fragment */ +struct fragment_attribute { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct fragment_attribute *fattr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct fragment_attribute *fattr, + const char *buf, size_t count); + struct of_overlay_info *ovinfo; +}; + /** * struct of_overlay_info - Holds a single overlay info + * @info: info node that contains the target and overlay * @target: target of the overlay operation * @overlay: pointer to the overlay contents node * @@ -33,8 +50,13 @@ * records. */ struct of_overlay_info { + struct of_overlay *ov; + struct device_node *info; struct device_node *target; struct device_node *overlay; + struct attribute_group attr_group; + struct attribute *attrs[2]; + struct fragment_attribute target_attr; bool is_symbols_node; }; @@ -52,12 +74,27 @@ struct of_overlay { struct list_head node; int count; struct of_overlay_info *ovinfo_tab; + const struct attribute_group **attr_groups; struct of_changeset cset; + struct kobject kobj; + int target_index; + struct device_node *target_root; }; +/* master enable switch; once set to 0 can't be re-enabled */ +static atomic_t ov_enable = ATOMIC_INIT(1); + +static int __init of_overlay_disable_setup(char *str __always_unused) +{ + atomic_set(&ov_enable, 0); + return 1; +} +__setup("of_overlay_disable", of_overlay_disable_setup); + static int of_overlay_apply_one(struct of_overlay *ov, struct device_node *target, const struct device_node *overlay, bool is_symbols_node); +static int overlay_removal_is_ok(struct of_overlay *ov); static BLOCKING_NOTIFIER_HEAD(of_overlay_chain); @@ -309,30 +346,92 @@ static int of_overlay_apply(struct of_overlay *ov) /* * Find the target node using a number of different strategies - * in order of preference + * in order of preference. Respects the target index if available. * * "target" property containing the phandle of the target * "target-path" property containing the path of the target */ -static struct device_node *find_target_node(struct device_node *info_node) +static struct device_node *find_target_node(struct of_overlay *ov, + struct device_node *info_node, int index) { + struct device_node *target = NULL, *np; const char *path; + char *newpath; u32 val; int ret; /* first try to go by using the target as a phandle */ - ret = of_property_read_u32(info_node, "target", &val); - if (ret == 0) - return of_find_node_by_phandle(val); + ret = of_property_read_u32_index(info_node, "target", index, &val); + if (ret == 0) { + target = of_find_node_by_phandle(val); + if (!target) { + pr_err("%s: Could not find target phandle 0x%x\n", + __func__, val); + return NULL; + } + goto check_root; + } - /* now try to locate by path */ - ret = of_property_read_string(info_node, "target-path", &path); - if (ret == 0) - return of_find_node_by_path(path); + /* failed, try to locate by path */ + ret = of_property_read_string_index(info_node, "target-path", index, + &path); + if (ret == 0) { + + if (!ov->target_root) { + target = of_find_node_by_path(path); + if (!target) + pr_err("%s: Could not find target path \"%s\"\n", + __func__, path); + return target; + } + + /* remove preceding '/' from path; relative path */ + if (*path == '/') { + while (*path == '/') + path++; + + newpath = kasprintf(GFP_KERNEL, "%s%s%s", + of_node_full_name(ov->target_root), + *path ? "/" : "", path); + if (!newpath) { + pr_err("%s: Could not allocate \"%s%s%s\"\n", + __func__, + of_node_full_name(ov->target_root), + *path ? "/" : "", path); + return NULL; + } + target = of_find_node_by_path(newpath); + kfree(newpath); + + return target; + + } + /* target is an alias, need to check */ + target = of_find_node_by_path(path); + if (!target) { + pr_err("%s: Could not find alias \"%s\"\n", + __func__, path); + return NULL; + } + goto check_root; + } + + return NULL; - pr_err("Failed to find target for node %p (%s)\n", - info_node, info_node->name); +check_root: + if (!ov->target_root) + return target; + /* got a target, but we have to check it's under target root */ + for (np = target; np; np = np->parent) { + if (np == ov->target_root) + return target; + } + pr_err("%s: target \"%s\" not under target_root \"%s\"\n", + __func__, of_node_full_name(target), + of_node_full_name(ov->target_root)); + /* target is not under target_root */ + of_node_put(target); return NULL; } @@ -357,10 +456,12 @@ static int of_fill_overlay_info(struct of_overlay *ov, if (ovinfo->overlay == NULL) goto err_fail; - ovinfo->target = find_target_node(info_node); + ovinfo->target = find_target_node(ov, info_node, ov->target_index); if (ovinfo->target == NULL) goto err_fail; + ovinfo->info = of_node_get(info_node); + return 0; err_fail: @@ -371,6 +472,17 @@ static int of_fill_overlay_info(struct of_overlay *ov, return -EINVAL; } +static ssize_t target_show(struct kobject *kobj, + struct fragment_attribute *fattr, char *buf) +{ + struct of_overlay_info *ovinfo = fattr->ovinfo; + + return snprintf(buf, PAGE_SIZE, "%s\n", + of_node_full_name(ovinfo->target)); +} + +static const struct fragment_attribute target_template_attr = __ATTR_RO(target); + /** * of_build_overlay_info() - Build an overlay info array * @ov Overlay to build @@ -388,7 +500,7 @@ static int of_build_overlay_info(struct of_overlay *ov, { struct device_node *node; struct of_overlay_info *ovinfo; - int cnt, err; + int i, cnt, err; /* worst case; every child is a node */ cnt = 0; @@ -425,14 +537,45 @@ static int of_build_overlay_info(struct of_overlay *ov, /* if nothing filled, return error */ if (cnt == 0) { - kfree(ovinfo); - return -ENODEV; + err = -ENODEV; + goto err_free_ovinfo; } ov->count = cnt; ov->ovinfo_tab = ovinfo; + ov->attr_groups = kcalloc(cnt + 1, + sizeof(struct attribute_group *), GFP_KERNEL); + if (ov->attr_groups == NULL) { + err = -ENOMEM; + goto err_free_ovinfo; + } + + for (i = 0; i < cnt; i++) { + ovinfo = &ov->ovinfo_tab[i]; + + ov->attr_groups[i] = &ovinfo->attr_group; + + ovinfo->target_attr = target_template_attr; + /* make lockdep happy */ + sysfs_attr_init(&ovinfo->target_attr.attr); + ovinfo->target_attr.ovinfo = ovinfo; + + ovinfo->attrs[0] = &ovinfo->target_attr.attr; + ovinfo->attrs[1] = NULL; + + /* NOTE: direct reference to the full_name */ + ovinfo->attr_group.name = kbasename(ovinfo->info->full_name); + ovinfo->attr_group.attrs = ovinfo->attrs; + + } + ov->attr_groups[i] = NULL; + return 0; + +err_free_ovinfo: + kfree(ovinfo); + return err; } /** @@ -449,46 +592,201 @@ static int of_free_overlay_info(struct of_overlay *ov) struct of_overlay_info *ovinfo; int i; + /* free attribute groups space */ + kfree(ov->attr_groups); + /* do it in reverse */ for (i = ov->count - 1; i >= 0; i--) { ovinfo = &ov->ovinfo_tab[i]; of_node_put(ovinfo->target); of_node_put(ovinfo->overlay); + of_node_put(ovinfo->info); } kfree(ov->ovinfo_tab); return 0; } +static int of_overlay_add_symbols( + struct device_node *tree, + struct of_overlay *ov) +{ + struct of_overlay_info *ovinfo; + struct device_node *root_sym = NULL; + struct device_node *child = NULL; + struct property *prop; + const char *path, *s; + char *new_path; + int i, len, err; + + /* both may fail (if no fixups are required) */ + root_sym = of_find_node_by_path("/__symbols__"); + child = of_get_child_by_name(tree, "__symbols__"); + + err = 0; + /* do nothing if either is NULL */ + if (!root_sym || !child) + goto out; + + for_each_property_of_node(child, prop) { + + /* skip properties added automatically */ + if (of_prop_cmp(prop->name, "name") == 0) + continue; + + err = of_property_read_string(child, + prop->name, &path); + if (err != 0) { + pr_err("Could not find symbol '%s'\n", prop->name); + continue; + } + + /* now find fragment index */ + s = path; + + /* compare paths to find fragment index */ + for (i = 0, ovinfo = NULL, len = -1; i < ov->count; i++) { + ovinfo = &ov->ovinfo_tab[i]; + + pr_debug("#%d: overlay->name=%s target->name=%s\n", + i, ovinfo->overlay->full_name, + ovinfo->target->full_name); + + len = strlen(ovinfo->overlay->full_name); + if (strncasecmp(path, ovinfo->overlay->full_name, + len) == 0 && path[len] == '/') + break; + } + + if (i >= ov->count) + continue; + + pr_debug("found target at #%d\n", i); + new_path = kasprintf(GFP_KERNEL, "%s%s", + ovinfo->target->full_name, + path + len); + if (!new_path) { + pr_err("Failed to allocate propname for \"%s\"\n", + prop->name); + err = -ENOMEM; + break; + } + + err = of_changeset_add_property_string(&ov->cset, root_sym, + prop->name, new_path); + + /* free always */ + kfree(new_path); + + if (err) { + pr_err("Failed to add property for \"%s\"\n", + prop->name); + break; + } + } + +out: + of_node_put(child); + of_node_put(root_sym); + + return err; +} + static LIST_HEAD(ov_list); static DEFINE_IDR(ov_idr); -/** - * of_overlay_create() - Create and apply an overlay - * @tree: Device node containing all the overlays - * - * Creates and applies an overlay while also keeping track - * of the overlay in a list. This list can be used to prevent - * illegal overlay removals. - * - * Returns the id of the created overlay, or a negative error number - */ -int of_overlay_create(struct device_node *tree) +static inline struct of_overlay *kobj_to_overlay(struct kobject *kobj) +{ + return container_of(kobj, struct of_overlay, kobj); +} + +void of_overlay_release(struct kobject *kobj) +{ + struct of_overlay *ov = kobj_to_overlay(kobj); + + of_node_put(ov->target_root); + kfree(ov); +} + +static ssize_t enable_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&ov_enable)); +} + +static ssize_t enable_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int ret; + bool new_enable; + + ret = strtobool(buf, &new_enable); + if (ret != 0) + return ret; + /* if we've disabled it, no going back */ + if (atomic_read(&ov_enable) == 0) + return -EPERM; + atomic_set(&ov_enable, (int)new_enable); + return count; +} + +static struct kobj_attribute enable_attr = __ATTR_RW(enable); + +static const struct attribute *overlay_global_attrs[] = { + &enable_attr.attr, + NULL +}; + +static ssize_t can_remove_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct of_overlay *ov = kobj_to_overlay(kobj); + + return snprintf(buf, PAGE_SIZE, "%d\n", overlay_removal_is_ok(ov)); +} + +static struct kobj_attribute can_remove_attr = __ATTR_RO(can_remove); + +static struct attribute *overlay_attrs[] = { + &can_remove_attr.attr, + NULL +}; + +static struct kobj_type of_overlay_ktype = { + .release = of_overlay_release, + .sysfs_ops = &kobj_sysfs_ops, /* default kobj sysfs ops */ + .default_attrs = overlay_attrs, +}; + +static struct kset *ov_kset; + +static int __of_overlay_create(struct device_node *tree, + int target_index, struct device_node *target_root) { struct of_overlay *ov; int err, id; + /* administratively disabled */ + if (!atomic_read(&ov_enable)) + return -EPERM; + /* allocate the overlay structure */ ov = kzalloc(sizeof(*ov), GFP_KERNEL); if (ov == NULL) return -ENOMEM; ov->id = -1; + ov->target_index = target_index; + ov->target_root = of_node_get(target_root); + INIT_LIST_HEAD(&ov->node); of_changeset_init(&ov->cset); + /* initialize kobject */ + kobject_init(&ov->kobj, &of_overlay_ktype); + mutex_lock(&of_mutex); id = idr_alloc(&ov_idr, ov, 0, 0, GFP_KERNEL); @@ -518,11 +816,25 @@ int of_overlay_create(struct device_node *tree) if (err) goto err_abort_trans; + err = of_overlay_add_symbols(tree, ov); + if (err) { + pr_err("%s: of_overlay_add_symbols() failed for tree@%s\n", + __func__, tree->full_name); + goto err_abort_trans; + } + /* apply the changeset */ err = __of_changeset_apply(&ov->cset); if (err) goto err_revert_overlay; + ov->kobj.kset = ov_kset; + err = kobject_add(&ov->kobj, NULL, "%d", id); + if (err != 0) { + pr_err("%s: kobject_add() failed for tree@%s\n", + __func__, tree->full_name); + goto err_cancel_overlay; + } /* add to the tail of the overlay list */ list_add_tail(&ov->node, &ov_list); @@ -533,6 +845,8 @@ int of_overlay_create(struct device_node *tree) return id; +err_cancel_overlay: + of_changeset_revert(&ov->cset); err_revert_overlay: err_abort_trans: of_free_overlay_info(ov); @@ -540,13 +854,66 @@ int of_overlay_create(struct device_node *tree) idr_remove(&ov_idr, ov->id); err_destroy_trans: of_changeset_destroy(&ov->cset); + of_node_put(ov->target_root); kfree(ov); mutex_unlock(&of_mutex); return err; } + +/** + * of_overlay_create() - Create and apply an overlay + * @tree: Device node containing all the overlays + * + * Creates and applies an overlay while also keeping track + * of the overlay in a list. This list can be used to prevent + * illegal overlay removals. + * + * Returns the id of the created overlay, or a negative error number + */ +int of_overlay_create(struct device_node *tree) +{ + return __of_overlay_create(tree, 0, NULL); +} EXPORT_SYMBOL_GPL(of_overlay_create); +/** + * of_overlay_create_target_index() - Create and apply an overlay + * @tree: Device node containing all the overlays + * @index: Index to use in the target properties + * + * Creates and applies an overlay while also keeping track + * of the overlay in a list. This list can be used to prevent + * illegal overlay removals. + * + * Returns the id of the created overlay, or a negative error number + */ +int of_overlay_create_target_index(struct device_node *tree, int index) +{ + return __of_overlay_create(tree, index, NULL); +} +EXPORT_SYMBOL_GPL(of_overlay_create_target_index); + +/** + * of_overlay_create_target_root() - Create and apply an overlay + * under which will be limited to target_root + * @tree: Device node containing all the overlays + * @target_root: Target root for the overlay. + * + * Creates and applies an overlay while also keeping track + * of the overlay in a list. This list can be used to prevent + * illegal overlay removals. The overlay is only allowed to + * target nodes under the target_root node. + * + * Returns the id of the created overlay, or an negative error number + */ +int of_overlay_create_target_root(struct device_node *tree, + struct device_node *target_root) +{ + return __of_overlay_create(tree, 0, target_root); +} +EXPORT_SYMBOL_GPL(of_overlay_create_target_root); + /* check whether the given node, lies under the given tree */ static int overlay_subtree_check(struct device_node *tree, struct device_node *dn) @@ -646,12 +1013,14 @@ int of_overlay_destroy(int id) of_overlay_notify(ov, OF_OVERLAY_PRE_REMOVE); list_del(&ov->node); + sysfs_remove_groups(&ov->kobj, ov->attr_groups); __of_changeset_revert(&ov->cset); of_overlay_notify(ov, OF_OVERLAY_POST_REMOVE); of_free_overlay_info(ov); idr_remove(&ov_idr, id); of_changeset_destroy(&ov->cset); - kfree(ov); + + kobject_put(&ov->kobj); err = 0; @@ -681,7 +1050,7 @@ int of_overlay_destroy_all(void) __of_changeset_revert(&ov->cset); of_free_overlay_info(ov); idr_remove(&ov_idr, ov->id); - kfree(ov); + kobject_put(&ov->kobj); } mutex_unlock(&of_mutex); @@ -689,3 +1058,18 @@ int of_overlay_destroy_all(void) return 0; } EXPORT_SYMBOL_GPL(of_overlay_destroy_all); + +/* called from of_init() */ +int of_overlay_init(void) +{ + int rc; + + ov_kset = kset_create_and_add("overlays", NULL, &of_kset->kobj); + if (!ov_kset) + return -ENOMEM; + + rc = sysfs_create_files(&ov_kset->kobj, overlay_global_attrs); + WARN(rc, "%s: error adding global attributes\n", __func__); + + return rc; +} diff --git a/drivers/of/unittest-data/testcases.dts b/drivers/of/unittest-data/testcases.dts index ce49463d9d32b..65b3250d09a36 100644 --- a/drivers/of/unittest-data/testcases.dts +++ b/drivers/of/unittest-data/testcases.dts @@ -76,5 +76,15 @@ target = <0x00000000>; }; }; + overlay16 { + fragment@0 { + target = <0x00000000 0x00000004>; + }; + }; + overlay18 { + fragment@0 { + target = <0x00000000>; + }; + }; }; }; }; diff --git a/drivers/of/unittest-data/tests-overlay.dtsi b/drivers/of/unittest-data/tests-overlay.dtsi index 7b8001ab9f3a7..8b8876d1e86b7 100644 --- a/drivers/of/unittest-data/tests-overlay.dtsi +++ b/drivers/of/unittest-data/tests-overlay.dtsi @@ -111,6 +111,30 @@ }; }; }; + + unittest16: test-unittest16 { + compatible = "unittest"; + status = "disabled"; + reg = <16>; + }; + + unittest17: test-unittest17 { + compatible = "unittest"; + status = "disabled"; + reg = <17>; + }; + + unittest18: test-unittest18 { + compatible = "unittest"; + status = "disabled"; + reg = <18>; + }; + + unittest19: test-unittest19 { + compatible = "unittest"; + status = "disabled"; + reg = <19>; + }; }; }; @@ -326,5 +350,44 @@ }; }; + /* test enable using indirect functionality */ + overlay16 { + fragment@0 { + target = <&unittest17>, <&unittest16>; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test enable using target root (relative path) */ + overlay17 { + fragment@0 { + target-path = "/"; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test enable using target phandle */ + overlay18 { + fragment@0 { + target = <&unittest18>; + __overlay__ { + status = "okay"; + }; + }; + }; + + /* test trying to enable out of root (should fail) */ + overlay19 { + fragment@0 { + target = <&unittest19>; + __overlay__ { + status = "okay"; + }; + }; + }; }; }; diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 7c6aff7618009..7130151db6aa0 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -25,6 +25,9 @@ #include +#include +#include + #include "of_private.h" static struct unittest_results { @@ -608,6 +611,59 @@ static void __init of_unittest_changeset(void) #endif } +static void __init of_unittest_changeset_helper(void) +{ +#ifdef CONFIG_OF_DYNAMIC + struct device_node *n1, *n2, *n21, *parent, *np; + struct of_changeset chgset; + + of_changeset_init(&chgset); + + parent = of_find_node_by_path("/testcase-data/changeset"); + + unittest(parent, "testcase setup failure\n"); + n1 = of_changeset_create_device_node(&chgset, + parent, "/testcase-data/changeset/n1"); + unittest(n1, "testcase setup failure\n"); + n2 = of_changeset_create_device_node(&chgset, + parent, "/testcase-data/changeset/n2"); + unittest(n2, "testcase setup failure\n"); + n21 = of_changeset_create_device_node(&chgset, n2, "%s/%s", + "/testcase-data/changeset/n2", "n21"); + unittest(n21, "testcase setup failure\n"); + + unittest(!of_changeset_add_property_string(&chgset, parent, + "prop-add", "foo"), "fail add prop\n"); + + unittest(!of_changeset_attach_node(&chgset, n1), "fail n1 attach\n"); + unittest(!of_changeset_attach_node(&chgset, n2), "fail n2 attach\n"); + unittest(!of_changeset_attach_node(&chgset, n21), "fail n21 attach\n"); + + unittest(!of_changeset_apply(&chgset), "apply failed\n"); + + /* Make sure node names are constructed correctly */ + np = of_find_node_by_path("/testcase-data/changeset/n1"); + unittest(np, "'%s' not added\n", n1->full_name); + of_node_put(np); + + /* Make sure node names are constructed correctly */ + np = of_find_node_by_path("/testcase-data/changeset/n2"); + unittest(np, "'%s' not added\n", n2->full_name); + of_node_put(np); + + np = of_find_node_by_path("/testcase-data/changeset/n2/n21"); + unittest(np, "'%s' not added\n", n21->full_name); + of_node_put(np); + + unittest(!of_changeset_revert(&chgset), "revert failed\n"); + + of_changeset_destroy(&chgset); + + of_node_put(parent); +#endif +} + + static void __init of_unittest_parse_interrupts(void) { struct device_node *np; @@ -956,7 +1012,7 @@ static int attach_node_and_children(struct device_node *np) of_node_clear_flag(np, OF_DETACHED); raw_spin_unlock_irqrestore(&devtree_lock, flags); - __of_attach_node_sysfs(np); + __of_attach_node_post(np); mutex_unlock(&of_mutex); while (child) { @@ -1014,7 +1070,7 @@ static int __init unittest_data_add(void) if (!of_root) { of_root = unittest_data_node; for_each_of_allnodes(np) - __of_attach_node_sysfs(np); + __of_attach_node_post(np); of_aliases = of_find_node_by_path("/aliases"); of_chosen = of_find_node_by_path("/chosen"); return 0; @@ -1933,6 +1989,273 @@ static inline void of_unittest_overlay_i2c_15(void) { } #endif +static void of_unittest_overlay_16(void) +{ + int ret; + int overlay_nr = 16; + int unittest_nr = 16; + enum overlay_type ovtype = PDEV_OVERLAY; + int before = 0; + int after = 1; + struct device_node *np = NULL; + int id = -1; + + /* unittest device must not be in before state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != before) { + unittest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !before ? "enabled" : "disabled"); + return; + } + + np = of_find_node_by_path(overlay_path(overlay_nr)); + if (np == NULL) { + unittest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr)); + ret = -EINVAL; + goto out; + } + + /* unittest16 is at index #1 */ + ret = of_overlay_create_target_index(np, 1); + if (ret < 0) { + unittest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr)); + goto out; + } + id = ret; + of_unittest_track_overlay(id); + + ret = 0; + +out: + of_node_put(np); + + if (ret) + return; + + /* unittest device must be to set to after state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != after) { + unittest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !after ? "enabled" : "disabled"); + return; + } + + unittest(1, "overlay test %d passed\n", 16); +} + +static void of_unittest_overlay_17(void) +{ + int ret; + int overlay_nr = 17; + int unittest_nr = 17; + enum overlay_type ovtype = PDEV_OVERLAY; + int before = 0; + int after = 1; + const char *root_path; + struct device_node *np = NULL, *target_root = NULL; + int id = -1; + + /* unittest device must not be in before state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != before) { + unittest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !before ? "enabled" : "disabled"); + return; + } + + np = of_find_node_by_path(overlay_path(overlay_nr)); + if (np == NULL) { + unittest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr)); + ret = -EINVAL; + goto out; + } + + root_path = "/testcase-data/overlay-node/test-bus/test-unittest17"; + target_root = of_find_node_by_path(root_path); + if (!target_root) { + unittest(0, "could not find target_root node @\"%s\"\n", + root_path); + ret = -EINVAL; + goto out; + } + + ret = of_overlay_create_target_root(np, target_root); + of_node_put(target_root); + + if (ret < 0) { + unittest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr)); + goto out; + } + id = ret; + of_unittest_track_overlay(id); + + ret = 0; + +out: + of_node_put(np); + + if (ret) + return; + + /* unittest device must be to set to after state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != after) { + unittest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !after ? "enabled" : "disabled"); + return; + } + + unittest(1, "overlay test %d passed\n", 17); +} + +static void of_unittest_overlay_18(void) +{ + int ret; + int overlay_nr = 18; + int unittest_nr = 18; + enum overlay_type ovtype = PDEV_OVERLAY; + int before = 0; + int after = 1; + const char *root_path; + struct device_node *np = NULL, *target_root = NULL; + int id = -1; + + /* unittest device must not be in before state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != before) { + unittest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !before ? "enabled" : "disabled"); + return; + } + + np = of_find_node_by_path(overlay_path(overlay_nr)); + if (np == NULL) { + unittest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr)); + ret = -EINVAL; + goto out; + } + + root_path = "/testcase-data/overlay-node/test-bus/test-unittest18"; + target_root = of_find_node_by_path(root_path); + if (!target_root) { + unittest(0, "could not find target_root node @\"%s\"\n", + root_path); + ret = -EINVAL; + goto out; + } + + ret = of_overlay_create_target_root(np, target_root); + of_node_put(target_root); + + if (ret < 0) { + unittest(0, "could not create overlay from \"%s\"\n", + overlay_path(overlay_nr)); + goto out; + } + id = ret; + of_unittest_track_overlay(id); + + ret = 0; + +out: + of_node_put(np); + + if (ret) + return; + + /* unittest device must be to set to after state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != after) { + unittest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !after ? "enabled" : "disabled"); + return; + } + + unittest(1, "overlay test %d passed\n", 18); +} + +static void of_unittest_overlay_19(void) +{ + int ret; + int overlay_nr = 19; + int unittest_nr = 19; + enum overlay_type ovtype = PDEV_OVERLAY; + int before = 0; + int after = 0; + const char *root_path; + struct device_node *np = NULL, *target_root = NULL; + int id = -1; + + /* unittest device must not be in before state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != before) { + unittest(0, "overlay @\"%s\" with device @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !before ? "enabled" : "disabled"); + return; + } + + np = of_find_node_by_path(overlay_path(overlay_nr)); + if (np == NULL) { + unittest(0, "could not find overlay node @\"%s\"\n", + overlay_path(overlay_nr)); + ret = -EINVAL; + goto out; + } + + root_path = "/testcase-data/overlay-node/test-bus/test-unittest19"; + target_root = of_find_node_by_path(root_path); + if (!target_root) { + unittest(0, "could not find target_root node @\"%s\"\n", + root_path); + ret = -EINVAL; + goto out; + } + + ret = of_overlay_create_target_root(np, target_root); + of_node_put(target_root); + + if (ret >= 0) { + unittest(0, "created overlay from \"%s\" while we shouldn't\n", + overlay_path(overlay_nr)); + id = ret; + of_unittest_track_overlay(id); + ret = -EINVAL; + goto out; + } + + ret = 0; + +out: + of_node_put(np); + + if (ret) + return; + + /* unittest device must be to set to after state */ + if (of_unittest_device_exists(unittest_nr, ovtype) != after) { + unittest(0, "overlay @\"%s\" failed to create @\"%s\" %s\n", + overlay_path(overlay_nr), + unittest_path(unittest_nr, ovtype), + !after ? "enabled" : "disabled"); + return; + } + + unittest(1, "overlay test %d passed\n", 16); +} + + static void __init of_unittest_overlay(void) { struct device_node *bus_np = NULL; @@ -1983,6 +2306,12 @@ static void __init of_unittest_overlay(void) of_unittest_overlay_10(); of_unittest_overlay_11(); + of_unittest_overlay_16(); + + of_unittest_overlay_17(); + of_unittest_overlay_18(); + of_unittest_overlay_19(); + #if IS_BUILTIN(CONFIG_I2C) if (unittest(of_unittest_overlay_i2c_init() == 0, "i2c init failed\n")) goto out; @@ -2329,6 +2658,70 @@ static inline __init void of_unittest_overlay_high_level(void) {} #endif +#define PHANDLE_LOOKUPS 1000 + +static void __init of_unittest_phandle_hash(void) +{ + struct device_node *node; + phandle max_phandle; + u32 ph; + unsigned long flags; + int i, j, total; + ktime_t start, end; + s64 dur[2]; + int dec, frac; + + /* test only available when hashing is available */ + if (!of_phandle_ht_available()) { + pr_warn("phandle hash test requires hash to be initialized\n"); + return; + } + + /* find the maximum phandle of the tree */ + raw_spin_lock_irqsave(&devtree_lock, flags); + max_phandle = 0; + total = 0; + for_each_of_allnodes(node) { + if (node->phandle != (phandle)-1U && + node->phandle > max_phandle) + max_phandle = node->phandle; + total++; + } + raw_spin_unlock_irqrestore(&devtree_lock, flags); + max_phandle++; + + pr_debug("phandle: max-phandle #%u, #%d total nodes\n", + (u32)max_phandle, total); + + /* perform random lookups using the hash */ + for (j = 0; j < 2; j++) { + + /* disabled for pass #0, enabled for pass #1 */ + of_phandle_ht_is_disabled = j == 0; + + start = ktime_get_raw(); + for (i = 0; i < PHANDLE_LOOKUPS; i++) { + ph = prandom_u32() % max_phandle; + node = of_find_node_by_phandle(ph); + of_node_put(node); + } + end = ktime_get_raw(); + + dur[j] = ktime_to_us(end) - ktime_to_us(start); + pr_debug("#%d lookups in %lld us (%s)\n", + PHANDLE_LOOKUPS, dur[j], + j == 0 ? "original" : "hashed"); + } + + unittest(dur[0] > dur[1], "Non hashing phandles are faster!?"); + + dec = (int)div64_s64(dur[0] * 10 + 5, dur[1]); + frac = dec % 10; + dec /= 10; + pr_info("the hash method is %d.%d times faster than the original\n", + dec, frac); +} + static int __init of_unittest(void) { struct device_node *np; @@ -2358,11 +2751,13 @@ static int __init of_unittest(void) of_unittest_property_string(); of_unittest_property_copy(); of_unittest_changeset(); + of_unittest_changeset_helper(); of_unittest_parse_interrupts(); of_unittest_parse_interrupts_extended(); of_unittest_match_node(); of_unittest_platform_populate(); of_unittest_overlay(); + of_unittest_phandle_hash(); /* Double check linkage after removing testcase data */ of_unittest_check_tree_linkage(); diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c index 22558bf294246..3b3ec4a8d6026 100644 --- a/drivers/pinctrl/pinctrl-mcp23s08.c +++ b/drivers/pinctrl/pinctrl-mcp23s08.c @@ -836,10 +836,13 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev, break; case MCP_TYPE_S18: + one_regmap_config = + devm_kmemdup(dev, &mcp23x17_regmap, + sizeof(struct regmap_config), GFP_KERNEL); if (!one_regmap_config) return -ENOMEM; mcp->regmap = devm_regmap_init(dev, &mcp23sxx_spi_regmap, mcp, - &mcp23x17_regmap); + one_regmap_config); mcp->reg_shift = 1; mcp->chip.ngpio = 16; mcp->chip.label = "mcp23s18"; diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index a813239300c3d..7a7b9d8ae4e54 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -23,6 +23,8 @@ #include #include +static struct class pwm_class; + struct pwm_export { struct device child; struct pwm_device *pwm; @@ -239,6 +241,10 @@ static struct attribute *pwm_attrs[] = { }; ATTRIBUTE_GROUPS(pwm); +static const struct device_type pwm_channel_type = { + .name = "pwm_channel", +}; + static void pwm_export_release(struct device *child) { struct pwm_export *export = child_to_pwm_export(child); @@ -248,6 +254,7 @@ static void pwm_export_release(struct device *child) static int pwm_export_child(struct device *parent, struct pwm_device *pwm) { + struct pwm_chip *chip = dev_get_drvdata(parent); struct pwm_export *export; int ret; @@ -265,9 +272,11 @@ static int pwm_export_child(struct device *parent, struct pwm_device *pwm) export->child.release = pwm_export_release; export->child.parent = parent; + export->child.type = &pwm_channel_type; export->child.devt = MKDEV(0, 0); + export->child.class = &pwm_class; export->child.groups = pwm_groups; - dev_set_name(&export->child, "pwm%u", pwm->hwpwm); + dev_set_name(&export->child, "pwm-%d:%u", chip->base, pwm->hwpwm); ret = device_register(&export->child); if (ret) { @@ -372,7 +381,6 @@ ATTRIBUTE_GROUPS(pwm_chip); static struct class pwm_class = { .name = "pwm", .owner = THIS_MODULE, - .dev_groups = pwm_chip_groups, }; static int pwmchip_sysfs_match(struct device *parent, const void *data) @@ -388,7 +396,8 @@ void pwmchip_sysfs_export(struct pwm_chip *chip) * If device_create() fails the pwm_chip is still usable by * the kernel its just not exported. */ - parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip, + parent = device_create_with_groups(&pwm_class, chip->dev, MKDEV(0, 0), + chip, pwm_chip_groups, "pwmchip%d", chip->base); if (IS_ERR(parent)) { dev_warn(chip->dev, diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index 3ebbcd591d8d7..9086f8f93acc2 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -929,6 +929,14 @@ static void *pru_i_da_to_va(struct pru_rproc *pru, u32 da, int len) u32 offset; void *va = NULL; + /* GNU binutils do not support multiple address spaces. The + * default linker script from the official GNU pru-ld places + * IRAM at an arbitrary high offset, in order to differentiate it + * from DRAM. Hence we need to strip the artificial offset + * from the IRAM address. + */ + da &= ~0xf0000000u; + if (len <= 0) return NULL; diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index cda10719d1d1b..bad88bfb69f2d 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -724,11 +724,11 @@ static int spidev_probe(struct spi_device *spi) * compatible string, it is a Linux implementation thing * rather than a description of the hardware. */ - if (spi->dev.of_node && !of_match_device(spidev_dt_ids, &spi->dev)) { - dev_err(&spi->dev, "buggy DT: spidev listed directly in DT\n"); - WARN_ON(spi->dev.of_node && - !of_match_device(spidev_dt_ids, &spi->dev)); - } +// if (spi->dev.of_node && !of_match_device(spidev_dt_ids, &spi->dev)) { +// dev_err(&spi->dev, "buggy DT: spidev listed directly in DT\n"); +// WARN_ON(spi->dev.of_node && +// !of_match_device(spidev_dt_ids, &spi->dev)); +// } spidev_probe_acpi(spi); diff --git a/drivers/staging/fbtft/fb_ssd1306.c b/drivers/staging/fbtft/fb_ssd1306.c index 96c58de85288e..602f2b5c46818 100644 --- a/drivers/staging/fbtft/fb_ssd1306.c +++ b/drivers/staging/fbtft/fb_ssd1306.c @@ -64,6 +64,8 @@ static int init_display(struct fbtft_par *par) write_reg(par, 0x3F); else if (par->info->var.yres == 48) write_reg(par, 0x2F); + else if (par->info->var.yres == 39) + write_reg(par, 0x27); /* https://libstock.mikroe.com/projects/download/1111/2577/1411057038_oled_b_click___e_mikroc_arm.zip */ else write_reg(par, 0x1F); @@ -81,16 +83,24 @@ static int init_display(struct fbtft_par *par) /* Set Memory Addressing Mode */ write_reg(par, 0x20); - /* Vertical addressing mode */ + /* Vertical Addressing mode */ write_reg(par, 0x01); /* Set Segment Re-map */ - /* column address 127 is mapped to SEG0 */ - write_reg(par, 0xA0 | 0x1); + if (par->info->var.yres == 39) + /* no segment re-map */ + write_reg(par, 0xA0 | 0x0); + else + /* column address 127 is mapped to SEG0 */ + write_reg(par, 0xA0 | 0x1); /* Set COM Output Scan Direction */ - /* remapped mode. Scan from COM[N-1] to COM0 */ - write_reg(par, 0xC8); + if (par->info->var.yres == 39) + /* no columnt re-map mode. Scan from COM0 to COM[N-1] */ + write_reg(par, 0xC0 | 0x0); + else + /* remapped mode. Scan from COM[N-1] to COM0 */ + write_reg(par, 0xC0 | 0x8); /* Set COM Pins Hardware Configuration */ write_reg(par, 0xDA); @@ -100,18 +110,27 @@ static int init_display(struct fbtft_par *par) else if (par->info->var.yres == 48) /* A[4]=1b, Alternative COM pin configuration */ write_reg(par, 0x12); + else if (par->info->var.yres == 39) + /* A[4]=1b, Alternative COM pin configuration */ + write_reg(par, 0x12); else /* A[4]=0b, Sequential COM pin configuration */ write_reg(par, 0x02); /* Set Pre-charge Period */ write_reg(par, 0xD9); - write_reg(par, 0xF1); + if (par->info->var.yres == 39) + write_reg(par, 0x25); + else + write_reg(par, 0xF1); /* Set VCOMH Deselect Level */ write_reg(par, 0xDB); - /* according to the datasheet, this value is out of bounds */ - write_reg(par, 0x40); + if (par->info->var.yres == 39) + write_reg(par, 0x20); + else + /* according to the datasheet, this value is out of bounds */ + write_reg(par, 0x40); /* Entire Display ON */ /* Resume to RAM content display. Output follows RAM content */ @@ -142,17 +161,35 @@ static void set_addr_win_64x48(struct fbtft_par *par) write_reg(par, 0x5); } +static void set_addr_win_96x39(struct fbtft_par *par) +{ + /* Set Page Address */ + write_reg(par, 0xB0); + + /* Set Column Address */ + write_reg(par, 0x21); + write_reg(par, 0x00); + write_reg(par, 0x5F); + + /* Set Page Address Range */ + write_reg(par, 0x22); + write_reg(par, 0x0); + write_reg(par, 0x4); +} + static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye) { - /* Set Lower Column Start Address for Page Addressing Mode */ + /* Set Lower Column Start Address for Vertical Addressing Mode */ write_reg(par, 0x00 | 0x0); - /* Set Higher Column Start Address for Page Addressing Mode */ + /* Set Higher Column Start Address for Vertical Addressing Mode */ write_reg(par, 0x10 | 0x0); /* Set Display Start Line */ write_reg(par, 0x40 | 0x0); if (par->info->var.xres == 64 && par->info->var.yres == 48) set_addr_win_64x48(par); + else if (par->info->var.xres == 96 && par->info->var.yres == 39) + set_addr_win_96x39(par); } static int blank(struct fbtft_par *par, bool on) @@ -196,11 +233,19 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len) *buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i; buf++; } + if (yres % 8) { + *buf = 0x00; + for (i = 0; i < (yres - (y * 8)); i++) + *buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i; + buf++; + y++; + } } /* Write data */ gpio_set_value(par->gpio.dc, 1); - ret = par->fbtftops.write(par, par->txbuf.buf, xres * yres / 8); + ret = par->fbtftops.write(par, par->txbuf.buf, xres * (yres / 8 + (yres % 8 != 0))); + //ret = par->fbtftops.write(par, par->txbuf.buf, xres * 128 / 8); if (ret < 0) dev_err(par->info->device, "write failed and returned: %d\n", ret); diff --git a/drivers/staging/fbtft/fbtft_device.c b/drivers/staging/fbtft/fbtft_device.c index 0d974738c1c4e..c60d7505813c4 100644 --- a/drivers/staging/fbtft/fbtft_device.c +++ b/drivers/staging/fbtft/fbtft_device.c @@ -391,6 +391,25 @@ static struct fbtft_device_display displays[] = { }, } } + }, { + .name = "mikroe_oledb", + .spi = &(struct spi_board_info) { + .modalias = "fb_ssd1306", + .max_speed_hz = 16000000, + .mode = SPI_MODE_0, + .platform_data = &(struct fbtft_platform_data) { + .display = { + .width = 96, + .height = 39, + .buswidth = 8, + }, + .gpios = (const struct fbtft_gpio []) { + { "reset", 45 }, + { "dc", 110 }, + {}, + }, + } + } }, { .name = "admatec_c-berry28", .spi = &(struct spi_board_info) { diff --git a/drivers/staging/greybus/Documentation/firmware/authenticate.c b/drivers/staging/greybus/Documentation/firmware/authenticate.c index b836f0a20c367..806e75b7f4051 100644 --- a/drivers/staging/greybus/Documentation/firmware/authenticate.c +++ b/drivers/staging/greybus/Documentation/firmware/authenticate.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * Sample code to test CAP protocol * diff --git a/drivers/staging/greybus/Documentation/firmware/firmware.c b/drivers/staging/greybus/Documentation/firmware/firmware.c index c73dee9d13c17..31d9c23e2eeb1 100644 --- a/drivers/staging/greybus/Documentation/firmware/firmware.c +++ b/drivers/staging/greybus/Documentation/firmware/firmware.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * Sample code to test firmware-management protocol * diff --git a/drivers/staging/greybus/TODO b/drivers/staging/greybus/TODO new file mode 100644 index 0000000000000..3b90a5711998f --- /dev/null +++ b/drivers/staging/greybus/TODO @@ -0,0 +1,5 @@ +* Convert all uses of the old GPIO API from to the + GPIO descriptor API in and look up GPIO + lines from device tree or ACPI. +* Convert the GPIO driver to use the GPIO irqchip library + GPIOLIB_IRQCHIP instead of reimplementing the same. diff --git a/drivers/staging/greybus/arche-apb-ctrl.c b/drivers/staging/greybus/arche-apb-ctrl.c index 0412f3d06efb8..cc8d6fc831b41 100644 --- a/drivers/staging/greybus/arche-apb-ctrl.c +++ b/drivers/staging/greybus/arche-apb-ctrl.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Arche Platform driver to control APB. * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -73,14 +72,14 @@ static int coldboot_seq(struct platform_device *pdev) int ret; if (apb->init_disabled || - apb->state == ARCHE_PLATFORM_STATE_ACTIVE) + apb->state == ARCHE_PLATFORM_STATE_ACTIVE) return 0; /* Hold APB in reset state */ assert_reset(apb->resetn_gpio); if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING && - gpio_is_valid(apb->spi_en_gpio)) + gpio_is_valid(apb->spi_en_gpio)) devm_gpio_free(dev, apb->spi_en_gpio); /* Enable power to APB */ @@ -123,7 +122,7 @@ static int fw_flashing_seq(struct platform_device *pdev) int ret; if (apb->init_disabled || - apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING) + apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING) return 0; ret = regulator_enable(apb->vcore); @@ -147,7 +146,7 @@ static int fw_flashing_seq(struct platform_device *pdev) flags = GPIOF_OUT_INIT_LOW; ret = devm_gpio_request_one(dev, apb->spi_en_gpio, - flags, "apb_spi_en"); + flags, "apb_spi_en"); if (ret) { dev_err(dev, "Failed requesting SPI bus en gpio %d\n", apb->spi_en_gpio); @@ -175,11 +174,11 @@ static int standby_boot_seq(struct platform_device *pdev) * then we do not want to change the state */ if (apb->state == ARCHE_PLATFORM_STATE_STANDBY || - apb->state == ARCHE_PLATFORM_STATE_OFF) + apb->state == ARCHE_PLATFORM_STATE_OFF) return 0; if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING && - gpio_is_valid(apb->spi_en_gpio)) + gpio_is_valid(apb->spi_en_gpio)) devm_gpio_free(dev, apb->spi_en_gpio); /* @@ -204,7 +203,7 @@ static void poweroff_seq(struct platform_device *pdev) return; if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING && - gpio_is_valid(apb->spi_en_gpio)) + gpio_is_valid(apb->spi_en_gpio)) devm_gpio_free(dev, apb->spi_en_gpio); /* disable the clock */ @@ -252,7 +251,8 @@ void apb_ctrl_poweroff(struct device *dev) } static ssize_t state_store(struct device *dev, - struct device_attribute *attr, const char *buf, size_t count) + struct device_attribute *attr, + const char *buf, size_t count) { struct platform_device *pdev = to_platform_device(dev); struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev); @@ -298,7 +298,7 @@ static ssize_t state_store(struct device *dev, } static ssize_t state_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { struct arche_apb_ctrl_drvdata *apb = dev_get_drvdata(dev); @@ -320,7 +320,7 @@ static ssize_t state_show(struct device *dev, static DEVICE_ATTR_RW(state); static int apb_ctrl_get_devtree_data(struct platform_device *pdev, - struct arche_apb_ctrl_drvdata *apb) + struct arche_apb_ctrl_drvdata *apb) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; @@ -332,10 +332,10 @@ static int apb_ctrl_get_devtree_data(struct platform_device *pdev, return apb->resetn_gpio; } ret = devm_gpio_request_one(dev, apb->resetn_gpio, - GPIOF_OUT_INIT_LOW, "apb-reset"); + GPIOF_OUT_INIT_LOW, "apb-reset"); if (ret) { dev_err(dev, "Failed requesting reset gpio %d\n", - apb->resetn_gpio); + apb->resetn_gpio); return ret; } @@ -345,10 +345,10 @@ static int apb_ctrl_get_devtree_data(struct platform_device *pdev, return apb->boot_ret_gpio; } ret = devm_gpio_request_one(dev, apb->boot_ret_gpio, - GPIOF_OUT_INIT_LOW, "boot retention"); + GPIOF_OUT_INIT_LOW, "boot retention"); if (ret) { dev_err(dev, "Failed requesting bootret gpio %d\n", - apb->boot_ret_gpio); + apb->boot_ret_gpio); return ret; } @@ -359,10 +359,10 @@ static int apb_ctrl_get_devtree_data(struct platform_device *pdev, return apb->pwroff_gpio; } ret = devm_gpio_request_one(dev, apb->pwroff_gpio, - GPIOF_IN, "pwroff_n"); + GPIOF_IN, "pwroff_n"); if (ret) { dev_err(dev, "Failed requesting pwroff_n gpio %d\n", - apb->pwroff_gpio); + apb->pwroff_gpio); return ret; } @@ -372,10 +372,10 @@ static int apb_ctrl_get_devtree_data(struct platform_device *pdev, dev_warn(dev, "failed to get clock en gpio\n"); } else if (gpio_is_valid(apb->clk_en_gpio)) { ret = devm_gpio_request_one(dev, apb->clk_en_gpio, - GPIOF_OUT_INIT_LOW, "apb_clk_en"); + GPIOF_OUT_INIT_LOW, "apb_clk_en"); if (ret) { dev_warn(dev, "Failed requesting APB clock en gpio %d\n", - apb->clk_en_gpio); + apb->clk_en_gpio); return ret; } } @@ -408,7 +408,7 @@ static int apb_ctrl_get_devtree_data(struct platform_device *pdev, apb->spi_en_gpio = of_get_named_gpio(np, "spi-en-gpio", 0); if (apb->spi_en_gpio >= 0) { if (of_property_read_bool(pdev->dev.of_node, - "spi-en-active-high")) + "spi-en-active-high")) apb->spi_en_polarity_high = true; } diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c index 21ac92d0f5334..4c36e88766e75 100644 --- a/drivers/staging/greybus/arche-platform.c +++ b/drivers/staging/greybus/arche-platform.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Arche Platform driver to enable Unipro link. * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -234,7 +233,7 @@ arche_platform_coldboot_seq(struct arche_platform_drvdata *arche_pdata) ret = clk_prepare_enable(arche_pdata->svc_ref_clk); if (ret) { dev_err(arche_pdata->dev, "failed to enable svc_ref_clk: %d\n", - ret); + ret); return ret; } @@ -270,7 +269,7 @@ arche_platform_fw_flashing_seq(struct arche_platform_drvdata *arche_pdata) ret = clk_prepare_enable(arche_pdata->svc_ref_clk); if (ret) { dev_err(arche_pdata->dev, "failed to enable svc_ref_clk: %d\n", - ret); + ret); return ret; } @@ -313,10 +312,10 @@ arche_platform_poweroff_seq(struct arche_platform_drvdata *arche_pdata) } static ssize_t state_store(struct device *dev, - struct device_attribute *attr, const char *buf, size_t count) + struct device_attribute *attr, + const char *buf, size_t count) { - struct platform_device *pdev = to_platform_device(dev); - struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev); + struct arche_platform_drvdata *arche_pdata = dev_get_drvdata(dev); int ret = 0; mutex_lock(&arche_pdata->platform_state_mutex); @@ -377,7 +376,7 @@ static ssize_t state_store(struct device *dev, } static ssize_t state_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { struct arche_platform_drvdata *arche_pdata = dev_get_drvdata(dev); @@ -444,11 +443,11 @@ static int arche_platform_probe(struct platform_device *pdev) /* setup svc reset gpio */ arche_pdata->is_reset_act_hi = of_property_read_bool(np, - "svc,reset-active-high"); + "svc,reset-active-high"); arche_pdata->svc_reset_gpio = of_get_named_gpio(np, "svc,reset-gpio", 0); - if (arche_pdata->svc_reset_gpio < 0) { + if (!gpio_is_valid(arche_pdata->svc_reset_gpio)) { dev_err(dev, "failed to get reset-gpio\n"); return arche_pdata->svc_reset_gpio; } @@ -458,7 +457,7 @@ static int arche_platform_probe(struct platform_device *pdev) return ret; } ret = gpio_direction_output(arche_pdata->svc_reset_gpio, - arche_pdata->is_reset_act_hi); + arche_pdata->is_reset_act_hi); if (ret) { dev_err(dev, "failed to set svc-reset gpio dir:%d\n", ret); return ret; @@ -466,8 +465,9 @@ static int arche_platform_probe(struct platform_device *pdev) arche_platform_set_state(arche_pdata, ARCHE_PLATFORM_STATE_OFF); arche_pdata->svc_sysboot_gpio = of_get_named_gpio(np, - "svc,sysboot-gpio", 0); - if (arche_pdata->svc_sysboot_gpio < 0) { + "svc,sysboot-gpio", + 0); + if (!gpio_is_valid(arche_pdata->svc_sysboot_gpio)) { dev_err(dev, "failed to get sysboot gpio\n"); return arche_pdata->svc_sysboot_gpio; } @@ -484,8 +484,9 @@ static int arche_platform_probe(struct platform_device *pdev) /* setup the clock request gpio first */ arche_pdata->svc_refclk_req = of_get_named_gpio(np, - "svc,refclk-req-gpio", 0); - if (arche_pdata->svc_refclk_req < 0) { + "svc,refclk-req-gpio", + 0); + if (!gpio_is_valid(arche_pdata->svc_refclk_req)) { dev_err(dev, "failed to get svc clock-req gpio\n"); return arche_pdata->svc_refclk_req; } @@ -526,7 +527,7 @@ static int arche_platform_probe(struct platform_device *pdev) "wake detect"); if (ret) { dev_err(dev, "Failed requesting wake_detect gpio %d\n", - arche_pdata->wake_detect_gpio); + arche_pdata->wake_detect_gpio); return ret; } diff --git a/drivers/staging/greybus/arche_platform.h b/drivers/staging/greybus/arche_platform.h index bcffc69d09609..02056351d25a4 100644 --- a/drivers/staging/greybus/arche_platform.h +++ b/drivers/staging/greybus/arche_platform.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Arche Platform driver to enable Unipro link. * * Copyright 2015-2016 Google Inc. * Copyright 2015-2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __ARCHE_PLATFORM_H diff --git a/drivers/staging/greybus/arpc.h b/drivers/staging/greybus/arpc.h index c0b63c0130c59..3534ba1a4e6ca 100644 --- a/drivers/staging/greybus/arpc.h +++ b/drivers/staging/greybus/arpc.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. diff --git a/drivers/staging/greybus/audio_apbridgea.c b/drivers/staging/greybus/audio_apbridgea.c index 1b4252d5d2553..7ebb1bde5cb77 100644 --- a/drivers/staging/greybus/audio_apbridgea.c +++ b/drivers/staging/greybus/audio_apbridgea.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Audio Device Class Protocol helpers * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #include "greybus.h" diff --git a/drivers/staging/greybus/audio_apbridgea.h b/drivers/staging/greybus/audio_apbridgea.h index b94cb05c89e48..42ac6059bfc7a 100644 --- a/drivers/staging/greybus/audio_apbridgea.h +++ b/drivers/staging/greybus/audio_apbridgea.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: BSD-3-Clause /** * Copyright (c) 2015-2016 Google Inc. * All rights reserved. diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c index a6d01f0761f32..35acd55ca5ab7 100644 --- a/drivers/staging/greybus/audio_codec.c +++ b/drivers/staging/greybus/audio_codec.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * APBridge ALSA SoC dummy codec driver * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include #include @@ -48,7 +47,7 @@ static int gbaudio_module_enable_tx(struct gbaudio_codec_info *codec, int module_state, ret = 0; u16 data_cport, i2s_port, cportid; u8 sig_bits, channels; - uint32_t format, rate; + u32 format, rate; struct gbaudio_data_connection *data; struct gbaudio_stream_params *params; @@ -183,7 +182,7 @@ static int gbaudio_module_enable_rx(struct gbaudio_codec_info *codec, int module_state, ret = 0; u16 data_cport, i2s_port, cportid; u8 sig_bits, channels; - uint32_t format, rate; + u32 format, rate; struct gbaudio_data_connection *data; struct gbaudio_stream_params *params; @@ -320,7 +319,7 @@ int gbaudio_module_update(struct gbaudio_codec_info *codec, char intf_name[NAME_SIZE], dir[NAME_SIZE]; dev_dbg(module->dev, "%s:Module update %s sequence\n", w->name, - enable ? "Enable":"Disable"); + enable ? "Enable" : "Disable"); if ((w->id != snd_soc_dapm_aif_in) && (w->id != snd_soc_dapm_aif_out)) { dev_dbg(codec->dev, "No action required for %s\n", w->name); @@ -413,7 +412,7 @@ static int gbcodec_hw_params(struct snd_pcm_substream *substream, { int ret; u8 sig_bits, channels; - uint32_t format, rate; + u32 format, rate; struct gbaudio_module_info *module; struct gbaudio_data_connection *data; struct gb_bundle *bundle; @@ -568,7 +567,7 @@ static int gbcodec_prepare(struct snd_pcm_substream *substream, if (ret) { mutex_unlock(&codec->lock); dev_err_ratelimited(dai->dev, "set_data_size failed:%d\n", - ret); + ret); return ret; } @@ -588,9 +587,8 @@ static int gbcodec_mute_stream(struct snd_soc_dai *dai, int mute, int stream) struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev); struct gbaudio_stream_params *params; - dev_dbg(dai->dev, "Mute:%d, Direction:%s\n", mute, - stream ? "CAPTURE":"PLAYBACK"); + stream ? "CAPTURE" : "PLAYBACK"); mutex_lock(&codec->lock); @@ -828,7 +826,7 @@ int gbaudio_register_module(struct gbaudio_module_info *module) module->num_dapm_widgets); if (module->controls) snd_soc_add_codec_controls(codec, module->controls, - module->num_controls); + module->num_controls); if (module->dapm_routes) snd_soc_dapm_add_routes(&codec->dapm, module->dapm_routes, module->num_dapm_routes); @@ -843,8 +841,8 @@ int gbaudio_register_module(struct gbaudio_module_info *module) * from codec->jack_list */ list_for_each_entry(jack, &codec->jack_list, list) { - if ((jack == &module->headset_jack) - || (jack == &module->button_jack)) + if ((jack == &module->headset_jack) || + (jack == &module->button_jack)) snd_device_register(codec->card->snd_card, jack->jack); } @@ -908,7 +906,6 @@ static void gbaudio_codec_clean_data_rx(struct gbaudio_data_connection *data) data->state[1] = GBAUDIO_CODEC_SHUTDOWN; } - static void gbaudio_codec_cleanup(struct gbaudio_module_info *module) { struct gbaudio_data_connection *data; @@ -924,7 +921,6 @@ static void gbaudio_codec_cleanup(struct gbaudio_module_info *module) if (cap_state > GBAUDIO_CODEC_SHUTDOWN) gbaudio_codec_clean_data_rx(data); - } } @@ -973,7 +969,7 @@ void gbaudio_unregister_module(struct gbaudio_module_info *module) dev_dbg(codec->dev, "Removing %d controls\n", module->num_controls); snd_soc_remove_codec_controls(codec, module->controls, - module->num_controls); + module->num_controls); } if (module->dapm_widgets) { dev_dbg(codec->dev, "Removing %d widgets\n", diff --git a/drivers/staging/greybus/audio_codec.h b/drivers/staging/greybus/audio_codec.h index 6fb064c69a364..4efd8b3ebe07b 100644 --- a/drivers/staging/greybus/audio_codec.h +++ b/drivers/staging/greybus/audio_codec.h @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus audio driver * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __LINUX_GBAUDIO_CODEC_H @@ -24,7 +23,10 @@ enum { NUM_CODEC_DAIS, }; -/* device_type should be same as defined in audio.h (Android media layer) */ +/* + * device_type should be same as defined in audio.h + * (Android media layer) + */ enum { GBAUDIO_DEVICE_NONE = 0x0, /* reserved bits */ @@ -54,7 +56,7 @@ enum gbaudio_codec_state { struct gbaudio_stream_params { int state; u8 sig_bits, channels; - uint32_t format, rate; + u32 format, rate; }; struct gbaudio_codec_dai { @@ -160,7 +162,7 @@ struct gbaudio_module_info { }; int gbaudio_tplg_parse_data(struct gbaudio_module_info *module, - struct gb_audio_topology *tplg_data); + struct gb_audio_topology *tplg_data); void gbaudio_tplg_release(struct gbaudio_module_info *module); int gbaudio_module_update(struct gbaudio_codec_info *codec, @@ -184,12 +186,12 @@ extern int gb_audio_gb_enable_widget(struct gb_connection *connection, extern int gb_audio_gb_disable_widget(struct gb_connection *connection, u8 widget_id); extern int gb_audio_gb_get_pcm(struct gb_connection *connection, - u16 data_cport, uint32_t *format, - uint32_t *rate, u8 *channels, + u16 data_cport, u32 *format, + u32 *rate, u8 *channels, u8 *sig_bits); extern int gb_audio_gb_set_pcm(struct gb_connection *connection, - u16 data_cport, uint32_t format, - uint32_t rate, u8 channels, + u16 data_cport, u32 format, + u32 rate, u8 channels, u8 sig_bits); extern int gb_audio_gb_set_tx_data_size(struct gb_connection *connection, u16 data_cport, u16 size); diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c index 7884d8482dc0b..8894f1c87d481 100644 --- a/drivers/staging/greybus/audio_gb.c +++ b/drivers/staging/greybus/audio_gb.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Audio Device Class Protocol helpers * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #include "greybus.h" diff --git a/drivers/staging/greybus/audio_manager.c b/drivers/staging/greybus/audio_manager.c index aa6508b44fab2..5889f04f5b2da 100644 --- a/drivers/staging/greybus/audio_manager.c +++ b/drivers/staging/greybus/audio_manager.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #include @@ -90,8 +89,8 @@ void gb_audio_manager_remove_all(void) list_for_each_entry_safe(module, next, &modules_list, list) { list_del(&module->list); - kobject_put(&module->kobj); ida_simple_remove(&module_id, module->id); + kobject_put(&module->kobj); } is_empty = list_empty(&modules_list); diff --git a/drivers/staging/greybus/audio_manager.h b/drivers/staging/greybus/audio_manager.h index 5ab8f5e0ed3fb..dcb1a20f5ff15 100644 --- a/drivers/staging/greybus/audio_manager.h +++ b/drivers/staging/greybus/audio_manager.h @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #ifndef _GB_AUDIO_MANAGER_H_ diff --git a/drivers/staging/greybus/audio_manager_module.c b/drivers/staging/greybus/audio_manager_module.c index adc16977452d1..52342e832e3b0 100644 --- a/drivers/staging/greybus/audio_manager_module.c +++ b/drivers/staging/greybus/audio_manager_module.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/audio_manager_private.h b/drivers/staging/greybus/audio_manager_private.h index 079ce953c2565..9d9b58fc54c4d 100644 --- a/drivers/staging/greybus/audio_manager_private.h +++ b/drivers/staging/greybus/audio_manager_private.h @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #ifndef _GB_AUDIO_MANAGER_PRIVATE_H_ diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c index 34ebd147052f4..283fbed0c8edd 100644 --- a/drivers/staging/greybus/audio_manager_sysfs.c +++ b/drivers/staging/greybus/audio_manager_sysfs.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2015-2016 Google Inc. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/audio_module.c b/drivers/staging/greybus/audio_module.c index 094c3be79b337..d065334efa239 100644 --- a/drivers/staging/greybus/audio_module.c +++ b/drivers/staging/greybus/audio_module.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus audio driver * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include #include diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c index 07fac3948f3a6..b71078339e860 100644 --- a/drivers/staging/greybus/audio_topology.c +++ b/drivers/staging/greybus/audio_topology.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus audio driver * Copyright 2015-2016 Google Inc. * Copyright 2015-2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include "audio_codec.h" @@ -145,7 +144,7 @@ static const char **gb_generate_enum_strings(struct gbaudio_module_info *gb, __u8 *data; items = le32_to_cpu(gbenum->items); - strings = devm_kzalloc(gb->dev, sizeof(char *) * items, GFP_KERNEL); + strings = devm_kcalloc(gb->dev, items, sizeof(char *), GFP_KERNEL); data = gbenum->names; for (i = 0; i < items; i++) { @@ -997,7 +996,7 @@ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module, ret = gbaudio_validate_kcontrol_count(w); if (ret) { - dev_err(module->dev, "Inavlid kcontrol count=%d for %s\n", + dev_err(module->dev, "Invalid kcontrol count=%d for %s\n", w->ncontrols, w->name); return ret; } diff --git a/drivers/staging/greybus/authentication.c b/drivers/staging/greybus/authentication.c index 6c5dcb1c226bc..a5d7c53df987b 100644 --- a/drivers/staging/greybus/authentication.c +++ b/drivers/staging/greybus/authentication.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Component Authentication Protocol (CAP) Driver. * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include "greybus.h" @@ -203,7 +202,7 @@ static int cap_release(struct inode *inode, struct file *file) } static int cap_ioctl(struct gb_cap *cap, unsigned int cmd, - void __user *buf) + void __user *buf) { struct cap_ioc_get_endpoint_uid endpoint_uid; struct cap_ioc_get_ims_certificate *ims_cert; diff --git a/drivers/staging/greybus/bootrom.c b/drivers/staging/greybus/bootrom.c index 06df0ce03150f..e85ffae85dffe 100644 --- a/drivers/staging/greybus/bootrom.c +++ b/drivers/staging/greybus/bootrom.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * BOOTROM Greybus driver. * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/bundle.c b/drivers/staging/greybus/bundle.c index d2ef57d090bef..81c018da12481 100644 --- a/drivers/staging/greybus/bundle.c +++ b/drivers/staging/greybus/bundle.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus bundles * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include "greybus.h" diff --git a/drivers/staging/greybus/bundle.h b/drivers/staging/greybus/bundle.h index 0c3491def96c2..ffcfd43802ccb 100644 --- a/drivers/staging/greybus/bundle.h +++ b/drivers/staging/greybus/bundle.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus bundles * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __BUNDLE_H diff --git a/drivers/staging/greybus/camera.c b/drivers/staging/greybus/camera.c index a64517eabff4c..6dded10f4155f 100644 --- a/drivers/staging/greybus/camera.c +++ b/drivers/staging/greybus/camera.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Camera protocol driver. * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -919,7 +918,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam, /* Retrieve number of streams to configure */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) return -EINVAL; ret = kstrtouint(token, 10, &nstreams); @@ -930,7 +929,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam, return -EINVAL; token = strsep(&buf, ";"); - if (token == NULL) + if (!token) return -EINVAL; ret = kstrtouint(token, 10, &flags); @@ -947,7 +946,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam, /* width */ token = strsep(&buf, ";"); - if (token == NULL) { + if (!token) { ret = -EINVAL; goto done; } @@ -957,7 +956,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam, /* height */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) goto done; ret = kstrtouint(token, 10, &stream->height); @@ -966,7 +965,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam, /* Image format code */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) goto done; ret = kstrtouint(token, 16, &stream->format); @@ -1010,7 +1009,7 @@ static ssize_t gb_camera_debugfs_capture(struct gb_camera *gcam, /* Request id */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) return -EINVAL; ret = kstrtouint(token, 10, &request_id); if (ret < 0) @@ -1018,7 +1017,7 @@ static ssize_t gb_camera_debugfs_capture(struct gb_camera *gcam, /* Stream mask */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) return -EINVAL; ret = kstrtouint(token, 16, &streams_mask); if (ret < 0) @@ -1026,7 +1025,7 @@ static ssize_t gb_camera_debugfs_capture(struct gb_camera *gcam, /* number of frames */ token = strsep(&buf, ";"); - if (token == NULL) + if (!token) return -EINVAL; ret = kstrtouint(token, 10, &num_frames); if (ret < 0) @@ -1175,32 +1174,22 @@ static int gb_camera_debugfs_init(struct gb_camera *gcam) gcam->bundle->id); gcam->debugfs.root = debugfs_create_dir(dirname, gb_debugfs_get()); - if (IS_ERR(gcam->debugfs.root)) { - gcam_err(gcam, "debugfs root create failed (%ld)\n", - PTR_ERR(gcam->debugfs.root)); - return PTR_ERR(gcam->debugfs.root); - } - gcam->debugfs.buffers = vmalloc(sizeof(*gcam->debugfs.buffers) * - GB_CAMERA_DEBUGFS_BUFFER_MAX); + gcam->debugfs.buffers = + vmalloc(array_size(GB_CAMERA_DEBUGFS_BUFFER_MAX, + sizeof(*gcam->debugfs.buffers))); if (!gcam->debugfs.buffers) return -ENOMEM; for (i = 0; i < ARRAY_SIZE(gb_camera_debugfs_entries); ++i) { const struct gb_camera_debugfs_entry *entry = &gb_camera_debugfs_entries[i]; - struct dentry *dentry; gcam->debugfs.buffers[i].length = 0; - dentry = debugfs_create_file(entry->name, entry->mask, - gcam->debugfs.root, gcam, - &gb_camera_debugfs_ops); - if (IS_ERR(dentry)) { - gcam_err(gcam, - "debugfs operation %s create failed (%ld)\n", - entry->name, PTR_ERR(dentry)); - return PTR_ERR(dentry); + debugfs_create_file(entry->name, entry->mask, + gcam->debugfs.root, gcam, + &gb_camera_debugfs_ops); } } diff --git a/drivers/staging/greybus/connection.c b/drivers/staging/greybus/connection.c index 2cf64640e8ec5..2103168b585e8 100644 --- a/drivers/staging/greybus/connection.c +++ b/drivers/staging/greybus/connection.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus connections * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/connection.h b/drivers/staging/greybus/connection.h index 4d9f4c64176cb..ec3f1d3ef3b96 100644 --- a/drivers/staging/greybus/connection.h +++ b/drivers/staging/greybus/connection.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus connections * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __CONNECTION_H diff --git a/drivers/staging/greybus/control.c b/drivers/staging/greybus/control.c index 5b30be30a3a48..35f945a12b11f 100644 --- a/drivers/staging/greybus/control.c +++ b/drivers/staging/greybus/control.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus CPort control protocol. * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/control.h b/drivers/staging/greybus/control.h index 4dcaec8b9cfe8..643ddb9e0f924 100644 --- a/drivers/staging/greybus/control.h +++ b/drivers/staging/greybus/control.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus CPort control protocol * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __CONTROL_H diff --git a/drivers/staging/greybus/core.c b/drivers/staging/greybus/core.c index ba761905b7908..dafa430d176ee 100644 --- a/drivers/staging/greybus/core.c +++ b/drivers/staging/greybus/core.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus "Core" * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/staging/greybus/debugfs.c b/drivers/staging/greybus/debugfs.c index a9d4d3da99a0d..56e20c30feb5d 100644 --- a/drivers/staging/greybus/debugfs.c +++ b/drivers/staging/greybus/debugfs.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus debugfs code * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/es2.c b/drivers/staging/greybus/es2.c index f7b24e0eaa6f3..b082d81833a0a 100644 --- a/drivers/staging/greybus/es2.c +++ b/drivers/staging/greybus/es2.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus "AP" USB driver for "ES2" controller chips * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include #include @@ -761,6 +760,7 @@ static int check_urb_status(struct urb *urb) case -EOVERFLOW: dev_err(dev, "%s: overflow actual length is %d\n", __func__, urb->actual_length); + /* fall through */ case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: diff --git a/drivers/staging/greybus/firmware.h b/drivers/staging/greybus/firmware.h index f4f0db1cefe82..946221307ef67 100644 --- a/drivers/staging/greybus/firmware.h +++ b/drivers/staging/greybus/firmware.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Firmware Management Header * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __FIRMWARE_H diff --git a/drivers/staging/greybus/fw-core.c b/drivers/staging/greybus/fw-core.c index 454a98957ba53..388866d92f5bc 100644 --- a/drivers/staging/greybus/fw-core.c +++ b/drivers/staging/greybus/fw-core.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Firmware Core Bundle Driver. * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/staging/greybus/fw-download.c b/drivers/staging/greybus/fw-download.c index 8a1a413c6cb38..d3b7cccbc10d5 100644 --- a/drivers/staging/greybus/fw-download.c +++ b/drivers/staging/greybus/fw-download.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Firmware Download Protocol Driver. * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/fw-management.c b/drivers/staging/greybus/fw-management.c index 3cd6cf0a656b8..71aec14f8181e 100644 --- a/drivers/staging/greybus/fw-management.c +++ b/drivers/staging/greybus/fw-management.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Firmware Management Protocol Driver. * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/gb-camera.h b/drivers/staging/greybus/gb-camera.h index d45dabc5b3679..ee293e461fc32 100644 --- a/drivers/staging/greybus/gb-camera.h +++ b/drivers/staging/greybus/gb-camera.h @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Camera protocol driver. * * Copyright 2015 Google Inc. - * - * Released under the GPLv2 only. */ #ifndef __GB_CAMERA_H #define __GB_CAMERA_H diff --git a/drivers/staging/greybus/gbphy.c b/drivers/staging/greybus/gbphy.c index 80c1da8224e6d..6cb85c3d35726 100644 --- a/drivers/staging/greybus/gbphy.c +++ b/drivers/staging/greybus/gbphy.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Bridged-Phy Bus driver * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/staging/greybus/gbphy.h b/drivers/staging/greybus/gbphy.h index 8ee68055ccc4b..99463489d7d66 100644 --- a/drivers/staging/greybus/gbphy.h +++ b/drivers/staging/greybus/gbphy.h @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Bridged-Phy Bus driver * * Copyright 2016 Google Inc. - * - * Released under the GPLv2 only. */ #ifndef __GBPHY_H diff --git a/drivers/staging/greybus/gpio.c b/drivers/staging/greybus/gpio.c index ee5f998b174f5..b1d4698019a14 100644 --- a/drivers/staging/greybus/gpio.c +++ b/drivers/staging/greybus/gpio.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * GPIO Greybus driver. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/greybus.h b/drivers/staging/greybus/greybus.h index c9bb93f239271..d03ddb7c9df08 100644 --- a/drivers/staging/greybus/greybus.h +++ b/drivers/staging/greybus/greybus.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus driver and device API * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __LINUX_GREYBUS_H diff --git a/drivers/staging/greybus/greybus_authentication.h b/drivers/staging/greybus/greybus_authentication.h index 4784ed98e8a31..03ea9615b217c 100644 --- a/drivers/staging/greybus/greybus_authentication.h +++ b/drivers/staging/greybus/greybus_authentication.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * Greybus Component Authentication User Header * diff --git a/drivers/staging/greybus/greybus_firmware.h b/drivers/staging/greybus/greybus_firmware.h index 277a2acce6fdf..b58281a63ba4d 100644 --- a/drivers/staging/greybus/greybus_firmware.h +++ b/drivers/staging/greybus/greybus_firmware.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * Greybus Firmware Management User Header * diff --git a/drivers/staging/greybus/greybus_manifest.h b/drivers/staging/greybus/greybus_manifest.h index d135945cefe1c..2cec5cf7a8469 100644 --- a/drivers/staging/greybus/greybus_manifest.h +++ b/drivers/staging/greybus/greybus_manifest.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus manifest definition * diff --git a/drivers/staging/greybus/greybus_protocols.h b/drivers/staging/greybus/greybus_protocols.h index b1be0b0af4645..9bd7b6dfb476c 100644 --- a/drivers/staging/greybus/greybus_protocols.h +++ b/drivers/staging/greybus/greybus_protocols.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) /* * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. diff --git a/drivers/staging/greybus/greybus_trace.h b/drivers/staging/greybus/greybus_trace.h index f8feae4dc3b5b..7b5e2c6b1f6b9 100644 --- a/drivers/staging/greybus/greybus_trace.h +++ b/drivers/staging/greybus/greybus_trace.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus driver and device API * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #undef TRACE_SYSTEM #define TRACE_SYSTEM greybus diff --git a/drivers/staging/greybus/hd.c b/drivers/staging/greybus/hd.c index 185ae3fa10fd6..969f866976737 100644 --- a/drivers/staging/greybus/hd.c +++ b/drivers/staging/greybus/hd.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Host Device * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/hd.h b/drivers/staging/greybus/hd.h index e7927bb1761c4..6cf024a20a584 100644 --- a/drivers/staging/greybus/hd.h +++ b/drivers/staging/greybus/hd.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Host Device * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __HD_H diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c index 465101bbab697..04053ff075a67 100644 --- a/drivers/staging/greybus/hid.c +++ b/drivers/staging/greybus/hid.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * HID class driver for the Greybus. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/i2c.c b/drivers/staging/greybus/i2c.c index c2a50087000c7..58a37deb6579a 100644 --- a/drivers/staging/greybus/i2c.c +++ b/drivers/staging/greybus/i2c.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * I2C bridge driver for the Greybus "generic" I2C module. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/interface.c b/drivers/staging/greybus/interface.c index 71e5cc234e78e..d7b5b89a2f406 100644 --- a/drivers/staging/greybus/interface.c +++ b/drivers/staging/greybus/interface.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus interface code * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/interface.h b/drivers/staging/greybus/interface.h index bd31b8c18d5be..1c00c5bb3ec9e 100644 --- a/drivers/staging/greybus/interface.h +++ b/drivers/staging/greybus/interface.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Interface Block code * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __INTERFACE_H diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index 0f538b8c3a076..40680eaf3974f 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Lights protocol driver. * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -1000,11 +999,7 @@ static int gb_lights_channel_config(struct gb_light *light, light->has_flash = true; - ret = gb_lights_channel_flash_config(channel); - if (ret < 0) - return ret; - - return ret; + return gb_lights_channel_flash_config(channel); } static int gb_lights_light_config(struct gb_lights *glights, u8 id) @@ -1103,21 +1098,21 @@ static void gb_lights_channel_release(struct gb_channel *channel) static void gb_lights_light_release(struct gb_light *light) { int i; - int count; light->ready = false; - count = light->channels_count; - if (light->has_flash) gb_lights_light_v4l2_unregister(light); + light->has_flash = false; - for (i = 0; i < count; i++) { + for (i = 0; i < light->channels_count; i++) gb_lights_channel_release(&light->channels[i]); - light->channels_count--; - } + light->channels_count = 0; + kfree(light->channels); + light->channels = NULL; kfree(light->name); + light->name = NULL; } static void gb_lights_release(struct gb_lights *glights) diff --git a/drivers/staging/greybus/log.c b/drivers/staging/greybus/log.c index 5c5bedaf69a62..15a88574dbb02 100644 --- a/drivers/staging/greybus/log.c +++ b/drivers/staging/greybus/log.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus driver for the log protocol * * Copyright 2016 Google Inc. - * - * Released under the GPLv2 only. */ #include #include diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c index 93e86798ec1c2..42f6f3de967c4 100644 --- a/drivers/staging/greybus/loopback.c +++ b/drivers/staging/greybus/loopback.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Loopback bridge driver for the Greybus loopback module. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -58,12 +57,7 @@ static struct gb_loopback_device gb_dev; struct gb_loopback_async_operation { struct gb_loopback *gb; struct gb_operation *operation; - struct timeval ts; - struct timer_list timer; - struct list_head entry; - struct work_struct work; - struct kref kref; - bool pending; + ktime_t ts; int (*completion)(struct gb_loopback_async_operation *op_async); }; @@ -72,7 +66,6 @@ struct gb_loopback { struct dentry *file; struct kfifo kfifo_lat; - struct kfifo kfifo_ts; struct mutex mutex; struct task_struct *task; struct list_head entry; @@ -82,7 +75,7 @@ struct gb_loopback { atomic_t outstanding_operations; /* Per connection stats */ - struct timeval ts; + ktime_t ts; struct gb_loopback_stats latency; struct gb_loopback_stats throughput; struct gb_loopback_stats requests_per_second; @@ -262,7 +255,6 @@ static void gb_loopback_check_attr(struct gb_loopback *gb) gb->iteration_max, kfifo_depth); } kfifo_reset_out(&gb->kfifo_lat); - kfifo_reset_out(&gb->kfifo_ts); switch (gb->type) { case GB_LOOPBACK_TYPE_PING: @@ -377,21 +369,9 @@ static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) return NSEC_PER_DAY - t2 + t1; } -static u64 gb_loopback_calc_latency(struct timeval *ts, struct timeval *te) -{ - u64 t1, t2; - - t1 = timeval_to_ns(ts); - t2 = timeval_to_ns(te); - - return __gb_loopback_calc_latency(t1, t2); -} - -static void gb_loopback_push_latency_ts(struct gb_loopback *gb, - struct timeval *ts, struct timeval *te) +static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te) { - kfifo_in(&gb->kfifo_ts, (unsigned char *)ts, sizeof(*ts)); - kfifo_in(&gb->kfifo_ts, (unsigned char *)te, sizeof(*te)); + return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te)); } static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, @@ -399,10 +379,10 @@ static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, void *response, int response_size) { struct gb_operation *operation; - struct timeval ts, te; + ktime_t ts, te; int ret; - do_gettimeofday(&ts); + ts = ktime_get(); operation = gb_operation_create(gb->connection, type, request_size, response_size, GFP_KERNEL); if (!operation) @@ -430,11 +410,10 @@ static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, } } - do_gettimeofday(&te); + te = ktime_get(); /* Calculate the total time the message took */ - gb_loopback_push_latency_ts(gb, &ts, &te); - gb->elapsed_nsecs = gb_loopback_calc_latency(&ts, &te); + gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te); out_put_operation: gb_operation_put(operation); @@ -442,56 +421,6 @@ static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, return ret; } -static void __gb_loopback_async_operation_destroy(struct kref *kref) -{ - struct gb_loopback_async_operation *op_async; - - op_async = container_of(kref, struct gb_loopback_async_operation, kref); - - list_del(&op_async->entry); - if (op_async->operation) - gb_operation_put(op_async->operation); - atomic_dec(&op_async->gb->outstanding_operations); - wake_up(&op_async->gb->wq_completion); - kfree(op_async); -} - -static void gb_loopback_async_operation_get(struct gb_loopback_async_operation - *op_async) -{ - kref_get(&op_async->kref); -} - -static void gb_loopback_async_operation_put(struct gb_loopback_async_operation - *op_async) -{ - unsigned long flags; - - spin_lock_irqsave(&gb_dev.lock, flags); - kref_put(&op_async->kref, __gb_loopback_async_operation_destroy); - spin_unlock_irqrestore(&gb_dev.lock, flags); -} - -static struct gb_loopback_async_operation * - gb_loopback_operation_find(u16 id) -{ - struct gb_loopback_async_operation *op_async; - bool found = false; - unsigned long flags; - - spin_lock_irqsave(&gb_dev.lock, flags); - list_for_each_entry(op_async, &gb_dev.list_op_async, entry) { - if (op_async->operation->id == id) { - gb_loopback_async_operation_get(op_async); - found = true; - break; - } - } - spin_unlock_irqrestore(&gb_dev.lock, flags); - - return found ? op_async : NULL; -} - static void gb_loopback_async_wait_all(struct gb_loopback *gb) { wait_event(gb->wq_completion, @@ -502,87 +431,42 @@ static void gb_loopback_async_operation_callback(struct gb_operation *operation) { struct gb_loopback_async_operation *op_async; struct gb_loopback *gb; - struct timeval te; - bool err = false; - - do_gettimeofday(&te); - op_async = gb_loopback_operation_find(operation->id); - if (!op_async) - return; + ktime_t te; + int result; + te = ktime_get(); + result = gb_operation_result(operation); + op_async = gb_operation_get_data(operation); gb = op_async->gb; + mutex_lock(&gb->mutex); - if (!op_async->pending || gb_operation_result(operation)) { - err = true; + if (!result && op_async->completion) + result = op_async->completion(op_async); + + if (!result) { + gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te); } else { - if (op_async->completion) - if (op_async->completion(op_async)) - err = true; + gb->error++; + if (result == -ETIMEDOUT) + gb->requests_timedout++; } - if (!err) { - gb_loopback_push_latency_ts(gb, &op_async->ts, &te); - gb->elapsed_nsecs = gb_loopback_calc_latency(&op_async->ts, - &te); - } + gb->iteration_count++; + gb_loopback_calculate_stats(gb, result); - if (op_async->pending) { - if (err) - gb->error++; - gb->iteration_count++; - op_async->pending = false; - del_timer_sync(&op_async->timer); - gb_loopback_async_operation_put(op_async); - gb_loopback_calculate_stats(gb, err); - } mutex_unlock(&gb->mutex); dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", operation->id); - gb_loopback_async_operation_put(op_async); -} - -static void gb_loopback_async_operation_work(struct work_struct *work) -{ - struct gb_loopback *gb; - struct gb_operation *operation; - struct gb_loopback_async_operation *op_async; - - op_async = container_of(work, struct gb_loopback_async_operation, work); - gb = op_async->gb; - operation = op_async->operation; - - mutex_lock(&gb->mutex); - if (op_async->pending) { - gb->requests_timedout++; - gb->error++; - gb->iteration_count++; - op_async->pending = false; - gb_loopback_async_operation_put(op_async); - gb_loopback_calculate_stats(gb, true); - } - mutex_unlock(&gb->mutex); - - dev_dbg(&gb->connection->bundle->dev, "timeout operation %d\n", - operation->id); - - gb_operation_cancel(operation, -ETIMEDOUT); - gb_loopback_async_operation_put(op_async); -} - -static void gb_loopback_async_operation_timeout(unsigned long data) -{ - struct gb_loopback_async_operation *op_async; - u16 id = data; + /* Wake up waiters */ + atomic_dec(&op_async->gb->outstanding_operations); + wake_up(&gb->wq_completion); - op_async = gb_loopback_operation_find(id); - if (!op_async) { - pr_err("operation %d not found - time out ?\n", id); - return; - } - schedule_work(&op_async->work); + /* Release resources */ + gb_operation_put(operation); + kfree(op_async); } static int gb_loopback_async_operation(struct gb_loopback *gb, int type, @@ -593,15 +477,11 @@ static int gb_loopback_async_operation(struct gb_loopback *gb, int type, struct gb_loopback_async_operation *op_async; struct gb_operation *operation; int ret; - unsigned long flags; op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); if (!op_async) return -ENOMEM; - INIT_WORK(&op_async->work, gb_loopback_async_operation_work); - kref_init(&op_async->kref); - operation = gb_operation_create(gb->connection, type, request_size, response_size, GFP_KERNEL); if (!operation) { @@ -612,35 +492,24 @@ static int gb_loopback_async_operation(struct gb_loopback *gb, int type, if (request_size) memcpy(operation->request->payload, request, request_size); + gb_operation_set_data(operation, op_async); + op_async->gb = gb; op_async->operation = operation; op_async->completion = completion; - spin_lock_irqsave(&gb_dev.lock, flags); - list_add_tail(&op_async->entry, &gb_dev.list_op_async); - spin_unlock_irqrestore(&gb_dev.lock, flags); + op_async->ts = ktime_get(); - do_gettimeofday(&op_async->ts); - op_async->pending = true; atomic_inc(&gb->outstanding_operations); - mutex_lock(&gb->mutex); ret = gb_operation_request_send(operation, gb_loopback_async_operation_callback, - 0, + jiffies_to_msecs(gb->jiffy_timeout), GFP_KERNEL); - if (ret) - goto error; - - setup_timer(&op_async->timer, gb_loopback_async_operation_timeout, - (unsigned long)operation->id); - op_async->timer.expires = jiffies + gb->jiffy_timeout; - add_timer(&op_async->timer); - - goto done; -error: - gb_loopback_async_operation_put(op_async); -done: - mutex_unlock(&gb->mutex); + if (ret) { + atomic_dec(&gb->outstanding_operations); + gb_operation_put(operation); + kfree(op_async); + } return ret; } @@ -854,7 +723,7 @@ static void gb_loopback_reset_stats(struct gb_loopback *gb) /* Should be initialized at least once per transaction set */ gb->apbridge_latency_ts = 0; gb->gbphy_latency_ts = 0; - memset(&gb->ts, 0, sizeof(struct timeval)); + gb->ts = ktime_set(0, 0); } static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) @@ -937,15 +806,15 @@ static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error) { u64 nlat; u32 lat; - struct timeval te; + ktime_t te; if (!error) { gb->requests_completed++; gb_loopback_calculate_latency_stats(gb); } - do_gettimeofday(&te); - nlat = gb_loopback_calc_latency(&gb->ts, &te); + te = ktime_get(); + nlat = gb_loopback_calc_latency(gb->ts, te); if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) { lat = gb_loopback_nsec_to_usec_latency(nlat); @@ -1029,9 +898,8 @@ static int gb_loopback_fn(void *data) size = gb->size; us_wait = gb->us_wait; type = gb->type; - if (gb->ts.tv_usec == 0 && gb->ts.tv_sec == 0) - do_gettimeofday(&gb->ts); - mutex_unlock(&gb->mutex); + if (ktime_to_ns(gb->ts) == 0) + gb->ts = ktime_get(); /* Else operations to perform */ if (gb->async) { @@ -1061,6 +929,7 @@ static int gb_loopback_fn(void *data) gb_loopback_calculate_stats(gb, !!error); } gb->send_count++; + mutex_unlock(&gb->mutex); if (us_wait) { if (us_wait < 20000) @@ -1243,18 +1112,12 @@ static int gb_loopback_probe(struct gb_bundle *bundle, retval = -ENOMEM; goto out_conn; } - if (kfifo_alloc(&gb->kfifo_ts, kfifo_depth * sizeof(struct timeval) * 2, - GFP_KERNEL)) { - retval = -ENOMEM; - goto out_kfifo0; - } - /* Fork worker thread */ mutex_init(&gb->mutex); gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); if (IS_ERR(gb->task)) { retval = PTR_ERR(gb->task); - goto out_kfifo1; + goto out_kfifo; } spin_lock_irqsave(&gb_dev.lock, flags); @@ -1268,9 +1131,7 @@ static int gb_loopback_probe(struct gb_bundle *bundle, return 0; -out_kfifo1: - kfifo_free(&gb->kfifo_ts); -out_kfifo0: +out_kfifo: kfifo_free(&gb->kfifo_lat); out_conn: device_unregister(dev); @@ -1304,7 +1165,6 @@ static void gb_loopback_disconnect(struct gb_bundle *bundle) kthread_stop(gb->task); kfifo_free(&gb->kfifo_lat); - kfifo_free(&gb->kfifo_ts); gb_connection_latency_tag_disable(gb->connection); debugfs_remove(gb->file); diff --git a/drivers/staging/greybus/manifest.c b/drivers/staging/greybus/manifest.c index 7b903770a6846..08db49264f2b3 100644 --- a/drivers/staging/greybus/manifest.c +++ b/drivers/staging/greybus/manifest.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus manifest parsing * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include "greybus.h" diff --git a/drivers/staging/greybus/manifest.h b/drivers/staging/greybus/manifest.h index d96428407cd75..f3c95a2556318 100644 --- a/drivers/staging/greybus/manifest.h +++ b/drivers/staging/greybus/manifest.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus manifest parsing * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __MANIFEST_H diff --git a/drivers/staging/greybus/module.c b/drivers/staging/greybus/module.c index 660b4674a76f5..b785382192de6 100644 --- a/drivers/staging/greybus/module.c +++ b/drivers/staging/greybus/module.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Module code * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include "greybus.h" diff --git a/drivers/staging/greybus/module.h b/drivers/staging/greybus/module.h index 88a97ce042434..b1ebcc6636db7 100644 --- a/drivers/staging/greybus/module.h +++ b/drivers/staging/greybus/module.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Module code * * Copyright 2016 Google Inc. * Copyright 2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __MODULE_H diff --git a/drivers/staging/greybus/operation.c b/drivers/staging/greybus/operation.c index 3023012808d91..c462b1c046cd4 100644 --- a/drivers/staging/greybus/operation.c +++ b/drivers/staging/greybus/operation.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -294,9 +293,9 @@ static void gb_operation_work(struct work_struct *work) gb_operation_put(operation); } -static void gb_operation_timeout(unsigned long arg) +static void gb_operation_timeout(struct timer_list *t) { - struct gb_operation *operation = (void *)arg; + struct gb_operation *operation = from_timer(operation, t, timer); if (gb_operation_result_set(operation, -ETIMEDOUT)) { /* @@ -541,8 +540,7 @@ gb_operation_create_common(struct gb_connection *connection, u8 type, goto err_request; } - setup_timer(&operation->timer, gb_operation_timeout, - (unsigned long)operation); + timer_setup(&operation->timer, gb_operation_timeout, 0); } operation->flags = op_flags; diff --git a/drivers/staging/greybus/operation.h b/drivers/staging/greybus/operation.h index 7529f01b25292..40b7b02fff889 100644 --- a/drivers/staging/greybus/operation.h +++ b/drivers/staging/greybus/operation.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus operations * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __OPERATION_H @@ -105,6 +104,8 @@ struct gb_operation { int active; struct list_head links; /* connection->operations */ + + void *private; }; static inline bool @@ -206,6 +207,17 @@ static inline int gb_operation_unidirectional(struct gb_connection *connection, request, request_size, GB_OPERATION_TIMEOUT_DEFAULT); } +static inline void *gb_operation_get_data(struct gb_operation *operation) +{ + return operation->private; +} + +static inline void gb_operation_set_data(struct gb_operation *operation, + void *data) +{ + operation->private = data; +} + int gb_operation_init(void); void gb_operation_exit(void); diff --git a/drivers/staging/greybus/power_supply.c b/drivers/staging/greybus/power_supply.c index 20cac20518d76..ae5c0285a9420 100644 --- a/drivers/staging/greybus/power_supply.c +++ b/drivers/staging/greybus/power_supply.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Power Supply driver for a Greybus module. * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -521,7 +520,7 @@ static int gb_power_supply_prop_descriptors_get(struct gb_power_supply *gbpsy) op = gb_operation_create(connection, GB_POWER_SUPPLY_TYPE_GET_PROP_DESCRIPTORS, - sizeof(req), sizeof(*resp) + props_count * + sizeof(*req), sizeof(*resp) + props_count * sizeof(struct gb_power_supply_props_desc), GFP_KERNEL); if (!op) diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index f0404bc37123d..4a6d394b6c448 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * PWM Greybus driver. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c index 729d258115685..838acbe84ca01 100644 --- a/drivers/staging/greybus/raw.c +++ b/drivers/staging/greybus/raw.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus driver for the Raw protocol * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include #include diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c index 101ca5097fc9f..38e85033fc4b3 100644 --- a/drivers/staging/greybus/sdio.c +++ b/drivers/staging/greybus/sdio.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SD/MMC Greybus driver. * * Copyright 2014-2015 Google Inc. * Copyright 2014-2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/spi.c b/drivers/staging/greybus/spi.c index c893552b5c0b8..47d896992b351 100644 --- a/drivers/staging/greybus/spi.c +++ b/drivers/staging/greybus/spi.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SPI bridge PHY driver. * * Copyright 2014-2016 Google Inc. * Copyright 2014-2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/spilib.c b/drivers/staging/greybus/spilib.c index 1e7321a1404cc..2e07c6b413345 100644 --- a/drivers/staging/greybus/spilib.c +++ b/drivers/staging/greybus/spilib.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus SPI library * * Copyright 2014-2016 Google Inc. * Copyright 2014-2016 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/spilib.h b/drivers/staging/greybus/spilib.h index cb6092578a922..043d4d32c3eed 100644 --- a/drivers/staging/greybus/spilib.h +++ b/drivers/staging/greybus/spilib.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus SPI library header * diff --git a/drivers/staging/greybus/svc.c b/drivers/staging/greybus/svc.c index 516f827e5ed9a..a2bb7e1a3db34 100644 --- a/drivers/staging/greybus/svc.c +++ b/drivers/staging/greybus/svc.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SVC Greybus driver. * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include @@ -1138,7 +1137,6 @@ static int gb_svc_intf_reset_recv(struct gb_operation *op) struct gb_svc *svc = gb_connection_get_data(op->connection); struct gb_message *request = op->request; struct gb_svc_intf_reset_request *reset; - u8 intf_id; if (request->payload_size < sizeof(*reset)) { dev_warn(&svc->dev, "short reset request received (%zu < %zu)\n", @@ -1147,8 +1145,6 @@ static int gb_svc_intf_reset_recv(struct gb_operation *op) } reset = request->payload; - intf_id = reset->intf_id; - /* FIXME Reset the interface here */ return 0; diff --git a/drivers/staging/greybus/svc.h b/drivers/staging/greybus/svc.h index 226c2a396fc80..ad01783bac9cd 100644 --- a/drivers/staging/greybus/svc.h +++ b/drivers/staging/greybus/svc.h @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus SVC code * * Copyright 2015 Google Inc. * Copyright 2015 Linaro Ltd. - * - * Released under the GPLv2 only. */ #ifndef __SVC_H diff --git a/drivers/staging/greybus/svc_watchdog.c b/drivers/staging/greybus/svc_watchdog.c index 779fbea5d4ba8..7868ad8211c5d 100644 --- a/drivers/staging/greybus/svc_watchdog.c +++ b/drivers/staging/greybus/svc_watchdog.c @@ -1,9 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SVC Greybus "watchdog" driver. * * Copyright 2016 Google Inc. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/staging/greybus/tools/lbtest b/drivers/staging/greybus/tools/lbtest index d7353f1a2a6f6..47c481239e986 100755 --- a/drivers/staging/greybus/tools/lbtest +++ b/drivers/staging/greybus/tools/lbtest @@ -1,4 +1,5 @@ #!/usr/bin/env python +# SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2015 Google, Inc. # Copyright (c) 2015 Linaro, Ltd. diff --git a/drivers/staging/greybus/tools/loopback_test.c b/drivers/staging/greybus/tools/loopback_test.c index fbe589fca8408..1452df84e2b7c 100644 --- a/drivers/staging/greybus/tools/loopback_test.c +++ b/drivers/staging/greybus/tools/loopback_test.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * Loopback test application * @@ -20,6 +21,7 @@ #include #define MAX_NUM_DEVICES 10 +#define MAX_SYSFS_PREFIX 0x80 #define MAX_SYSFS_PATH 0x200 #define CSV_MAX_LINE 0x1000 #define SYSFS_MAX_INT 0x20 @@ -68,7 +70,7 @@ struct loopback_results { }; struct loopback_device { - char name[MAX_SYSFS_PATH]; + char name[MAX_STR_LEN]; char sysfs_entry[MAX_SYSFS_PATH]; char debugfs_entry[MAX_SYSFS_PATH]; struct loopback_results results; @@ -94,8 +96,8 @@ struct loopback_test { int stop_all; int poll_count; char test_name[MAX_STR_LEN]; - char sysfs_prefix[MAX_SYSFS_PATH]; - char debugfs_prefix[MAX_SYSFS_PATH]; + char sysfs_prefix[MAX_SYSFS_PREFIX]; + char debugfs_prefix[MAX_SYSFS_PREFIX]; struct timespec poll_timeout; struct loopback_device devices[MAX_NUM_DEVICES]; struct loopback_results aggregate_results; @@ -644,7 +646,7 @@ int find_loopback_devices(struct loopback_test *t) static int open_poll_files(struct loopback_test *t) { struct loopback_device *dev; - char buf[MAX_STR_LEN]; + char buf[MAX_SYSFS_PATH + MAX_STR_LEN]; char dummy; int fds_idx = 0; int i; @@ -662,7 +664,7 @@ static int open_poll_files(struct loopback_test *t) goto err; } read(t->fds[fds_idx].fd, &dummy, 1); - t->fds[fds_idx].events = POLLERR|POLLPRI; + t->fds[fds_idx].events = POLLERR | POLLPRI; t->fds[fds_idx].revents = 0; fds_idx++; } @@ -914,10 +916,10 @@ int main(int argc, char *argv[]) t.iteration_max = atoi(optarg); break; case 'S': - snprintf(t.sysfs_prefix, MAX_SYSFS_PATH, "%s", optarg); + snprintf(t.sysfs_prefix, MAX_SYSFS_PREFIX, "%s", optarg); break; case 'D': - snprintf(t.debugfs_prefix, MAX_SYSFS_PATH, "%s", optarg); + snprintf(t.debugfs_prefix, MAX_SYSFS_PREFIX, "%s", optarg); break; case 'm': t.mask = atol(optarg); @@ -968,10 +970,10 @@ int main(int argc, char *argv[]) } if (!strcmp(t.sysfs_prefix, "")) - snprintf(t.sysfs_prefix, MAX_SYSFS_PATH, "%s", sysfs_prefix); + snprintf(t.sysfs_prefix, MAX_SYSFS_PREFIX, "%s", sysfs_prefix); if (!strcmp(t.debugfs_prefix, "")) - snprintf(t.debugfs_prefix, MAX_SYSFS_PATH, "%s", debugfs_prefix); + snprintf(t.debugfs_prefix, MAX_SYSFS_PREFIX, "%s", debugfs_prefix); ret = find_loopback_devices(&t); if (ret) diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index c6d01b800d3cc..f36d470aed241 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -1,11 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * UART driver for the Greybus "generic" UART module. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. * - * Released under the GPLv2 only. - * * Heavily based on drivers/usb/class/cdc-acm.c and * drivers/usb/serial/usb-serial.c. */ @@ -538,9 +537,9 @@ static void gb_tty_set_termios(struct tty_struct *tty, } if (C_CRTSCTS(tty) && C_BAUD(tty) != B0) - newline.flow_control |= GB_SERIAL_AUTO_RTSCTS_EN; + newline.flow_control = GB_SERIAL_AUTO_RTSCTS_EN; else - newline.flow_control &= ~GB_SERIAL_AUTO_RTSCTS_EN; + newline.flow_control = 0; if (memcmp(&gb_tty->line_coding, &newline, sizeof(newline))) { memcpy(&gb_tty->line_coding, &newline, sizeof(newline)); diff --git a/drivers/staging/greybus/usb.c b/drivers/staging/greybus/usb.c index f93a76d02de65..1c246c73a085e 100644 --- a/drivers/staging/greybus/usb.c +++ b/drivers/staging/greybus/usb.c @@ -1,11 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * USB host driver for the Greybus "generic" USB module. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. - * */ #include #include diff --git a/drivers/staging/greybus/vibrator.c b/drivers/staging/greybus/vibrator.c index 5cd8a50d41adb..3e5dedeacd5c4 100644 --- a/drivers/staging/greybus/vibrator.c +++ b/drivers/staging/greybus/vibrator.c @@ -1,10 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Greybus Vibrator protocol driver. * * Copyright 2014 Google Inc. * Copyright 2014 Linaro Ltd. - * - * Released under the GPLv2 only. */ #include diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 2bb0b092bff24..851d29ff3d19e 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1579,10 +1579,10 @@ static int __init omap8250_console_fixup(void) } add_preferred_console("ttyS", idx, options); - pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", + pr_info("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", idx, idx); - pr_err("This ensures that you still see kernel messages. Please\n"); - pr_err("update your kernel commandline.\n"); + pr_info("This ensures that you still see kernel messages. Please\n"); + pr_info("update your kernel commandline.\n"); return 0; } console_initcall(omap8250_console_fixup); diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 26a22b100df10..24bae2eaa07f8 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1652,6 +1652,31 @@ static int serial_omap_probe_rs485(struct uart_omap_port *up, return 0; } +static int serial_omap_of_get_port_line(struct device_node *np) +{ + unsigned long val; + const char *hwmod; + int ret; + + /* first try the serial alias */ + ret = of_alias_get_id(np, "serial"); + if (ret >= 0) + return ret; + + /* no? calculate it from hwmods */ + ret = of_property_read_string(np, "ti,hwmods", &hwmod); + if (ret != 0 || strncmp(hwmod, "uart", 4) || + kstrtoul(hwmod + 4, 10, &val)) + return -ENODEV; + + /* numbering of hwmods is +1 */ + ret = (int)val - 1; + if (ret < 0) + return -ENODEV; + + return ret; +} + static int serial_omap_probe(struct platform_device *pdev) { struct omap_uart_port_info *omap_up_info = dev_get_platdata(&pdev->dev); @@ -1669,7 +1694,10 @@ static int serial_omap_probe(struct platform_device *pdev) return -EPROBE_DEFER; wakeirq = irq_of_parse_and_map(pdev->dev.of_node, 1); omap_up_info = of_get_uart_port_info(&pdev->dev); - pdev->dev.platform_data = omap_up_info; + ret = platform_device_add_data(pdev, omap_up_info, + sizeof(*omap_up_info)); + if (ret != 0) + return ret; } else { uartirq = platform_get_irq(pdev, 0); if (uartirq < 0) @@ -1695,7 +1723,7 @@ static int serial_omap_probe(struct platform_device *pdev) up->port.ops = &serial_omap_pops; if (pdev->dev.of_node) - ret = of_alias_get_id(pdev->dev.of_node, "serial"); + ret = serial_omap_of_get_port_line(pdev->dev.of_node); else ret = pdev->id; diff --git a/drivers/uio/Kconfig b/drivers/uio/Kconfig index 7e8dc78a97961..531e862b04b13 100644 --- a/drivers/uio/Kconfig +++ b/drivers/uio/Kconfig @@ -129,7 +129,7 @@ config UIO_PRUSS select GENERIC_ALLOCATOR depends on HAS_IOMEM && HAS_DMA help - PRUSS driver for OMAPL138/DA850/AM18XX devices + PRUSS driver for OMAPL138/DA850/AM18XX and AM33XX devices PRUSS driver requires user space components, examples and user space driver is available from below SVN repo - you may use anonymous login @@ -142,6 +142,14 @@ config UIO_PRUSS To compile this driver as a module, choose M here: the module will be called uio_pruss. +config UIO_PRUSS_SHMEM + tristate "Texas Instruments PRUSS_SHMEM driver" + help + PRUSS SHMEM driver for AM3X and AM5X devices + + To compile this driver as a module, choose M here: the module + will be called uio_pruss_shmem. + config UIO_MF624 tristate "Humusoft MF624 DAQ PCI card driver" depends on PCI diff --git a/drivers/uio/Makefile b/drivers/uio/Makefile index c285dd2a4539b..a4bdec8960350 100644 --- a/drivers/uio/Makefile +++ b/drivers/uio/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_UIO_SERCOS3) += uio_sercos3.o obj-$(CONFIG_UIO_PCI_GENERIC) += uio_pci_generic.o obj-$(CONFIG_UIO_NETX) += uio_netx.o obj-$(CONFIG_UIO_PRUSS) += uio_pruss.o +obj-$(CONFIG_UIO_PRUSS_SHMEM) += uio_pruss_shmem.o obj-$(CONFIG_UIO_MF624) += uio_mf624.o obj-$(CONFIG_UIO_FSL_ELBC_GPCM) += uio_fsl_elbc_gpcm.o obj-$(CONFIG_UIO_HV_GENERIC) += uio_hv_generic.o diff --git a/drivers/uio/uio_pruss.c b/drivers/uio/uio_pruss.c index 31d5b1d3b5af6..f0cffbe48b980 100644 --- a/drivers/uio/uio_pruss.c +++ b/drivers/uio/uio_pruss.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -27,13 +28,23 @@ #include #include #include +#include +#include +#include +#include +#include #define DRV_NAME "pruss_uio" #define DRV_VERSION "1.0" +/* XXX the sram pool support right now is supported for prussv1, even though + * it seems to me it should be easy to support this for prussv2 as well. + */ +#ifdef CONFIG_ARCH_DAVINCI_DA850 static int sram_pool_sz = SZ_16K; module_param(sram_pool_sz, int, 0); MODULE_PARM_DESC(sram_pool_sz, "sram pool size to allocate "); +#endif static int extram_pool_sz = SZ_256K; module_param(extram_pool_sz, int, 0); @@ -63,14 +74,16 @@ MODULE_PARM_DESC(extram_pool_sz, "external ram pool size to allocate"); struct uio_pruss_dev { struct uio_info *info; struct clk *pruss_clk; - dma_addr_t sram_paddr; dma_addr_t ddr_paddr; void __iomem *prussio_vaddr; - unsigned long sram_vaddr; void *ddr_vaddr; unsigned int hostirq_start; unsigned int pintc_base; +#ifdef CONFIG_ARCH_DAVINCI_DA850 + dma_addr_t sram_paddr; + unsigned long sram_vaddr; struct gen_pool *sram_pool; +#endif }; static irqreturn_t pruss_handler(int irq, struct uio_info *info) @@ -106,14 +119,18 @@ static void pruss_cleanup(struct device *dev, struct uio_pruss_dev *gdev) dma_free_coherent(dev, extram_pool_sz, gdev->ddr_vaddr, gdev->ddr_paddr); } +#ifdef CONFIG_ARCH_DAVINCI_DA850 if (gdev->sram_vaddr) gen_pool_free(gdev->sram_pool, gdev->sram_vaddr, sram_pool_sz); +#endif kfree(gdev->info); clk_disable(gdev->pruss_clk); clk_put(gdev->pruss_clk); kfree(gdev); + pm_runtime_put(dev); + pm_runtime_disable(dev); } static int pruss_probe(struct platform_device *pdev) @@ -121,9 +138,15 @@ static int pruss_probe(struct platform_device *pdev) struct uio_info *p; struct uio_pruss_dev *gdev; struct resource *regs_prussio; + struct resource res; struct device *dev = &pdev->dev; int ret = -ENODEV, cnt = 0, len; struct uio_pruss_pdata *pdata = dev_get_platdata(dev); + struct pinctrl *pinctrl; + + int count; + struct device_node *child; + const char *pin_name; gdev = kzalloc(sizeof(struct uio_pruss_dev), GFP_KERNEL); if (!gdev) @@ -135,14 +158,24 @@ static int pruss_probe(struct platform_device *pdev) return -ENOMEM; } + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync() failed\n"); + pm_runtime_disable(dev); + kfree(gdev->info); + kfree(gdev); + return ret; + } + +#ifdef CONFIG_ARCH_DAVINCI_DA850 /* Power on PRU in case its not done as part of boot-loader */ gdev->pruss_clk = clk_get(dev, "pruss"); if (IS_ERR(gdev->pruss_clk)) { dev_err(dev, "Failed to get clock\n"); ret = PTR_ERR(gdev->pruss_clk); - kfree(gdev->info); - kfree(gdev); - return ret; + gdev->pruss_clk = NULL; + goto out_free; } else { ret = clk_enable(gdev->pruss_clk); if (ret) { @@ -153,11 +186,21 @@ static int pruss_probe(struct platform_device *pdev) return ret; } } +#endif - regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!regs_prussio) { - dev_err(dev, "No PRUSS I/O resource specified\n"); - goto out_free; + if (dev->of_node) { + ret = of_address_to_resource(dev->of_node, 0, &res); + if (IS_ERR_VALUE(ret)) { + dev_err(dev, "failed to parse DT reg\n"); + goto out_free; + } + regs_prussio = &res; + } else { + regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs_prussio) { + dev_err(dev, "No PRUSS I/O resource specified\n"); + goto out_free; + } } if (!regs_prussio->start) { @@ -165,7 +208,64 @@ static int pruss_probe(struct platform_device *pdev) goto out_free; } - if (pdata->sram_pool) { + + /* XXX this stuff below is complete garbage: + * 1. It's quite common for pruss to have no pinctrl. + * 2. If the pins are to be used as fast PRU I/O then requesting + * the same pins as regular gpios is a conflict (which would be + * detected by the kernel if the dts bothered to declare the + * gpio-to-pinctrl mapping). + * 3. If you really do just want to request a bunch of regular + * gpios, then use gpio-of-helper. That's what it's for. + * + * - Matthijs van Duin + */ +#if 0 + pinctrl = devm_pinctrl_get_select_default(dev); + if (IS_ERR(pinctrl)) + dev_warn(dev, + "pins are not configured from the driver\n"); + else{ + count = of_get_child_count(dev->of_node); + if (!count){ + dev_info(dev, "No children\n"); + return -ENODEV; + } + // Run through all children. They have lables for easy reference. + for_each_child_of_node(dev->of_node, child){ + enum of_gpio_flags flags; + unsigned gpio; + + count = of_gpio_count(child); + + ret = of_property_count_strings(child, "pin-names"); + if (ret < 0) { + dev_err(dev, "Failed to get pin-names\n"); + continue; + } + if(count != ret){ + dev_err(dev, "The number of gpios (%d) does not match"\ + " the number of pin names (%d)\n", count, ret); + continue; + } + + for(cnt=0; cntsram_pool) { gdev->sram_pool = pdata->sram_pool; gdev->sram_vaddr = (unsigned long)gen_pool_dma_alloc(gdev->sram_pool, @@ -175,6 +275,7 @@ static int pruss_probe(struct platform_device *pdev) goto out_free; } } +#endif gdev->ddr_vaddr = dma_alloc_coherent(dev, extram_pool_sz, &(gdev->ddr_paddr), GFP_KERNEL | GFP_DMA); @@ -186,26 +287,45 @@ static int pruss_probe(struct platform_device *pdev) len = resource_size(regs_prussio); gdev->prussio_vaddr = ioremap(regs_prussio->start, len); if (!gdev->prussio_vaddr) { - dev_err(dev, "Can't remap PRUSS I/O address range\n"); + dev_err(dev, "Can't remap PRUSS I/O address range\n"); goto out_free; } - gdev->pintc_base = pdata->pintc_base; + if (dev->of_node) { + ret = of_property_read_u32(dev->of_node, + "ti,pintc-offset", + &gdev->pintc_base); + if (ret < 0) { + dev_err(dev, "Can't parse ti,pintc-offset property\n"); + goto out_free; + } + } else + gdev->pintc_base = pdata->pintc_base; gdev->hostirq_start = platform_get_irq(pdev, 0); for (cnt = 0, p = gdev->info; cnt < MAX_PRUSS_EVT; cnt++, p++) { + p->mem[0].name = "pruss"; p->mem[0].addr = regs_prussio->start; p->mem[0].size = resource_size(regs_prussio); p->mem[0].memtype = UIO_MEM_PHYS; + /* note: some userspace code uses hardcoded indices... */ +#ifdef CONFIG_ARCH_DAVINCI_DA850 + p->mem[1].name = "sram"; p->mem[1].addr = gdev->sram_paddr; p->mem[1].size = sram_pool_sz; p->mem[1].memtype = UIO_MEM_PHYS; + p->mem[2].name = "ddr"; p->mem[2].addr = gdev->ddr_paddr; p->mem[2].size = extram_pool_sz; p->mem[2].memtype = UIO_MEM_PHYS; - +#else + p->mem[1].name = "ddr"; + p->mem[1].addr = gdev->ddr_paddr; + p->mem[1].size = extram_pool_sz; + p->mem[1].memtype = UIO_MEM_PHYS; +#endif p->name = kasprintf(GFP_KERNEL, "pruss_evt%d", cnt); p->version = DRV_VERSION; @@ -235,11 +355,20 @@ static int pruss_remove(struct platform_device *dev) return 0; } +static const struct of_device_id pruss_dt_ids[] = { + { .compatible = "ti,pruss-v1", .data = NULL, }, + { .compatible = "ti,pruss-v2", .data = NULL, }, + {}, +}; +MODULE_DEVICE_TABLE(of, pruss_dt_ids); + + static struct platform_driver pruss_driver = { .probe = pruss_probe, .remove = pruss_remove, .driver = { .name = DRV_NAME, + .of_match_table = pruss_dt_ids, }, }; diff --git a/drivers/uio/uio_pruss_shmem.c b/drivers/uio/uio_pruss_shmem.c new file mode 100644 index 0000000000000..2259ba70e4f04 --- /dev/null +++ b/drivers/uio/uio_pruss_shmem.c @@ -0,0 +1,163 @@ +/* + * Programmable Real-Time Unit Sub System (PRUSS) UIO driver (uio_pruss) + * + * This driver exports PRUSS host event out interrupts and PRUSS, L3 RAM, + * and DDR RAM to user space for applications interacting with PRUSS firmware + * + * Copyright (C) 2010-11 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "pruss_uio_shmem" +#define DRV_VERSION "1.0" + +struct uio_pruss_shmem_dev { + struct uio_info *info; + void __iomem *prussio_vaddr; +}; + +static void pruss_shmem_cleanup(struct device *dev, struct uio_pruss_shmem_dev *gdev) +{ + struct uio_info *p = gdev->info; + + uio_unregister_device(p); + iounmap(gdev->prussio_vaddr); + kfree(gdev->info); + kfree(gdev); +} + +static int pruss_shmem_probe(struct platform_device *pdev) +{ + struct uio_pruss_shmem_dev *gdev; + struct resource *regs_prussio; + struct resource res; + struct device *dev = &pdev->dev; + int ret, len; + struct uio_info *p; + + dev_info(dev, "Allocating gdev\n"); + gdev = kzalloc(sizeof(struct uio_pruss_shmem_dev), GFP_KERNEL); + if (!gdev) + return -ENOMEM; + + dev_info(dev, "Allocating info\n"); + gdev->info = kzalloc(sizeof(struct uio_info), GFP_KERNEL); + if (!gdev->info) { + ret = -ENOMEM; + goto err_free_gdev; + } + + dev_info(dev, "Requesting resource\n"); + if (dev->of_node) { + ret = of_address_to_resource(dev->of_node, 0, &res); + if (IS_ERR_VALUE(ret)) { + dev_err(dev, "Failed to parse DT reg\n"); + goto err_free_info; + } + regs_prussio = &res; + } else { + regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs_prussio) { + dev_err(dev, "No PRUSS I/O resource specified\n"); + goto err_free_info; + } + } + if (!regs_prussio) { + dev_err(dev, "No PRUSS I/O resource specified\n"); + ret = -EIO; + goto err_free_info; + } + if (!regs_prussio->start) { + dev_err(dev, "Invalid memory resource\n"); + ret = -EIO; + goto err_free_info; + } + dev_info(dev, "Mapping resource\n"); + len = resource_size(regs_prussio); + gdev->prussio_vaddr = ioremap(regs_prussio->start, len); + if (!gdev->prussio_vaddr) { + dev_err(dev, "Can't remap PRUSS I/O address range\n"); + ret = -ENOMEM; + goto err_free_info; + } + + p = gdev->info; + p->mem[0].addr = regs_prussio->start; + p->mem[0].size = resource_size(regs_prussio); + p->mem[0].memtype = UIO_MEM_PHYS; + + p->name = "pruss_shmem"; + p->version = DRV_VERSION; + + dev_info(dev, "Registering with uio driver\n"); + ret = uio_register_device(dev, p); + if (ret < 0) { + goto err_free_register; + } + + dev_info(dev, "Saving platform data\n"); + platform_set_drvdata(pdev, gdev); + return 0; + +err_free_register: + uio_unregister_device(p); + iounmap(gdev->prussio_vaddr); +err_free_info: + kfree(gdev->info); +err_free_gdev: + kfree(gdev); + + return ret; +} + +static int pruss_shmem_remove(struct platform_device *dev) +{ + struct uio_pruss_shmem_dev *gdev = platform_get_drvdata(dev); + + pruss_shmem_cleanup(&dev->dev, gdev); + return 0; +} + +static const struct of_device_id pruss_shmem_dt_ids[] = { + { .compatible = "ti,pruss-shmem", .data = NULL, }, + {}, +}; +MODULE_DEVICE_TABLE(of, pruss_shmem_dt_ids); + +static struct platform_driver pruss_shmem_driver = { + .probe = pruss_shmem_probe, + .remove = pruss_shmem_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = pruss_shmem_dt_ids, + }, +}; + +module_platform_driver(pruss_shmem_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_AUTHOR("Amit Chatterjee "); +MODULE_AUTHOR("Pratheesh Gangadhar "); +MODULE_AUTHOR("Jason Kridner "); diff --git a/firmware/am335x-bone-scale-data.bin b/firmware/am335x-bone-scale-data.bin new file mode 100644 index 0000000000000..1ce3c1c596d7a Binary files /dev/null and b/firmware/am335x-bone-scale-data.bin differ diff --git a/firmware/am335x-evm-scale-data.bin b/firmware/am335x-evm-scale-data.bin new file mode 100644 index 0000000000000..a222389d233fa Binary files /dev/null and b/firmware/am335x-evm-scale-data.bin differ diff --git a/firmware/am335x-pm-firmware.bin b/firmware/am335x-pm-firmware.bin new file mode 100755 index 0000000000000..1fc92e2bc40e9 Binary files /dev/null and b/firmware/am335x-pm-firmware.bin differ diff --git a/firmware/am335x-pm-firmware.elf b/firmware/am335x-pm-firmware.elf new file mode 100755 index 0000000000000..843fc22ce3484 Binary files /dev/null and b/firmware/am335x-pm-firmware.elf differ diff --git a/firmware/am43x-evm-scale-data.bin b/firmware/am43x-evm-scale-data.bin new file mode 100644 index 0000000000000..2d71341089816 Binary files /dev/null and b/firmware/am43x-evm-scale-data.bin differ diff --git a/fs/Kconfig b/fs/Kconfig index 7aee6d699fd6b..ec92031204c5a 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -248,6 +248,7 @@ source "fs/pstore/Kconfig" source "fs/sysv/Kconfig" source "fs/ufs/Kconfig" source "fs/exofs/Kconfig" +source "fs/aufs/Kconfig" endif # MISC_FILESYSTEMS diff --git a/fs/Makefile b/fs/Makefile index ef772f1eaff80..51779e68c0d07 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -129,3 +129,4 @@ obj-y += exofs/ # Multiple modules obj-$(CONFIG_CEPH_FS) += ceph/ obj-$(CONFIG_PSTORE) += pstore/ obj-$(CONFIG_EFIVAR_FS) += efivarfs/ +obj-$(CONFIG_AUFS_FS) += aufs/ diff --git a/fs/aufs/Kconfig b/fs/aufs/Kconfig new file mode 100644 index 0000000000000..9f436425716a4 --- /dev/null +++ b/fs/aufs/Kconfig @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: GPL-2.0 +config AUFS_FS + tristate "Aufs (Advanced multi layered unification filesystem) support" + help + Aufs is a stackable unification filesystem such as Unionfs, + which unifies several directories and provides a merged single + directory. + In the early days, aufs was entirely re-designed and + re-implemented Unionfs Version 1.x series. Introducing many + original ideas, approaches and improvements, it becomes totally + different from Unionfs while keeping the basic features. + +if AUFS_FS +choice + prompt "Maximum number of branches" + default AUFS_BRANCH_MAX_127 + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_127 + bool "127" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_511 + bool "511" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_1023 + bool "1023" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_32767 + bool "32767" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +endchoice + +config AUFS_SBILIST + bool + depends on AUFS_MAGIC_SYSRQ || PROC_FS + default y + help + Automatic configuration for internal use. + When aufs supports Magic SysRq or /proc, enabled automatically. + +config AUFS_HNOTIFY + bool "Detect direct branch access (bypassing aufs)" + help + If you want to modify files on branches directly, eg. bypassing aufs, + and want aufs to detect the changes of them fully, then enable this + option and use 'udba=notify' mount option. + Currently there is only one available configuration, "fsnotify". + It will have a negative impact to the performance. + See detail in aufs.5. + +choice + prompt "method" if AUFS_HNOTIFY + default AUFS_HFSNOTIFY +config AUFS_HFSNOTIFY + bool "fsnotify" + select FSNOTIFY +endchoice + +config AUFS_EXPORT + bool "NFS-exportable aufs" + depends on EXPORTFS + help + If you want to export your mounted aufs via NFS, then enable this + option. There are several requirements for this configuration. + See detail in aufs.5. + +config AUFS_INO_T_64 + bool + depends on AUFS_EXPORT + depends on 64BIT && !(ALPHA || S390) + default y + help + Automatic configuration for internal use. + /* typedef unsigned long/int __kernel_ino_t */ + /* alpha and s390x are int */ + +config AUFS_XATTR + bool "support for XATTR/EA (including Security Labels)" + help + If your branch fs supports XATTR/EA and you want to make them + available in aufs too, then enable this opsion and specify the + branch attributes for EA. + See detail in aufs.5. + +config AUFS_FHSM + bool "File-based Hierarchical Storage Management" + help + Hierarchical Storage Management (or HSM) is a well-known feature + in the storage world. Aufs provides this feature as file-based. + with multiple branches. + These multiple branches are prioritized, ie. the topmost one + should be the fastest drive and be used heavily. + +config AUFS_RDU + bool "Readdir in userspace" + help + Aufs has two methods to provide a merged view for a directory, + by a user-space library and by kernel-space natively. The latter + is always enabled but sometimes large and slow. + If you enable this option, install the library in aufs2-util + package, and set some environment variables for your readdir(3), + then the work will be handled in user-space which generally + shows better performance in most cases. + See detail in aufs.5. + +config AUFS_DIRREN + bool "Workaround for rename(2)-ing a directory" + help + By default, aufs returns EXDEV error in renameing a dir who has + his child on the lower branch, since it is a bad idea to issue + rename(2) internally for every lower branch. But user may not + accept this behaviour. So here is a workaround to allow such + rename(2) and store some extra infromation on the writable + branch. Obviously this costs high (and I don't like it). + To use this feature, you need to enable this configuration AND + to specify the mount option `dirren.' + See details in aufs.5 and the design documents. + +config AUFS_SHWH + bool "Show whiteouts" + help + If you want to make the whiteouts in aufs visible, then enable + this option and specify 'shwh' mount option. Although it may + sounds like philosophy or something, but in technically it + simply shows the name of whiteout with keeping its behaviour. + +config AUFS_BR_RAMFS + bool "Ramfs (initramfs/rootfs) as an aufs branch" + help + If you want to use ramfs as an aufs branch fs, then enable this + option. Generally tmpfs is recommended. + Aufs prohibited them to be a branch fs by default, because + initramfs becomes unusable after switch_root or something + generally. If you sets initramfs as an aufs branch and boot your + system by switch_root, you will meet a problem easily since the + files in initramfs may be inaccessible. + Unless you are going to use ramfs as an aufs branch fs without + switch_root or something, leave it N. + +config AUFS_BR_FUSE + bool "Fuse fs as an aufs branch" + depends on FUSE_FS + select AUFS_POLL + help + If you want to use fuse-based userspace filesystem as an aufs + branch fs, then enable this option. + It implements the internal poll(2) operation which is + implemented by fuse only (curretnly). + +config AUFS_POLL + bool + help + Automatic configuration for internal use. + +config AUFS_BR_HFSPLUS + bool "Hfsplus as an aufs branch" + depends on HFSPLUS_FS + default y + help + If you want to use hfsplus fs as an aufs branch fs, then enable + this option. This option introduces a small overhead at + copying-up a file on hfsplus. + +config AUFS_BDEV_LOOP + bool + depends on BLK_DEV_LOOP + default y + help + Automatic configuration for internal use. + Convert =[ym] into =y. + +config AUFS_DEBUG + bool "Debug aufs" + help + Enable this to compile aufs internal debug code. + It will have a negative impact to the performance. + +config AUFS_MAGIC_SYSRQ + bool + depends on AUFS_DEBUG && MAGIC_SYSRQ + default y + help + Automatic configuration for internal use. + When aufs supports Magic SysRq, enabled automatically. +endif diff --git a/fs/aufs/Makefile b/fs/aufs/Makefile new file mode 100644 index 0000000000000..2c819a64935e6 --- /dev/null +++ b/fs/aufs/Makefile @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0 + +include ${src}/magic.mk +ifeq (${CONFIG_AUFS_FS},m) +include ${src}/conf.mk +endif +-include ${src}/priv_def.mk + +# cf. include/linux/kernel.h +# enable pr_debug +ccflags-y += -DDEBUG +# sparse requires the full pathname +ifdef M +ccflags-y += -include ${M}/../../include/uapi/linux/aufs_type.h +else +ccflags-y += -include ${srctree}/include/uapi/linux/aufs_type.h +endif + +obj-$(CONFIG_AUFS_FS) += aufs.o +aufs-y := module.o sbinfo.o super.o branch.o xino.o sysaufs.o opts.o \ + wkq.o vfsub.o dcsub.o \ + cpup.o whout.o wbr_policy.o \ + dinfo.o dentry.o \ + dynop.o \ + finfo.o file.o f_op.o \ + dir.o vdir.o \ + iinfo.o inode.o i_op.o i_op_add.o i_op_del.o i_op_ren.o \ + mvdown.o ioctl.o + +# all are boolean +aufs-$(CONFIG_PROC_FS) += procfs.o plink.o +aufs-$(CONFIG_SYSFS) += sysfs.o +aufs-$(CONFIG_DEBUG_FS) += dbgaufs.o +aufs-$(CONFIG_AUFS_BDEV_LOOP) += loop.o +aufs-$(CONFIG_AUFS_HNOTIFY) += hnotify.o +aufs-$(CONFIG_AUFS_HFSNOTIFY) += hfsnotify.o +aufs-$(CONFIG_AUFS_EXPORT) += export.o +aufs-$(CONFIG_AUFS_XATTR) += xattr.o +aufs-$(CONFIG_FS_POSIX_ACL) += posix_acl.o +aufs-$(CONFIG_AUFS_DIRREN) += dirren.o +aufs-$(CONFIG_AUFS_FHSM) += fhsm.o +aufs-$(CONFIG_AUFS_POLL) += poll.o +aufs-$(CONFIG_AUFS_RDU) += rdu.o +aufs-$(CONFIG_AUFS_BR_HFSPLUS) += hfsplus.o +aufs-$(CONFIG_AUFS_DEBUG) += debug.o +aufs-$(CONFIG_AUFS_MAGIC_SYSRQ) += sysrq.o diff --git a/fs/aufs/aufs.h b/fs/aufs/aufs.h new file mode 100644 index 0000000000000..a62a85211b883 --- /dev/null +++ b/fs/aufs/aufs.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * all header files + */ + +#ifndef __AUFS_H__ +#define __AUFS_H__ + +#ifdef __KERNEL__ + +#define AuStub(type, name, body, ...) \ + static inline type name(__VA_ARGS__) { body; } + +#define AuStubVoid(name, ...) \ + AuStub(void, name, , __VA_ARGS__) +#define AuStubInt0(name, ...) \ + AuStub(int, name, return 0, __VA_ARGS__) + +#include "debug.h" + +#include "branch.h" +#include "cpup.h" +#include "dcsub.h" +#include "dbgaufs.h" +#include "dentry.h" +#include "dir.h" +#include "dirren.h" +#include "dynop.h" +#include "file.h" +#include "fstype.h" +#include "hbl.h" +#include "inode.h" +#include "lcnt.h" +#include "loop.h" +#include "module.h" +#include "opts.h" +#include "rwsem.h" +#include "super.h" +#include "sysaufs.h" +#include "vfsub.h" +#include "whout.h" +#include "wkq.h" + +#endif /* __KERNEL__ */ +#endif /* __AUFS_H__ */ diff --git a/fs/aufs/branch.c b/fs/aufs/branch.c new file mode 100644 index 0000000000000..598cf370a1dca --- /dev/null +++ b/fs/aufs/branch.c @@ -0,0 +1,1422 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * branch management + */ + +#include +#include +#include "aufs.h" + +/* + * free a single branch + */ +static void au_br_do_free(struct au_branch *br) +{ + int i; + struct au_wbr *wbr; + struct au_dykey **key; + + au_hnotify_fin_br(br); + /* always, regardless the mount option */ + au_dr_hino_free(&br->br_dirren); + au_xino_put(br); + + AuLCntZero(au_lcnt_read(&br->br_nfiles, /*do_rev*/0)); + au_lcnt_fin(&br->br_nfiles, /*do_sync*/0); + AuLCntZero(au_lcnt_read(&br->br_count, /*do_rev*/0)); + au_lcnt_fin(&br->br_count, /*do_sync*/0); + + wbr = br->br_wbr; + if (wbr) { + for (i = 0; i < AuBrWh_Last; i++) + dput(wbr->wbr_wh[i]); + AuDebugOn(atomic_read(&wbr->wbr_wh_running)); + AuRwDestroy(&wbr->wbr_wh_rwsem); + } + + if (br->br_fhsm) { + au_br_fhsm_fin(br->br_fhsm); + au_kfree_try_rcu(br->br_fhsm); + } + + key = br->br_dykey; + for (i = 0; i < AuBrDynOp; i++, key++) + if (*key) + au_dy_put(*key); + else + break; + + /* recursive lock, s_umount of branch's */ + /* synchronize_rcu(); */ /* why? */ + lockdep_off(); + path_put(&br->br_path); + lockdep_on(); + au_kfree_rcu(wbr); + au_lcnt_wait_for_fin(&br->br_nfiles); + au_lcnt_wait_for_fin(&br->br_count); + /* I don't know why, but percpu_refcount requires this */ + /* synchronize_rcu(); */ + au_kfree_rcu(br); +} + +/* + * frees all branches + */ +void au_br_free(struct au_sbinfo *sbinfo) +{ + aufs_bindex_t bmax; + struct au_branch **br; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + bmax = sbinfo->si_bbot + 1; + br = sbinfo->si_branch; + while (bmax--) + au_br_do_free(*br++); +} + +/* + * find the index of a branch which is specified by @br_id. + */ +int au_br_index(struct super_block *sb, aufs_bindex_t br_id) +{ + aufs_bindex_t bindex, bbot; + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) + if (au_sbr_id(sb, bindex) == br_id) + return bindex; + return -1; +} + +/* ---------------------------------------------------------------------- */ + +/* + * add a branch + */ + +static int test_overlap(struct super_block *sb, struct dentry *h_adding, + struct dentry *h_root) +{ + if (unlikely(h_adding == h_root + || au_test_loopback_overlap(sb, h_adding))) + return 1; + if (h_adding->d_sb != h_root->d_sb) + return 0; + return au_test_subdir(h_adding, h_root) + || au_test_subdir(h_root, h_adding); +} + +/* + * returns a newly allocated branch. @new_nbranch is a number of branches + * after adding a branch. + */ +static struct au_branch *au_br_alloc(struct super_block *sb, int new_nbranch, + int perm) +{ + struct au_branch *add_branch; + struct dentry *root; + struct inode *inode; + int err; + + err = -ENOMEM; + add_branch = kzalloc(sizeof(*add_branch), GFP_NOFS); + if (unlikely(!add_branch)) + goto out; + add_branch->br_xino = au_xino_alloc(/*nfile*/1); + if (unlikely(!add_branch->br_xino)) + goto out_br; + err = au_hnotify_init_br(add_branch, perm); + if (unlikely(err)) + goto out_xino; + + if (au_br_writable(perm)) { + /* may be freed separately at changing the branch permission */ + add_branch->br_wbr = kzalloc(sizeof(*add_branch->br_wbr), + GFP_NOFS); + if (unlikely(!add_branch->br_wbr)) + goto out_hnotify; + } + + if (au_br_fhsm(perm)) { + err = au_fhsm_br_alloc(add_branch); + if (unlikely(err)) + goto out_wbr; + } + + root = sb->s_root; + err = au_sbr_realloc(au_sbi(sb), new_nbranch, /*may_shrink*/0); + if (!err) + err = au_di_realloc(au_di(root), new_nbranch, /*may_shrink*/0); + if (!err) { + inode = d_inode(root); + err = au_hinode_realloc(au_ii(inode), new_nbranch, + /*may_shrink*/0); + } + if (!err) + return add_branch; /* success */ + +out_wbr: + au_kfree_rcu(add_branch->br_wbr); +out_hnotify: + au_hnotify_fin_br(add_branch); +out_xino: + au_xino_put(add_branch); +out_br: + au_kfree_rcu(add_branch); +out: + return ERR_PTR(err); +} + +/* + * test if the branch permission is legal or not. + */ +static int test_br(struct inode *inode, int brperm, char *path) +{ + int err; + + err = (au_br_writable(brperm) && IS_RDONLY(inode)); + if (!err) + goto out; + + err = -EINVAL; + pr_err("write permission for readonly mount or inode, %s\n", path); + +out: + return err; +} + +/* + * returns: + * 0: success, the caller will add it + * plus: success, it is already unified, the caller should ignore it + * minus: error + */ +static int test_add(struct super_block *sb, struct au_opt_add *add, int remount) +{ + int err; + aufs_bindex_t bbot, bindex; + struct dentry *root, *h_dentry; + struct inode *inode, *h_inode; + + root = sb->s_root; + bbot = au_sbbot(sb); + if (unlikely(bbot >= 0 + && au_find_dbindex(root, add->path.dentry) >= 0)) { + err = 1; + if (!remount) { + err = -EINVAL; + pr_err("%s duplicated\n", add->pathname); + } + goto out; + } + + err = -ENOSPC; /* -E2BIG; */ + if (unlikely(AUFS_BRANCH_MAX <= add->bindex + || AUFS_BRANCH_MAX - 1 <= bbot)) { + pr_err("number of branches exceeded %s\n", add->pathname); + goto out; + } + + err = -EDOM; + if (unlikely(add->bindex < 0 || bbot + 1 < add->bindex)) { + pr_err("bad index %d\n", add->bindex); + goto out; + } + + inode = d_inode(add->path.dentry); + err = -ENOENT; + if (unlikely(!inode->i_nlink)) { + pr_err("no existence %s\n", add->pathname); + goto out; + } + + err = -EINVAL; + if (unlikely(inode->i_sb == sb)) { + pr_err("%s must be outside\n", add->pathname); + goto out; + } + + if (unlikely(au_test_fs_unsuppoted(inode->i_sb))) { + pr_err("unsupported filesystem, %s (%s)\n", + add->pathname, au_sbtype(inode->i_sb)); + goto out; + } + + if (unlikely(inode->i_sb->s_stack_depth)) { + pr_err("already stacked, %s (%s)\n", + add->pathname, au_sbtype(inode->i_sb)); + goto out; + } + + err = test_br(d_inode(add->path.dentry), add->perm, add->pathname); + if (unlikely(err)) + goto out; + + if (bbot < 0) + return 0; /* success */ + + err = -EINVAL; + for (bindex = 0; bindex <= bbot; bindex++) + if (unlikely(test_overlap(sb, add->path.dentry, + au_h_dptr(root, bindex)))) { + pr_err("%s is overlapped\n", add->pathname); + goto out; + } + + err = 0; + if (au_opt_test(au_mntflags(sb), WARN_PERM)) { + h_dentry = au_h_dptr(root, 0); + h_inode = d_inode(h_dentry); + if ((h_inode->i_mode & S_IALLUGO) != (inode->i_mode & S_IALLUGO) + || !uid_eq(h_inode->i_uid, inode->i_uid) + || !gid_eq(h_inode->i_gid, inode->i_gid)) + pr_warn("uid/gid/perm %s %u/%u/0%o, %u/%u/0%o\n", + add->pathname, + i_uid_read(inode), i_gid_read(inode), + (inode->i_mode & S_IALLUGO), + i_uid_read(h_inode), i_gid_read(h_inode), + (h_inode->i_mode & S_IALLUGO)); + } + +out: + return err; +} + +/* + * initialize or clean the whiteouts for an adding branch + */ +static int au_br_init_wh(struct super_block *sb, struct au_branch *br, + int new_perm) +{ + int err, old_perm; + aufs_bindex_t bindex; + struct inode *h_inode; + struct au_wbr *wbr; + struct au_hinode *hdir; + struct dentry *h_dentry; + + err = vfsub_mnt_want_write(au_br_mnt(br)); + if (unlikely(err)) + goto out; + + wbr = br->br_wbr; + old_perm = br->br_perm; + br->br_perm = new_perm; + hdir = NULL; + h_inode = NULL; + bindex = au_br_index(sb, br->br_id); + if (0 <= bindex) { + hdir = au_hi(d_inode(sb->s_root), bindex); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + } else { + h_dentry = au_br_dentry(br); + h_inode = d_inode(h_dentry); + inode_lock_nested(h_inode, AuLsc_I_PARENT); + } + if (!wbr) + err = au_wh_init(br, sb); + else { + wbr_wh_write_lock(wbr); + err = au_wh_init(br, sb); + wbr_wh_write_unlock(wbr); + } + if (hdir) + au_hn_inode_unlock(hdir); + else + inode_unlock(h_inode); + vfsub_mnt_drop_write(au_br_mnt(br)); + br->br_perm = old_perm; + + if (!err && wbr && !au_br_writable(new_perm)) { + au_kfree_rcu(wbr); + br->br_wbr = NULL; + } + +out: + return err; +} + +static int au_wbr_init(struct au_branch *br, struct super_block *sb, + int perm) +{ + int err; + struct kstatfs kst; + struct au_wbr *wbr; + + wbr = br->br_wbr; + au_rw_init(&wbr->wbr_wh_rwsem); + atomic_set(&wbr->wbr_wh_running, 0); + + /* + * a limit for rmdir/rename a dir + * cf. AUFS_MAX_NAMELEN in include/uapi/linux/aufs_type.h + */ + err = vfs_statfs(&br->br_path, &kst); + if (unlikely(err)) + goto out; + err = -EINVAL; + if (kst.f_namelen >= NAME_MAX) + err = au_br_init_wh(sb, br, perm); + else + pr_err("%pd(%s), unsupported namelen %ld\n", + au_br_dentry(br), + au_sbtype(au_br_dentry(br)->d_sb), kst.f_namelen); + +out: + return err; +} + +/* initialize a new branch */ +static int au_br_init(struct au_branch *br, struct super_block *sb, + struct au_opt_add *add) +{ + int err; + struct au_branch *brbase; + struct file *xf; + struct inode *h_inode; + + err = 0; + br->br_perm = add->perm; + br->br_path = add->path; /* set first, path_get() later */ + spin_lock_init(&br->br_dykey_lock); + au_lcnt_init(&br->br_nfiles, /*release*/NULL); + au_lcnt_init(&br->br_count, /*release*/NULL); + br->br_id = au_new_br_id(sb); + AuDebugOn(br->br_id < 0); + + /* always, regardless the given option */ + err = au_dr_br_init(sb, br, &add->path); + if (unlikely(err)) + goto out_err; + + if (au_br_writable(add->perm)) { + err = au_wbr_init(br, sb, add->perm); + if (unlikely(err)) + goto out_err; + } + + if (au_opt_test(au_mntflags(sb), XINO)) { + brbase = au_sbr(sb, 0); + xf = au_xino_file(brbase->br_xino, /*idx*/-1); + AuDebugOn(!xf); + h_inode = d_inode(add->path.dentry); + err = au_xino_init_br(sb, br, h_inode->i_ino, &xf->f_path); + if (unlikely(err)) { + AuDebugOn(au_xino_file(br->br_xino, /*idx*/-1)); + goto out_err; + } + } + + sysaufs_br_init(br); + path_get(&br->br_path); + goto out; /* success */ + +out_err: + memset(&br->br_path, 0, sizeof(br->br_path)); +out: + return err; +} + +static void au_br_do_add_brp(struct au_sbinfo *sbinfo, aufs_bindex_t bindex, + struct au_branch *br, aufs_bindex_t bbot, + aufs_bindex_t amount) +{ + struct au_branch **brp; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + brp = sbinfo->si_branch + bindex; + memmove(brp + 1, brp, sizeof(*brp) * amount); + *brp = br; + sbinfo->si_bbot++; + if (unlikely(bbot < 0)) + sbinfo->si_bbot = 0; +} + +static void au_br_do_add_hdp(struct au_dinfo *dinfo, aufs_bindex_t bindex, + aufs_bindex_t bbot, aufs_bindex_t amount) +{ + struct au_hdentry *hdp; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + hdp = au_hdentry(dinfo, bindex); + memmove(hdp + 1, hdp, sizeof(*hdp) * amount); + au_h_dentry_init(hdp); + dinfo->di_bbot++; + if (unlikely(bbot < 0)) + dinfo->di_btop = 0; +} + +static void au_br_do_add_hip(struct au_iinfo *iinfo, aufs_bindex_t bindex, + aufs_bindex_t bbot, aufs_bindex_t amount) +{ + struct au_hinode *hip; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + hip = au_hinode(iinfo, bindex); + memmove(hip + 1, hip, sizeof(*hip) * amount); + au_hinode_init(hip); + iinfo->ii_bbot++; + if (unlikely(bbot < 0)) + iinfo->ii_btop = 0; +} + +static void au_br_do_add(struct super_block *sb, struct au_branch *br, + aufs_bindex_t bindex) +{ + struct dentry *root, *h_dentry; + struct inode *root_inode, *h_inode; + aufs_bindex_t bbot, amount; + + root = sb->s_root; + root_inode = d_inode(root); + bbot = au_sbbot(sb); + amount = bbot + 1 - bindex; + h_dentry = au_br_dentry(br); + au_sbilist_lock(); + au_br_do_add_brp(au_sbi(sb), bindex, br, bbot, amount); + au_br_do_add_hdp(au_di(root), bindex, bbot, amount); + au_br_do_add_hip(au_ii(root_inode), bindex, bbot, amount); + au_set_h_dptr(root, bindex, dget(h_dentry)); + h_inode = d_inode(h_dentry); + au_set_h_iptr(root_inode, bindex, au_igrab(h_inode), /*flags*/0); + au_sbilist_unlock(); +} + +int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount) +{ + int err; + aufs_bindex_t bbot, add_bindex; + struct dentry *root, *h_dentry; + struct inode *root_inode; + struct au_branch *add_branch; + + root = sb->s_root; + root_inode = d_inode(root); + IMustLock(root_inode); + IiMustWriteLock(root_inode); + err = test_add(sb, add, remount); + if (unlikely(err < 0)) + goto out; + if (err) { + err = 0; + goto out; /* success */ + } + + bbot = au_sbbot(sb); + add_branch = au_br_alloc(sb, bbot + 2, add->perm); + err = PTR_ERR(add_branch); + if (IS_ERR(add_branch)) + goto out; + + err = au_br_init(add_branch, sb, add); + if (unlikely(err)) { + au_br_do_free(add_branch); + goto out; + } + + add_bindex = add->bindex; + sysaufs_brs_del(sb, add_bindex); /* remove successors */ + au_br_do_add(sb, add_branch, add_bindex); + sysaufs_brs_add(sb, add_bindex); /* append successors */ + dbgaufs_brs_add(sb, add_bindex, /*topdown*/0); /* rename successors */ + + h_dentry = add->path.dentry; + if (!add_bindex) { + au_cpup_attr_all(root_inode, /*force*/1); + sb->s_maxbytes = h_dentry->d_sb->s_maxbytes; + } else + au_add_nlink(root_inode, d_inode(h_dentry)); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static unsigned long long au_farray_cb(struct super_block *sb, void *a, + unsigned long long max __maybe_unused, + void *arg) +{ + unsigned long long n; + struct file **p, *f; + struct hlist_bl_head *files; + struct hlist_bl_node *pos; + struct au_finfo *finfo; + + n = 0; + p = a; + files = &au_sbi(sb)->si_files; + hlist_bl_lock(files); + hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) { + f = finfo->fi_file; + if (file_count(f) + && !special_file(file_inode(f)->i_mode)) { + get_file(f); + *p++ = f; + n++; + AuDebugOn(n > max); + } + } + hlist_bl_unlock(files); + + return n; +} + +static struct file **au_farray_alloc(struct super_block *sb, + unsigned long long *max) +{ + struct au_sbinfo *sbi; + + sbi = au_sbi(sb); + *max = au_lcnt_read(&sbi->si_nfiles, /*do_rev*/1); + return au_array_alloc(max, au_farray_cb, sb, /*arg*/NULL); +} + +static void au_farray_free(struct file **a, unsigned long long max) +{ + unsigned long long ull; + + for (ull = 0; ull < max; ull++) + if (a[ull]) + fput(a[ull]); + kvfree(a); +} + +/* ---------------------------------------------------------------------- */ + +/* + * delete a branch + */ + +/* to show the line number, do not make it inlined function */ +#define AuVerbose(do_info, fmt, ...) do { \ + if (do_info) \ + pr_info(fmt, ##__VA_ARGS__); \ +} while (0) + +static int au_test_ibusy(struct inode *inode, aufs_bindex_t btop, + aufs_bindex_t bbot) +{ + return (inode && !S_ISDIR(inode->i_mode)) || btop == bbot; +} + +static int au_test_dbusy(struct dentry *dentry, aufs_bindex_t btop, + aufs_bindex_t bbot) +{ + return au_test_ibusy(d_inode(dentry), btop, bbot); +} + +/* + * test if the branch is deletable or not. + */ +static int test_dentry_busy(struct dentry *root, aufs_bindex_t bindex, + unsigned int sigen, const unsigned int verbose) +{ + int err, i, j, ndentry; + aufs_bindex_t btop, bbot; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry *d; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, root, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + for (i = 0; !err && i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = 0; !err && j < ndentry; j++) { + d = dpage->dentries[j]; + AuDebugOn(au_dcount(d) <= 0); + if (!au_digen_test(d, sigen)) { + di_read_lock_child(d, AuLock_IR); + if (unlikely(au_dbrange_test(d))) { + di_read_unlock(d, AuLock_IR); + continue; + } + } else { + di_write_lock_child(d); + if (unlikely(au_dbrange_test(d))) { + di_write_unlock(d); + continue; + } + err = au_reval_dpath(d, sigen); + if (!err) + di_downgrade_lock(d, AuLock_IR); + else { + di_write_unlock(d); + break; + } + } + + /* AuDbgDentry(d); */ + btop = au_dbtop(d); + bbot = au_dbbot(d); + if (btop <= bindex + && bindex <= bbot + && au_h_dptr(d, bindex) + && au_test_dbusy(d, btop, bbot)) { + err = -EBUSY; + AuVerbose(verbose, "busy %pd\n", d); + AuDbgDentry(d); + } + di_read_unlock(d, AuLock_IR); + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static int test_inode_busy(struct super_block *sb, aufs_bindex_t bindex, + unsigned int sigen, const unsigned int verbose) +{ + int err; + unsigned long long max, ull; + struct inode *i, **array; + aufs_bindex_t btop, bbot; + + array = au_iarray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + AuDbg("b%d\n", bindex); + for (ull = 0; !err && ull < max; ull++) { + i = array[ull]; + if (unlikely(!i)) + break; + if (i->i_ino == AUFS_ROOT_INO) + continue; + + /* AuDbgInode(i); */ + if (au_iigen(i, NULL) == sigen) + ii_read_lock_child(i); + else { + ii_write_lock_child(i); + err = au_refresh_hinode_self(i); + au_iigen_dec(i); + if (!err) + ii_downgrade_lock(i); + else { + ii_write_unlock(i); + break; + } + } + + btop = au_ibtop(i); + bbot = au_ibbot(i); + if (btop <= bindex + && bindex <= bbot + && au_h_iptr(i, bindex) + && au_test_ibusy(i, btop, bbot)) { + err = -EBUSY; + AuVerbose(verbose, "busy i%lu\n", i->i_ino); + AuDbgInode(i); + } + ii_read_unlock(i); + } + au_iarray_free(array, max); + +out: + return err; +} + +static int test_children_busy(struct dentry *root, aufs_bindex_t bindex, + const unsigned int verbose) +{ + int err; + unsigned int sigen; + + sigen = au_sigen(root->d_sb); + DiMustNoWaiters(root); + IiMustNoWaiters(d_inode(root)); + di_write_unlock(root); + err = test_dentry_busy(root, bindex, sigen, verbose); + if (!err) + err = test_inode_busy(root->d_sb, bindex, sigen, verbose); + di_write_lock_child(root); /* aufs_write_lock() calls ..._child() */ + + return err; +} + +static int test_dir_busy(struct file *file, aufs_bindex_t br_id, + struct file **to_free, int *idx) +{ + int err; + unsigned char matched, root; + aufs_bindex_t bindex, bbot; + struct au_fidir *fidir; + struct au_hfile *hfile; + + err = 0; + root = IS_ROOT(file->f_path.dentry); + if (root) { + get_file(file); + to_free[*idx] = file; + (*idx)++; + goto out; + } + + matched = 0; + fidir = au_fi(file)->fi_hdir; + AuDebugOn(!fidir); + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); bindex <= bbot; bindex++) { + hfile = fidir->fd_hfile + bindex; + if (!hfile->hf_file) + continue; + + if (hfile->hf_br->br_id == br_id) { + matched = 1; + break; + } + } + if (matched) + err = -EBUSY; + +out: + return err; +} + +static int test_file_busy(struct super_block *sb, aufs_bindex_t br_id, + struct file **to_free, int opened) +{ + int err, idx; + unsigned long long ull, max; + aufs_bindex_t btop; + struct file *file, **array; + struct dentry *root; + struct au_hfile *hfile; + + array = au_farray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + idx = 0; + root = sb->s_root; + di_write_unlock(root); + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + fi_read_lock(file); + btop = au_fbtop(file); + if (!d_is_dir(file->f_path.dentry)) { + hfile = &au_fi(file)->fi_htop; + if (hfile->hf_br->br_id == br_id) + err = -EBUSY; + } else + err = test_dir_busy(file, br_id, to_free, &idx); + fi_read_unlock(file); + if (unlikely(err)) + break; + } + di_write_lock_child(root); + au_farray_free(array, max); + AuDebugOn(idx > opened); + +out: + return err; +} + +static void br_del_file(struct file **to_free, unsigned long long opened, + aufs_bindex_t br_id) +{ + unsigned long long ull; + aufs_bindex_t bindex, btop, bbot, bfound; + struct file *file; + struct au_fidir *fidir; + struct au_hfile *hfile; + + for (ull = 0; ull < opened; ull++) { + file = to_free[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + AuDebugOn(!d_is_dir(file->f_path.dentry)); + bfound = -1; + fidir = au_fi(file)->fi_hdir; + AuDebugOn(!fidir); + fi_write_lock(file); + btop = au_fbtop(file); + bbot = au_fbbot_dir(file); + for (bindex = btop; bindex <= bbot; bindex++) { + hfile = fidir->fd_hfile + bindex; + if (!hfile->hf_file) + continue; + + if (hfile->hf_br->br_id == br_id) { + bfound = bindex; + break; + } + } + AuDebugOn(bfound < 0); + au_set_h_fptr(file, bfound, NULL); + if (bfound == btop) { + for (btop++; btop <= bbot; btop++) + if (au_hf_dir(file, btop)) { + au_set_fbtop(file, btop); + break; + } + } + fi_write_unlock(file); + } +} + +static void au_br_do_del_brp(struct au_sbinfo *sbinfo, + const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_branch **brp, **p; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + brp = sbinfo->si_branch + bindex; + if (bindex < bbot) + memmove(brp, brp + 1, sizeof(*brp) * (bbot - bindex)); + sbinfo->si_branch[0 + bbot] = NULL; + sbinfo->si_bbot--; + + p = au_krealloc(sbinfo->si_branch, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + sbinfo->si_branch = p; + /* harmless error */ +} + +static void au_br_do_del_hdp(struct au_dinfo *dinfo, const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_hdentry *hdp, *p; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + hdp = au_hdentry(dinfo, bindex); + if (bindex < bbot) + memmove(hdp, hdp + 1, sizeof(*hdp) * (bbot - bindex)); + /* au_h_dentry_init(au_hdentry(dinfo, bbot); */ + dinfo->di_bbot--; + + p = au_krealloc(dinfo->di_hdentry, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + dinfo->di_hdentry = p; + /* harmless error */ +} + +static void au_br_do_del_hip(struct au_iinfo *iinfo, const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_hinode *hip, *p; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + hip = au_hinode(iinfo, bindex); + if (bindex < bbot) + memmove(hip, hip + 1, sizeof(*hip) * (bbot - bindex)); + /* au_hinode_init(au_hinode(iinfo, bbot)); */ + iinfo->ii_bbot--; + + p = au_krealloc(iinfo->ii_hinode, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + iinfo->ii_hinode = p; + /* harmless error */ +} + +static void au_br_do_del(struct super_block *sb, aufs_bindex_t bindex, + struct au_branch *br) +{ + aufs_bindex_t bbot; + struct au_sbinfo *sbinfo; + struct dentry *root, *h_root; + struct inode *inode, *h_inode; + struct au_hinode *hinode; + + SiMustWriteLock(sb); + + root = sb->s_root; + inode = d_inode(root); + sbinfo = au_sbi(sb); + bbot = sbinfo->si_bbot; + + h_root = au_h_dptr(root, bindex); + hinode = au_hi(inode, bindex); + h_inode = au_igrab(hinode->hi_inode); + au_hiput(hinode); + + au_sbilist_lock(); + au_br_do_del_brp(sbinfo, bindex, bbot); + au_br_do_del_hdp(au_di(root), bindex, bbot); + au_br_do_del_hip(au_ii(inode), bindex, bbot); + au_sbilist_unlock(); + + /* ignore an error */ + au_dr_br_fin(sb, br); /* always, regardless the mount option */ + + dput(h_root); + iput(h_inode); + au_br_do_free(br); +} + +static unsigned long long empty_cb(struct super_block *sb, void *array, + unsigned long long max, void *arg) +{ + return max; +} + +int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount) +{ + int err, rerr, i; + unsigned long long opened; + unsigned int mnt_flags; + aufs_bindex_t bindex, bbot, br_id; + unsigned char do_wh, verbose; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *root; + struct file **to_free; + + err = 0; + opened = 0; + to_free = NULL; + root = sb->s_root; + bindex = au_find_dbindex(root, del->h_path.dentry); + if (bindex < 0) { + if (remount) + goto out; /* success */ + err = -ENOENT; + pr_err("%s no such branch\n", del->pathname); + goto out; + } + AuDbg("bindex b%d\n", bindex); + + err = -EBUSY; + mnt_flags = au_mntflags(sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + bbot = au_sbbot(sb); + if (unlikely(!bbot)) { + AuVerbose(verbose, "no more branches left\n"); + goto out; + } + + br = au_sbr(sb, bindex); + AuDebugOn(!path_equal(&br->br_path, &del->h_path)); + if (unlikely(au_lcnt_read(&br->br_count, /*do_rev*/1))) { + AuVerbose(verbose, "br %pd2 is busy now\n", del->h_path.dentry); + goto out; + } + + br_id = br->br_id; + opened = au_lcnt_read(&br->br_nfiles, /*do_rev*/1); + if (unlikely(opened)) { + to_free = au_array_alloc(&opened, empty_cb, sb, NULL); + err = PTR_ERR(to_free); + if (IS_ERR(to_free)) + goto out; + + err = test_file_busy(sb, br_id, to_free, opened); + if (unlikely(err)) { + AuVerbose(verbose, "%llu file(s) opened\n", opened); + goto out; + } + } + + wbr = br->br_wbr; + do_wh = wbr && (wbr->wbr_whbase || wbr->wbr_plink || wbr->wbr_orph); + if (do_wh) { + /* instead of WbrWhMustWriteLock(wbr) */ + SiMustWriteLock(sb); + for (i = 0; i < AuBrWh_Last; i++) { + dput(wbr->wbr_wh[i]); + wbr->wbr_wh[i] = NULL; + } + } + + err = test_children_busy(root, bindex, verbose); + if (unlikely(err)) { + if (do_wh) + goto out_wh; + goto out; + } + + err = 0; + if (to_free) { + /* + * now we confirmed the branch is deletable. + * let's free the remaining opened dirs on the branch. + */ + di_write_unlock(root); + br_del_file(to_free, opened, br_id); + di_write_lock_child(root); + } + + sysaufs_brs_del(sb, bindex); /* remove successors */ + dbgaufs_xino_del(br); /* remove one */ + au_br_do_del(sb, bindex, br); + sysaufs_brs_add(sb, bindex); /* append successors */ + dbgaufs_brs_add(sb, bindex, /*topdown*/1); /* rename successors */ + + if (!bindex) { + au_cpup_attr_all(d_inode(root), /*force*/1); + sb->s_maxbytes = au_sbr_sb(sb, 0)->s_maxbytes; + } else + au_sub_nlink(d_inode(root), d_inode(del->h_path.dentry)); + if (au_opt_test(mnt_flags, PLINK)) + au_plink_half_refresh(sb, br_id); + + goto out; /* success */ + +out_wh: + /* revert */ + rerr = au_br_init_wh(sb, br, br->br_perm); + if (rerr) + pr_warn("failed re-creating base whiteout, %s. (%d)\n", + del->pathname, rerr); +out: + if (to_free) + au_farray_free(to_free, opened); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_ibusy(struct super_block *sb, struct aufs_ibusy __user *arg) +{ + int err; + aufs_bindex_t btop, bbot; + struct aufs_ibusy ibusy; + struct inode *inode, *h_inode; + + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = copy_from_user(&ibusy, arg, sizeof(ibusy)); + if (!err) + err = !access_ok(VERIFY_WRITE, &arg->h_ino, sizeof(arg->h_ino)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + + err = -EINVAL; + si_read_lock(sb, AuLock_FLUSH); + if (unlikely(ibusy.bindex < 0 || ibusy.bindex > au_sbbot(sb))) + goto out_unlock; + + err = 0; + ibusy.h_ino = 0; /* invalid */ + inode = ilookup(sb, ibusy.ino); + if (!inode + || inode->i_ino == AUFS_ROOT_INO + || au_is_bad_inode(inode)) + goto out_unlock; + + ii_read_lock_child(inode); + btop = au_ibtop(inode); + bbot = au_ibbot(inode); + if (btop <= ibusy.bindex && ibusy.bindex <= bbot) { + h_inode = au_h_iptr(inode, ibusy.bindex); + if (h_inode && au_test_ibusy(inode, btop, bbot)) + ibusy.h_ino = h_inode->i_ino; + } + ii_read_unlock(inode); + iput(inode); + +out_unlock: + si_read_unlock(sb); + if (!err) { + err = __put_user(ibusy.h_ino, &arg->h_ino); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + } + } +out: + return err; +} + +long au_ibusy_ioctl(struct file *file, unsigned long arg) +{ + return au_ibusy(file->f_path.dentry->d_sb, (void __user *)arg); +} + +#ifdef CONFIG_COMPAT +long au_ibusy_compat_ioctl(struct file *file, unsigned long arg) +{ + return au_ibusy(file->f_path.dentry->d_sb, compat_ptr(arg)); +} +#endif + +/* ---------------------------------------------------------------------- */ + +/* + * change a branch permission + */ + +static void au_warn_ima(void) +{ +#ifdef CONFIG_IMA + /* since it doesn't support mark_files_ro() */ + AuWarn1("RW -> RO makes IMA to produce wrong message\n"); +#endif +} + +static int do_need_sigen_inc(int a, int b) +{ + return au_br_whable(a) && !au_br_whable(b); +} + +static int need_sigen_inc(int old, int new) +{ + return do_need_sigen_inc(old, new) + || do_need_sigen_inc(new, old); +} + +static int au_br_mod_files_ro(struct super_block *sb, aufs_bindex_t bindex) +{ + int err, do_warn; + unsigned int mnt_flags; + unsigned long long ull, max; + aufs_bindex_t br_id; + unsigned char verbose, writer; + struct file *file, *hf, **array; + struct au_hfile *hfile; + + mnt_flags = au_mntflags(sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + + array = au_farray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + do_warn = 0; + br_id = au_sbr_id(sb, bindex); + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + fi_read_lock(file); + if (unlikely(au_test_mmapped(file))) { + err = -EBUSY; + AuVerbose(verbose, "mmapped %pD\n", file); + AuDbgFile(file); + FiMustNoWaiters(file); + fi_read_unlock(file); + goto out_array; + } + + hfile = &au_fi(file)->fi_htop; + hf = hfile->hf_file; + if (!d_is_reg(file->f_path.dentry) + || !(file->f_mode & FMODE_WRITE) + || hfile->hf_br->br_id != br_id + || !(hf->f_mode & FMODE_WRITE)) + array[ull] = NULL; + else { + do_warn = 1; + get_file(file); + } + + FiMustNoWaiters(file); + fi_read_unlock(file); + fput(file); + } + + err = 0; + if (do_warn) + au_warn_ima(); + + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (!file) + continue; + + /* todo: already flushed? */ + /* + * fs/super.c:mark_files_ro() is gone, but aufs keeps its + * approach which resets f_mode and calls mnt_drop_write() and + * file_release_write() for each file, because the branch + * attribute in aufs world is totally different from the native + * fs rw/ro mode. + */ + /* fi_read_lock(file); */ + hfile = &au_fi(file)->fi_htop; + hf = hfile->hf_file; + /* fi_read_unlock(file); */ + spin_lock(&hf->f_lock); + writer = !!(hf->f_mode & FMODE_WRITER); + hf->f_mode &= ~(FMODE_WRITE | FMODE_WRITER); + spin_unlock(&hf->f_lock); + if (writer) { + put_write_access(file_inode(hf)); + __mnt_drop_write(hf->f_path.mnt); + } + } + +out_array: + au_farray_free(array, max); +out: + AuTraceErr(err); + return err; +} + +int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, + int *do_refresh) +{ + int err, rerr; + aufs_bindex_t bindex; + struct dentry *root; + struct au_branch *br; + struct au_br_fhsm *bf; + + root = sb->s_root; + bindex = au_find_dbindex(root, mod->h_root); + if (bindex < 0) { + if (remount) + return 0; /* success */ + err = -ENOENT; + pr_err("%s no such branch\n", mod->path); + goto out; + } + AuDbg("bindex b%d\n", bindex); + + err = test_br(d_inode(mod->h_root), mod->perm, mod->path); + if (unlikely(err)) + goto out; + + br = au_sbr(sb, bindex); + AuDebugOn(mod->h_root != au_br_dentry(br)); + if (br->br_perm == mod->perm) + return 0; /* success */ + + /* pre-allocate for non-fhsm --> fhsm */ + bf = NULL; + if (!au_br_fhsm(br->br_perm) && au_br_fhsm(mod->perm)) { + err = au_fhsm_br_alloc(br); + if (unlikely(err)) + goto out; + bf = br->br_fhsm; + br->br_fhsm = NULL; + } + + if (au_br_writable(br->br_perm)) { + /* remove whiteout base */ + err = au_br_init_wh(sb, br, mod->perm); + if (unlikely(err)) + goto out_bf; + + if (!au_br_writable(mod->perm)) { + /* rw --> ro, file might be mmapped */ + DiMustNoWaiters(root); + IiMustNoWaiters(d_inode(root)); + di_write_unlock(root); + err = au_br_mod_files_ro(sb, bindex); + /* aufs_write_lock() calls ..._child() */ + di_write_lock_child(root); + + if (unlikely(err)) { + rerr = -ENOMEM; + br->br_wbr = kzalloc(sizeof(*br->br_wbr), + GFP_NOFS); + if (br->br_wbr) + rerr = au_wbr_init(br, sb, br->br_perm); + if (unlikely(rerr)) { + AuIOErr("nested error %d (%d)\n", + rerr, err); + br->br_perm = mod->perm; + } + } + } + } else if (au_br_writable(mod->perm)) { + /* ro --> rw */ + err = -ENOMEM; + br->br_wbr = kzalloc(sizeof(*br->br_wbr), GFP_NOFS); + if (br->br_wbr) { + err = au_wbr_init(br, sb, mod->perm); + if (unlikely(err)) { + au_kfree_rcu(br->br_wbr); + br->br_wbr = NULL; + } + } + } + if (unlikely(err)) + goto out_bf; + + if (au_br_fhsm(br->br_perm)) { + if (!au_br_fhsm(mod->perm)) { + /* fhsm --> non-fhsm */ + au_br_fhsm_fin(br->br_fhsm); + au_kfree_rcu(br->br_fhsm); + br->br_fhsm = NULL; + } + } else if (au_br_fhsm(mod->perm)) + /* non-fhsm --> fhsm */ + br->br_fhsm = bf; + + *do_refresh |= need_sigen_inc(br->br_perm, mod->perm); + br->br_perm = mod->perm; + goto out; /* success */ + +out_bf: + au_kfree_try_rcu(bf); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs) +{ + int err; + struct kstatfs kstfs; + + err = vfs_statfs(&br->br_path, &kstfs); + if (!err) { + stfs->f_blocks = kstfs.f_blocks; + stfs->f_bavail = kstfs.f_bavail; + stfs->f_files = kstfs.f_files; + stfs->f_ffree = kstfs.f_ffree; + } + + return err; +} diff --git a/fs/aufs/branch.h b/fs/aufs/branch.h new file mode 100644 index 0000000000000..012459ad57f34 --- /dev/null +++ b/fs/aufs/branch.h @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * branch filesystems and xino for them + */ + +#ifndef __AUFS_BRANCH_H__ +#define __AUFS_BRANCH_H__ + +#ifdef __KERNEL__ + +#include +#include "dirren.h" +#include "dynop.h" +#include "lcnt.h" +#include "rwsem.h" +#include "super.h" + +/* ---------------------------------------------------------------------- */ + +/* a xino file */ +struct au_xino { + struct file **xi_file; + unsigned int xi_nfile; + + struct { + spinlock_t spin; + ino_t *array; + int total; + /* reserved for future use */ + /* unsigned long *bitmap; */ + wait_queue_head_t wqh; + } xi_nondir; + + struct mutex xi_mtx; /* protects xi_file array */ + struct hlist_bl_head xi_writing; + + atomic_t xi_truncating; + + struct kref xi_kref; +}; + +/* File-based Hierarchical Storage Management */ +struct au_br_fhsm { +#ifdef CONFIG_AUFS_FHSM + struct mutex bf_lock; + unsigned long bf_jiffy; + struct aufs_stfs bf_stfs; + int bf_readable; +#endif +}; + +/* members for writable branch only */ +enum {AuBrWh_BASE, AuBrWh_PLINK, AuBrWh_ORPH, AuBrWh_Last}; +struct au_wbr { + struct au_rwsem wbr_wh_rwsem; + struct dentry *wbr_wh[AuBrWh_Last]; + atomic_t wbr_wh_running; +#define wbr_whbase wbr_wh[AuBrWh_BASE] /* whiteout base */ +#define wbr_plink wbr_wh[AuBrWh_PLINK] /* pseudo-link dir */ +#define wbr_orph wbr_wh[AuBrWh_ORPH] /* dir for orphans */ + + /* mfs mode */ + unsigned long long wbr_bytes; +}; + +/* ext2 has 3 types of operations at least, ext3 has 4 */ +#define AuBrDynOp (AuDyLast * 4) + +#ifdef CONFIG_AUFS_HFSNOTIFY +/* support for asynchronous destruction */ +struct au_br_hfsnotify { + struct fsnotify_group *hfsn_group; +}; +#endif + +/* sysfs entries */ +struct au_brsysfs { + char name[16]; + struct attribute attr; +}; + +enum { + AuBrSysfs_BR, + AuBrSysfs_BRID, + AuBrSysfs_Last +}; + +/* protected by superblock rwsem */ +struct au_branch { + struct au_xino *br_xino; + + aufs_bindex_t br_id; + + int br_perm; + struct path br_path; + spinlock_t br_dykey_lock; + struct au_dykey *br_dykey[AuBrDynOp]; + au_lcnt_t br_nfiles; /* opened files */ + au_lcnt_t br_count; /* in-use for other */ + + struct au_wbr *br_wbr; + struct au_br_fhsm *br_fhsm; + +#ifdef CONFIG_AUFS_HFSNOTIFY + struct au_br_hfsnotify *br_hfsn; +#endif + +#ifdef CONFIG_SYSFS + /* entries under sysfs per mount-point */ + struct au_brsysfs br_sysfs[AuBrSysfs_Last]; +#endif + +#ifdef CONFIG_DEBUG_FS + struct dentry *br_dbgaufs; /* xino */ +#endif + + struct au_dr_br br_dirren; +}; + +/* ---------------------------------------------------------------------- */ + +static inline struct vfsmount *au_br_mnt(struct au_branch *br) +{ + return br->br_path.mnt; +} + +static inline struct dentry *au_br_dentry(struct au_branch *br) +{ + return br->br_path.dentry; +} + +static inline struct super_block *au_br_sb(struct au_branch *br) +{ + return au_br_mnt(br)->mnt_sb; +} + +static inline int au_br_rdonly(struct au_branch *br) +{ + return (sb_rdonly(au_br_sb(br)) + || !au_br_writable(br->br_perm)) + ? -EROFS : 0; +} + +static inline int au_br_hnotifyable(int brperm __maybe_unused) +{ +#ifdef CONFIG_AUFS_HNOTIFY + return !(brperm & AuBrPerm_RR); +#else + return 0; +#endif +} + +static inline int au_br_test_oflag(int oflag, struct au_branch *br) +{ + int err, exec_flag; + + err = 0; + exec_flag = oflag & __FMODE_EXEC; + if (unlikely(exec_flag && path_noexec(&br->br_path))) + err = -EACCES; + + return err; +} + +static inline void au_xino_get(struct au_branch *br) +{ + struct au_xino *xi; + + xi = br->br_xino; + if (xi) + kref_get(&xi->xi_kref); +} + +static inline int au_xino_count(struct au_branch *br) +{ + int v; + struct au_xino *xi; + + v = 0; + xi = br->br_xino; + if (xi) + v = kref_read(&xi->xi_kref); + + return v; +} + +/* ---------------------------------------------------------------------- */ + +/* branch.c */ +struct au_sbinfo; +void au_br_free(struct au_sbinfo *sinfo); +int au_br_index(struct super_block *sb, aufs_bindex_t br_id); +struct au_opt_add; +int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount); +struct au_opt_del; +int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount); +long au_ibusy_ioctl(struct file *file, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_ibusy_compat_ioctl(struct file *file, unsigned long arg); +#endif +struct au_opt_mod; +int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, + int *do_refresh); +struct aufs_stfs; +int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs); + +/* xino.c */ +static const loff_t au_loff_max = LLONG_MAX; + +aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry); +struct file *au_xino_create(struct super_block *sb, char *fpath, int silent, + int wbrtop); +struct file *au_xino_create2(struct super_block *sb, struct path *base, + struct file *copy_src); +struct au_xi_new { + struct au_xino *xi; /* switch between xino and xigen */ + int idx; + struct path *base; + struct file *copy_src; +}; +struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew); + +int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t *ino); +int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t ino); +ssize_t xino_fread(vfs_readf_t func, struct file *file, void *buf, size_t size, + loff_t *pos); +ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +int au_xib_trunc(struct super_block *sb); +int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin); + +struct au_xino *au_xino_alloc(unsigned int nfile); +int au_xino_put(struct au_branch *br); +struct file *au_xino_file1(struct au_xino *xi); + +struct au_opt_xino; +void au_xino_clr(struct super_block *sb); +int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount); +struct file *au_xino_def(struct super_block *sb); +int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t hino, + struct path *base); + +ino_t au_xino_new_ino(struct super_block *sb); +void au_xino_delete_inode(struct inode *inode, const int unlinked); + +void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, int idx); +int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + int *idx); + +int au_xino_path(struct seq_file *seq, struct file *file); + +/* ---------------------------------------------------------------------- */ + +/* @idx is signed to accept -1 meaning the first file */ +static inline struct file *au_xino_file(struct au_xino *xi, int idx) +{ + struct file *file; + + file = NULL; + if (!xi) + goto out; + + if (idx >= 0) { + if (idx < xi->xi_nfile) + file = xi->xi_file[idx]; + } else + file = au_xino_file1(xi); + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* Superblock to branch */ +static inline +aufs_bindex_t au_sbr_id(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_sbr(sb, bindex)->br_id; +} + +static inline +struct vfsmount *au_sbr_mnt(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_mnt(au_sbr(sb, bindex)); +} + +static inline +struct super_block *au_sbr_sb(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_sb(au_sbr(sb, bindex)); +} + +static inline int au_sbr_perm(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_sbr(sb, bindex)->br_perm; +} + +static inline int au_sbr_whable(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_whable(au_sbr_perm(sb, bindex)); +} + +/* ---------------------------------------------------------------------- */ + +#define wbr_wh_read_lock(wbr) au_rw_read_lock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_lock(wbr) au_rw_write_lock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_read_trylock(wbr) au_rw_read_trylock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_trylock(wbr) au_rw_write_trylock(&(wbr)->wbr_wh_rwsem) +/* +#define wbr_wh_read_trylock_nested(wbr) \ + au_rw_read_trylock_nested(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_trylock_nested(wbr) \ + au_rw_write_trylock_nested(&(wbr)->wbr_wh_rwsem) +*/ + +#define wbr_wh_read_unlock(wbr) au_rw_read_unlock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_unlock(wbr) au_rw_write_unlock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_downgrade_lock(wbr) au_rw_dgrade_lock(&(wbr)->wbr_wh_rwsem) + +#define WbrWhMustNoWaiters(wbr) AuRwMustNoWaiters(&(wbr)->wbr_wh_rwsem) +#define WbrWhMustAnyLock(wbr) AuRwMustAnyLock(&(wbr)->wbr_wh_rwsem) +#define WbrWhMustWriteLock(wbr) AuRwMustWriteLock(&(wbr)->wbr_wh_rwsem) + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_FHSM +static inline void au_br_fhsm_init(struct au_br_fhsm *brfhsm) +{ + mutex_init(&brfhsm->bf_lock); + brfhsm->bf_jiffy = 0; + brfhsm->bf_readable = 0; +} + +static inline void au_br_fhsm_fin(struct au_br_fhsm *brfhsm) +{ + mutex_destroy(&brfhsm->bf_lock); +} +#else +AuStubVoid(au_br_fhsm_init, struct au_br_fhsm *brfhsm) +AuStubVoid(au_br_fhsm_fin, struct au_br_fhsm *brfhsm) +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_BRANCH_H__ */ diff --git a/fs/aufs/conf.mk b/fs/aufs/conf.mk new file mode 100644 index 0000000000000..12782f8e0f381 --- /dev/null +++ b/fs/aufs/conf.mk @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +AuConfStr = CONFIG_AUFS_FS=${CONFIG_AUFS_FS} + +define AuConf +ifdef ${1} +AuConfStr += ${1}=${${1}} +endif +endef + +AuConfAll = BRANCH_MAX_127 BRANCH_MAX_511 BRANCH_MAX_1023 BRANCH_MAX_32767 \ + SBILIST \ + HNOTIFY HFSNOTIFY \ + EXPORT INO_T_64 \ + XATTR \ + FHSM \ + RDU \ + DIRREN \ + SHWH \ + BR_RAMFS \ + BR_FUSE POLL \ + BR_HFSPLUS \ + BDEV_LOOP \ + DEBUG MAGIC_SYSRQ +$(foreach i, ${AuConfAll}, \ + $(eval $(call AuConf,CONFIG_AUFS_${i}))) + +AuConfName = ${obj}/conf.str +${AuConfName}.tmp: FORCE + @echo ${AuConfStr} | tr ' ' '\n' | sed -e 's/^/"/' -e 's/$$/\\n"/' > $@ +${AuConfName}: ${AuConfName}.tmp + @diff -q $< $@ > /dev/null 2>&1 || { \ + echo ' GEN ' $@; \ + cp -p $< $@; \ + } +FORCE: +clean-files += ${AuConfName} ${AuConfName}.tmp +${obj}/sysfs.o: ${AuConfName} + +-include ${srctree}/${src}/conf_priv.mk diff --git a/fs/aufs/cpup.c b/fs/aufs/cpup.c new file mode 100644 index 0000000000000..8fdaf6329e386 --- /dev/null +++ b/fs/aufs/cpup.c @@ -0,0 +1,1449 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * copy-up functions, see wbr_policy.c for copy-down + */ + +#include +#include +#include +#include "aufs.h" + +void au_cpup_attr_flags(struct inode *dst, unsigned int iflags) +{ + const unsigned int mask = S_DEAD | S_SWAPFILE | S_PRIVATE + | S_NOATIME | S_NOCMTIME | S_AUTOMOUNT; + + BUILD_BUG_ON(sizeof(iflags) != sizeof(dst->i_flags)); + + dst->i_flags |= iflags & ~mask; + if (au_test_fs_notime(dst->i_sb)) + dst->i_flags |= S_NOATIME | S_NOCMTIME; +} + +void au_cpup_attr_timesizes(struct inode *inode) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + fsstack_copy_attr_times(inode, h_inode); + fsstack_copy_inode_size(inode, h_inode); +} + +void au_cpup_attr_nlink(struct inode *inode, int force) +{ + struct inode *h_inode; + struct super_block *sb; + aufs_bindex_t bindex, bbot; + + sb = inode->i_sb; + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (!force + && !S_ISDIR(h_inode->i_mode) + && au_opt_test(au_mntflags(sb), PLINK) + && au_plink_test(inode)) + return; + + /* + * 0 can happen in revalidating. + * h_inode->i_mutex may not be held here, but it is harmless since once + * i_nlink reaches 0, it will never become positive except O_TMPFILE + * case. + * todo: O_TMPFILE+linkat(AT_SYMLINK_FOLLOW) bypassing aufs may cause + * the incorrect link count. + */ + set_nlink(inode, h_inode->i_nlink); + + /* + * fewer nlink makes find(1) noisy, but larger nlink doesn't. + * it may includes whplink directory. + */ + if (S_ISDIR(h_inode->i_mode)) { + bbot = au_ibbot(inode); + for (bindex++; bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (h_inode) + au_add_nlink(inode, h_inode); + } + } +} + +void au_cpup_attr_changeable(struct inode *inode) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + inode->i_mode = h_inode->i_mode; + inode->i_uid = h_inode->i_uid; + inode->i_gid = h_inode->i_gid; + au_cpup_attr_timesizes(inode); + au_cpup_attr_flags(inode, h_inode->i_flags); +} + +void au_cpup_igen(struct inode *inode, struct inode *h_inode) +{ + struct au_iinfo *iinfo = au_ii(inode); + + IiMustWriteLock(inode); + + iinfo->ii_higen = h_inode->i_generation; + iinfo->ii_hsb1 = h_inode->i_sb; +} + +void au_cpup_attr_all(struct inode *inode, int force) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + au_cpup_attr_changeable(inode); + if (inode->i_nlink > 0) + au_cpup_attr_nlink(inode, force); + inode->i_rdev = h_inode->i_rdev; + inode->i_blkbits = h_inode->i_blkbits; + au_cpup_igen(inode, h_inode); +} + +/* ---------------------------------------------------------------------- */ + +/* Note: dt_dentry and dt_h_dentry are not dget/dput-ed */ + +/* keep the timestamps of the parent dir when cpup */ +void au_dtime_store(struct au_dtime *dt, struct dentry *dentry, + struct path *h_path) +{ + struct inode *h_inode; + + dt->dt_dentry = dentry; + dt->dt_h_path = *h_path; + h_inode = d_inode(h_path->dentry); + dt->dt_atime = h_inode->i_atime; + dt->dt_mtime = h_inode->i_mtime; + /* smp_mb(); */ +} + +void au_dtime_revert(struct au_dtime *dt) +{ + struct iattr attr; + int err; + + attr.ia_atime = dt->dt_atime; + attr.ia_mtime = dt->dt_mtime; + attr.ia_valid = ATTR_FORCE | ATTR_MTIME | ATTR_MTIME_SET + | ATTR_ATIME | ATTR_ATIME_SET; + + /* no delegation since this is a directory */ + err = vfsub_notify_change(&dt->dt_h_path, &attr, /*delegated*/NULL); + if (unlikely(err)) + pr_warn("restoring timestamps failed(%d). ignored\n", err); +} + +/* ---------------------------------------------------------------------- */ + +/* internal use only */ +struct au_cpup_reg_attr { + int valid; + struct kstat st; + unsigned int iflags; /* inode->i_flags */ +}; + +static noinline_for_stack +int cpup_iattr(struct dentry *dst, aufs_bindex_t bindex, struct dentry *h_src, + struct au_cpup_reg_attr *h_src_attr) +{ + int err, sbits, icex; + unsigned int mnt_flags; + unsigned char verbose; + struct iattr ia; + struct path h_path; + struct inode *h_isrc, *h_idst; + struct kstat *h_st; + struct au_branch *br; + + h_path.dentry = au_h_dptr(dst, bindex); + h_idst = d_inode(h_path.dentry); + br = au_sbr(dst->d_sb, bindex); + h_path.mnt = au_br_mnt(br); + h_isrc = d_inode(h_src); + ia.ia_valid = ATTR_FORCE | ATTR_UID | ATTR_GID + | ATTR_ATIME | ATTR_MTIME + | ATTR_ATIME_SET | ATTR_MTIME_SET; + if (h_src_attr && h_src_attr->valid) { + h_st = &h_src_attr->st; + ia.ia_uid = h_st->uid; + ia.ia_gid = h_st->gid; + ia.ia_atime = h_st->atime; + ia.ia_mtime = h_st->mtime; + if (h_idst->i_mode != h_st->mode + && !S_ISLNK(h_idst->i_mode)) { + ia.ia_valid |= ATTR_MODE; + ia.ia_mode = h_st->mode; + } + sbits = !!(h_st->mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(h_idst, h_src_attr->iflags); + } else { + ia.ia_uid = h_isrc->i_uid; + ia.ia_gid = h_isrc->i_gid; + ia.ia_atime = h_isrc->i_atime; + ia.ia_mtime = h_isrc->i_mtime; + if (h_idst->i_mode != h_isrc->i_mode + && !S_ISLNK(h_idst->i_mode)) { + ia.ia_valid |= ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + } + sbits = !!(h_isrc->i_mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(h_idst, h_isrc->i_flags); + } + /* no delegation since it is just created */ + err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL); + + /* is this nfs only? */ + if (!err && sbits && au_test_nfs(h_path.dentry->d_sb)) { + ia.ia_valid = ATTR_FORCE | ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL); + } + + icex = br->br_perm & AuBrAttr_ICEX; + if (!err) { + mnt_flags = au_mntflags(dst->d_sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + err = au_cpup_xattr(h_path.dentry, h_src, icex, verbose); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_copy_file(struct file *dst, struct file *src, loff_t len, + char *buf, unsigned long blksize) +{ + int err; + size_t sz, rbytes, wbytes; + unsigned char all_zero; + char *p, *zp; + struct inode *h_inode; + /* reduce stack usage */ + struct iattr *ia; + + zp = page_address(ZERO_PAGE(0)); + if (unlikely(!zp)) + return -ENOMEM; /* possible? */ + + err = 0; + all_zero = 0; + while (len) { + AuDbg("len %lld\n", len); + sz = blksize; + if (len < blksize) + sz = len; + + rbytes = 0; + /* todo: signal_pending? */ + while (!rbytes || err == -EAGAIN || err == -EINTR) { + rbytes = vfsub_read_k(src, buf, sz, &src->f_pos); + err = rbytes; + } + if (unlikely(err < 0)) + break; + + all_zero = 0; + if (len >= rbytes && rbytes == blksize) + all_zero = !memcmp(buf, zp, rbytes); + if (!all_zero) { + wbytes = rbytes; + p = buf; + while (wbytes) { + size_t b; + + b = vfsub_write_k(dst, p, wbytes, &dst->f_pos); + err = b; + /* todo: signal_pending? */ + if (unlikely(err == -EAGAIN || err == -EINTR)) + continue; + if (unlikely(err < 0)) + break; + wbytes -= b; + p += b; + } + if (unlikely(err < 0)) + break; + } else { + loff_t res; + + AuLabel(hole); + res = vfsub_llseek(dst, rbytes, SEEK_CUR); + err = res; + if (unlikely(res < 0)) + break; + } + len -= rbytes; + err = 0; + } + + /* the last block may be a hole */ + if (!err && all_zero) { + AuLabel(last hole); + + err = 1; + if (au_test_nfs(dst->f_path.dentry->d_sb)) { + /* nfs requires this step to make last hole */ + /* is this only nfs? */ + do { + /* todo: signal_pending? */ + err = vfsub_write_k(dst, "\0", 1, &dst->f_pos); + } while (err == -EAGAIN || err == -EINTR); + if (err == 1) + dst->f_pos--; + } + + if (err == 1) { + ia = (void *)buf; + ia->ia_size = dst->f_pos; + ia->ia_valid = ATTR_SIZE | ATTR_FILE; + ia->ia_file = dst; + h_inode = file_inode(dst); + inode_lock_nested(h_inode, AuLsc_I_CHILD2); + /* no delegation since it is just created */ + err = vfsub_notify_change(&dst->f_path, ia, + /*delegated*/NULL); + inode_unlock(h_inode); + } + } + + return err; +} + +int au_copy_file(struct file *dst, struct file *src, loff_t len) +{ + int err; + unsigned long blksize; + unsigned char do_kfree; + char *buf; + struct super_block *h_sb; + + err = -ENOMEM; + h_sb = file_inode(dst)->i_sb; + blksize = h_sb->s_blocksize; + if (!blksize || PAGE_SIZE < blksize) + blksize = PAGE_SIZE; + AuDbg("blksize %lu\n", blksize); + do_kfree = (blksize != PAGE_SIZE && blksize >= sizeof(struct iattr *)); + if (do_kfree) + buf = kmalloc(blksize, GFP_NOFS); + else + buf = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!buf)) + goto out; + + if (len > (1 << 22)) + AuDbg("copying a large file %lld\n", (long long)len); + + src->f_pos = 0; + dst->f_pos = 0; + err = au_do_copy_file(dst, src, len, buf, blksize); + if (do_kfree) { + AuDebugOn(!au_kfree_do_sz_test(blksize)); + au_kfree_do_rcu(buf); + } else + free_page((unsigned long)buf); + +out: + return err; +} + +static int au_do_copy(struct file *dst, struct file *src, loff_t len) +{ + int err; + struct super_block *h_src_sb; + struct inode *h_src_inode; + + h_src_inode = file_inode(src); + h_src_sb = h_src_inode->i_sb; + + /* XFS acquires inode_lock */ + if (!au_test_xfs(h_src_sb)) + err = au_copy_file(dst, src, len); + else { + inode_unlock_shared(h_src_inode); + err = au_copy_file(dst, src, len); + vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + } + + return err; +} + +static int au_clone_or_copy(struct file *dst, struct file *src, loff_t len) +{ + int err; + struct super_block *h_src_sb; + struct inode *h_src_inode; + + h_src_inode = file_inode(src); + h_src_sb = h_src_inode->i_sb; + if (h_src_sb != file_inode(dst)->i_sb + || !dst->f_op->clone_file_range) { + err = au_do_copy(dst, src, len); + goto out; + } + + if (!au_test_nfs(h_src_sb)) { + inode_unlock_shared(h_src_inode); + err = vfsub_clone_file_range(src, dst, len); + vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + } else + err = vfsub_clone_file_range(src, dst, len); + /* older XFS has a condition in cloning */ + if (unlikely(err != -EOPNOTSUPP)) + goto out; + + /* the backend fs on NFS may not support cloning */ + err = au_do_copy(dst, src, len); + +out: + AuTraceErr(err); + return err; +} + +/* + * to support a sparse file which is opened with O_APPEND, + * we need to close the file. + */ +static int au_cp_regular(struct au_cp_generic *cpg) +{ + int err, i; + enum { SRC, DST }; + struct { + aufs_bindex_t bindex; + unsigned int flags; + struct dentry *dentry; + int force_wr; + struct file *file; + } *f, file[] = { + { + .bindex = cpg->bsrc, + .flags = O_RDONLY | O_NOATIME | O_LARGEFILE, + }, + { + .bindex = cpg->bdst, + .flags = O_WRONLY | O_NOATIME | O_LARGEFILE, + .force_wr = !!au_ftest_cpup(cpg->flags, RWDST), + } + }; + struct au_branch *br; + struct super_block *sb, *h_src_sb; + struct inode *h_src_inode; + struct task_struct *tsk = current; + + /* bsrc branch can be ro/rw. */ + sb = cpg->dentry->d_sb; + f = file; + for (i = 0; i < 2; i++, f++) { + f->dentry = au_h_dptr(cpg->dentry, f->bindex); + f->file = au_h_open(cpg->dentry, f->bindex, f->flags, + /*file*/NULL, f->force_wr); + if (IS_ERR(f->file)) { + err = PTR_ERR(f->file); + if (i == SRC) + goto out; + else + goto out_src; + } + } + + /* try stopping to update while we copyup */ + h_src_inode = d_inode(file[SRC].dentry); + h_src_sb = h_src_inode->i_sb; + if (!au_test_nfs(h_src_sb)) + IMustLock(h_src_inode); + err = au_clone_or_copy(file[DST].file, file[SRC].file, cpg->len); + + /* i wonder if we had O_NO_DELAY_FPUT flag */ + if (tsk->flags & PF_KTHREAD) + __fput_sync(file[DST].file); + else { + /* it happened actually */ + fput(file[DST].file); + /* + * too bad. + * we have to call both since we don't know which place the file + * was added to. + */ + task_work_run(); + flush_delayed_fput(); + } + br = au_sbr(sb, file[DST].bindex); + au_lcnt_dec(&br->br_nfiles); + +out_src: + fput(file[SRC].file); + br = au_sbr(sb, file[SRC].bindex); + au_lcnt_dec(&br->br_nfiles); +out: + return err; +} + +static int au_do_cpup_regular(struct au_cp_generic *cpg, + struct au_cpup_reg_attr *h_src_attr) +{ + int err, rerr; + loff_t l; + struct path h_path; + struct inode *h_src_inode, *h_dst_inode; + + err = 0; + h_src_inode = au_h_iptr(d_inode(cpg->dentry), cpg->bsrc); + l = i_size_read(h_src_inode); + if (cpg->len == -1 || l < cpg->len) + cpg->len = l; + if (cpg->len) { + /* try stopping to update while we are referencing */ + vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + au_pin_hdir_unlock(cpg->pin); + + h_path.dentry = au_h_dptr(cpg->dentry, cpg->bsrc); + h_path.mnt = au_sbr_mnt(cpg->dentry->d_sb, cpg->bsrc); + h_src_attr->iflags = h_src_inode->i_flags; + if (!au_test_nfs(h_src_inode->i_sb)) + err = vfsub_getattr(&h_path, &h_src_attr->st); + else { + inode_unlock_shared(h_src_inode); + err = vfsub_getattr(&h_path, &h_src_attr->st); + vfsub_inode_lock_shared_nested(h_src_inode, + AuLsc_I_CHILD); + } + if (unlikely(err)) { + inode_unlock_shared(h_src_inode); + goto out; + } + h_src_attr->valid = 1; + if (!au_test_nfs(h_src_inode->i_sb)) { + err = au_cp_regular(cpg); + inode_unlock_shared(h_src_inode); + } else { + inode_unlock_shared(h_src_inode); + err = au_cp_regular(cpg); + } + rerr = au_pin_hdir_relock(cpg->pin); + if (!err && rerr) + err = rerr; + } + if (!err && (h_src_inode->i_state & I_LINKABLE)) { + h_path.dentry = au_h_dptr(cpg->dentry, cpg->bdst); + h_dst_inode = d_inode(h_path.dentry); + spin_lock(&h_dst_inode->i_lock); + h_dst_inode->i_state |= I_LINKABLE; + spin_unlock(&h_dst_inode->i_lock); + } + +out: + return err; +} + +static int au_do_cpup_symlink(struct path *h_path, struct dentry *h_src, + struct inode *h_dir) +{ + int err, symlen; + mm_segment_t old_fs; + union { + char *k; + char __user *u; + } sym; + + err = -ENOMEM; + sym.k = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!sym.k)) + goto out; + + /* unnecessary to support mmap_sem since symlink is not mmap-able */ + old_fs = get_fs(); + set_fs(KERNEL_DS); + symlen = vfs_readlink(h_src, sym.u, PATH_MAX); + err = symlen; + set_fs(old_fs); + + if (symlen > 0) { + sym.k[symlen] = 0; + err = vfsub_symlink(h_dir, h_path, sym.k); + } + free_page((unsigned long)sym.k); + +out: + return err; +} + +/* + * regardless 'acl' option, reset all ACL. + * All ACL will be copied up later from the original entry on the lower branch. + */ +static int au_reset_acl(struct inode *h_dir, struct path *h_path, umode_t mode) +{ + int err; + struct dentry *h_dentry; + struct inode *h_inode; + + h_dentry = h_path->dentry; + h_inode = d_inode(h_dentry); + /* forget_all_cached_acls(h_inode)); */ + err = vfsub_removexattr(h_dentry, XATTR_NAME_POSIX_ACL_ACCESS); + AuTraceErr(err); + if (err == -EOPNOTSUPP) + err = 0; + if (!err) + err = vfsub_acl_chmod(h_inode, mode); + + AuTraceErr(err); + return err; +} + +static int au_do_cpup_dir(struct au_cp_generic *cpg, struct dentry *dst_parent, + struct inode *h_dir, struct path *h_path) +{ + int err; + struct inode *dir, *inode; + + err = vfsub_removexattr(h_path->dentry, XATTR_NAME_POSIX_ACL_DEFAULT); + AuTraceErr(err); + if (err == -EOPNOTSUPP) + err = 0; + if (unlikely(err)) + goto out; + + /* + * strange behaviour from the users view, + * particularly setattr case + */ + dir = d_inode(dst_parent); + if (au_ibtop(dir) == cpg->bdst) + au_cpup_attr_nlink(dir, /*force*/1); + inode = d_inode(cpg->dentry); + au_cpup_attr_nlink(inode, /*force*/1); + +out: + return err; +} + +static noinline_for_stack +int cpup_entry(struct au_cp_generic *cpg, struct dentry *dst_parent, + struct au_cpup_reg_attr *h_src_attr) +{ + int err; + umode_t mode; + unsigned int mnt_flags; + unsigned char isdir, isreg, force; + const unsigned char do_dt = !!au_ftest_cpup(cpg->flags, DTIME); + struct au_dtime dt; + struct path h_path; + struct dentry *h_src, *h_dst, *h_parent; + struct inode *h_inode, *h_dir; + struct super_block *sb; + + /* bsrc branch can be ro/rw. */ + h_src = au_h_dptr(cpg->dentry, cpg->bsrc); + h_inode = d_inode(h_src); + AuDebugOn(h_inode != au_h_iptr(d_inode(cpg->dentry), cpg->bsrc)); + + /* try stopping to be referenced while we are creating */ + h_dst = au_h_dptr(cpg->dentry, cpg->bdst); + if (au_ftest_cpup(cpg->flags, RENAME)) + AuDebugOn(strncmp(h_dst->d_name.name, AUFS_WH_PFX, + AUFS_WH_PFX_LEN)); + h_parent = h_dst->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + AuDebugOn(h_parent != h_dst->d_parent); + + sb = cpg->dentry->d_sb; + h_path.mnt = au_sbr_mnt(sb, cpg->bdst); + if (do_dt) { + h_path.dentry = h_parent; + au_dtime_store(&dt, dst_parent, &h_path); + } + h_path.dentry = h_dst; + + isreg = 0; + isdir = 0; + mode = h_inode->i_mode; + switch (mode & S_IFMT) { + case S_IFREG: + isreg = 1; + err = vfsub_create(h_dir, &h_path, 0600, /*want_excl*/true); + if (!err) + err = au_do_cpup_regular(cpg, h_src_attr); + break; + case S_IFDIR: + isdir = 1; + err = vfsub_mkdir(h_dir, &h_path, mode); + if (!err) + err = au_do_cpup_dir(cpg, dst_parent, h_dir, &h_path); + break; + case S_IFLNK: + err = au_do_cpup_symlink(&h_path, h_src, h_dir); + break; + case S_IFCHR: + case S_IFBLK: + AuDebugOn(!capable(CAP_MKNOD)); + /*FALLTHROUGH*/ + case S_IFIFO: + case S_IFSOCK: + err = vfsub_mknod(h_dir, &h_path, mode, h_inode->i_rdev); + break; + default: + AuIOErr("Unknown inode type 0%o\n", mode); + err = -EIO; + } + if (!err) + err = au_reset_acl(h_dir, &h_path, mode); + + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, UDBA_NONE) + && !isdir + && au_opt_test(mnt_flags, XINO) + && (h_inode->i_nlink == 1 + || (h_inode->i_state & I_LINKABLE)) + /* todo: unnecessary? */ + /* && d_inode(cpg->dentry)->i_nlink == 1 */ + && cpg->bdst < cpg->bsrc + && !au_ftest_cpup(cpg->flags, KEEPLINO)) + au_xino_write(sb, cpg->bsrc, h_inode->i_ino, /*ino*/0); + /* ignore this error */ + + if (!err) { + force = 0; + if (isreg) { + force = !!cpg->len; + if (cpg->len == -1) + force = !!i_size_read(h_inode); + } + au_fhsm_wrote(sb, cpg->bdst, force); + } + + if (do_dt) + au_dtime_revert(&dt); + return err; +} + +static int au_do_ren_after_cpup(struct au_cp_generic *cpg, struct path *h_path) +{ + int err; + struct dentry *dentry, *h_dentry, *h_parent, *parent; + struct inode *h_dir; + aufs_bindex_t bdst; + + dentry = cpg->dentry; + bdst = cpg->bdst; + h_dentry = au_h_dptr(dentry, bdst); + if (!au_ftest_cpup(cpg->flags, OVERWRITE)) { + dget(h_dentry); + au_set_h_dptr(dentry, bdst, NULL); + err = au_lkup_neg(dentry, bdst, /*wh*/0); + if (!err) + h_path->dentry = dget(au_h_dptr(dentry, bdst)); + au_set_h_dptr(dentry, bdst, h_dentry); + } else { + err = 0; + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + dput(parent); + h_path->dentry = vfsub_lkup_one(&dentry->d_name, h_parent); + if (IS_ERR(h_path->dentry)) + err = PTR_ERR(h_path->dentry); + } + if (unlikely(err)) + goto out; + + h_parent = h_dentry->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + AuDbg("%pd %pd\n", h_dentry, h_path->dentry); + /* no delegation since it is just created */ + err = vfsub_rename(h_dir, h_dentry, h_dir, h_path, /*delegated*/NULL, + /*flags*/0); + dput(h_path->dentry); + +out: + return err; +} + +/* + * copyup the @dentry from @bsrc to @bdst. + * the caller must set the both of lower dentries. + * @len is for truncating when it is -1 copyup the entire file. + * in link/rename cases, @dst_parent may be different from the real one. + * basic->bsrc can be larger than basic->bdst. + * aufs doesn't touch the credential so + * security_inode_copy_up{,_xattr}() are unnecessary. + */ +static int au_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent) +{ + int err, rerr; + aufs_bindex_t old_ibtop; + unsigned char isdir, plink; + struct dentry *h_src, *h_dst, *h_parent; + struct inode *dst_inode, *h_dir, *inode, *delegated, *src_inode; + struct super_block *sb; + struct au_branch *br; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct path h_path; + struct au_cpup_reg_attr h_src_attr; + } *a; + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + a->h_src_attr.valid = 0; + + sb = cpg->dentry->d_sb; + br = au_sbr(sb, cpg->bdst); + a->h_path.mnt = au_br_mnt(br); + h_dst = au_h_dptr(cpg->dentry, cpg->bdst); + h_parent = h_dst->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + h_src = au_h_dptr(cpg->dentry, cpg->bsrc); + inode = d_inode(cpg->dentry); + + if (!dst_parent) + dst_parent = dget_parent(cpg->dentry); + else + dget(dst_parent); + + plink = !!au_opt_test(au_mntflags(sb), PLINK); + dst_inode = au_h_iptr(inode, cpg->bdst); + if (dst_inode) { + if (unlikely(!plink)) { + err = -EIO; + AuIOErr("hi%lu(i%lu) exists on b%d " + "but plink is disabled\n", + dst_inode->i_ino, inode->i_ino, cpg->bdst); + goto out_parent; + } + + if (dst_inode->i_nlink) { + const int do_dt = au_ftest_cpup(cpg->flags, DTIME); + + h_src = au_plink_lkup(inode, cpg->bdst); + err = PTR_ERR(h_src); + if (IS_ERR(h_src)) + goto out_parent; + if (unlikely(d_is_negative(h_src))) { + err = -EIO; + AuIOErr("i%lu exists on b%d " + "but not pseudo-linked\n", + inode->i_ino, cpg->bdst); + dput(h_src); + goto out_parent; + } + + if (do_dt) { + a->h_path.dentry = h_parent; + au_dtime_store(&a->dt, dst_parent, &a->h_path); + } + + a->h_path.dentry = h_dst; + delegated = NULL; + err = vfsub_link(h_src, h_dir, &a->h_path, &delegated); + if (!err && au_ftest_cpup(cpg->flags, RENAME)) + err = au_do_ren_after_cpup(cpg, &a->h_path); + if (do_dt) + au_dtime_revert(&a->dt); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + dput(h_src); + goto out_parent; + } else + /* todo: cpup_wh_file? */ + /* udba work */ + au_update_ibrange(inode, /*do_put_zero*/1); + } + + isdir = S_ISDIR(inode->i_mode); + old_ibtop = au_ibtop(inode); + err = cpup_entry(cpg, dst_parent, &a->h_src_attr); + if (unlikely(err)) + goto out_rev; + dst_inode = d_inode(h_dst); + inode_lock_nested(dst_inode, AuLsc_I_CHILD2); + /* todo: necessary? */ + /* au_pin_hdir_unlock(cpg->pin); */ + + err = cpup_iattr(cpg->dentry, cpg->bdst, h_src, &a->h_src_attr); + if (unlikely(err)) { + /* todo: necessary? */ + /* au_pin_hdir_relock(cpg->pin); */ /* ignore an error */ + inode_unlock(dst_inode); + goto out_rev; + } + + if (cpg->bdst < old_ibtop) { + if (S_ISREG(inode->i_mode)) { + err = au_dy_iaop(inode, cpg->bdst, dst_inode); + if (unlikely(err)) { + /* ignore an error */ + /* au_pin_hdir_relock(cpg->pin); */ + inode_unlock(dst_inode); + goto out_rev; + } + } + au_set_ibtop(inode, cpg->bdst); + } else + au_set_ibbot(inode, cpg->bdst); + au_set_h_iptr(inode, cpg->bdst, au_igrab(dst_inode), + au_hi_flags(inode, isdir)); + + /* todo: necessary? */ + /* err = au_pin_hdir_relock(cpg->pin); */ + inode_unlock(dst_inode); + if (unlikely(err)) + goto out_rev; + + src_inode = d_inode(h_src); + if (!isdir + && (src_inode->i_nlink > 1 + || src_inode->i_state & I_LINKABLE) + && plink) + au_plink_append(inode, cpg->bdst, h_dst); + + if (au_ftest_cpup(cpg->flags, RENAME)) { + a->h_path.dentry = h_dst; + err = au_do_ren_after_cpup(cpg, &a->h_path); + } + if (!err) + goto out_parent; /* success */ + + /* revert */ +out_rev: + a->h_path.dentry = h_parent; + au_dtime_store(&a->dt, dst_parent, &a->h_path); + a->h_path.dentry = h_dst; + rerr = 0; + if (d_is_positive(h_dst)) { + if (!isdir) { + /* no delegation since it is just created */ + rerr = vfsub_unlink(h_dir, &a->h_path, + /*delegated*/NULL, /*force*/0); + } else + rerr = vfsub_rmdir(h_dir, &a->h_path); + } + au_dtime_revert(&a->dt); + if (rerr) { + AuIOErr("failed removing broken entry(%d, %d)\n", err, rerr); + err = -EIO; + } +out_parent: + dput(dst_parent); + au_kfree_rcu(a); +out: + return err; +} + +#if 0 /* reserved */ +struct au_cpup_single_args { + int *errp; + struct au_cp_generic *cpg; + struct dentry *dst_parent; +}; + +static void au_call_cpup_single(void *args) +{ + struct au_cpup_single_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_single(a->cpg, a->dst_parent); + au_pin_hdir_release(a->cpg->pin); +} +#endif + +/* + * prevent SIGXFSZ in copy-up. + * testing CAP_MKNOD is for generic fs, + * but CAP_FSETID is for xfs only, currently. + */ +static int au_cpup_sio_test(struct au_pin *pin, umode_t mode) +{ + int do_sio; + struct super_block *sb; + struct inode *h_dir; + + do_sio = 0; + sb = au_pinned_parent(pin)->d_sb; + if (!au_wkq_test() + && (!au_sbi(sb)->si_plink_maint_pid + || au_plink_maint(sb, AuLock_NOPLM))) { + switch (mode & S_IFMT) { + case S_IFREG: + /* no condition about RLIMIT_FSIZE and the file size */ + do_sio = 1; + break; + case S_IFCHR: + case S_IFBLK: + do_sio = !capable(CAP_MKNOD); + break; + } + if (!do_sio) + do_sio = ((mode & (S_ISUID | S_ISGID)) + && !capable(CAP_FSETID)); + /* this workaround may be removed in the future */ + if (!do_sio) { + h_dir = au_pinned_h_dir(pin); + do_sio = h_dir->i_mode & S_ISVTX; + } + } + + return do_sio; +} + +#if 0 /* reserved */ +int au_sio_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent) +{ + int err, wkq_err; + struct dentry *h_dentry; + + h_dentry = au_h_dptr(cpg->dentry, cpg->bsrc); + if (!au_cpup_sio_test(pin, d_inode(h_dentry)->i_mode)) + err = au_cpup_single(cpg, dst_parent); + else { + struct au_cpup_single_args args = { + .errp = &err, + .cpg = cpg, + .dst_parent = dst_parent + }; + wkq_err = au_wkq_wait(au_call_cpup_single, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} +#endif + +/* + * copyup the @dentry from the first active lower branch to @bdst, + * using au_cpup_single(). + */ +static int au_cpup_simple(struct au_cp_generic *cpg) +{ + int err; + unsigned int flags_orig; + struct dentry *dentry; + + AuDebugOn(cpg->bsrc < 0); + + dentry = cpg->dentry; + DiMustWriteLock(dentry); + + err = au_lkup_neg(dentry, cpg->bdst, /*wh*/1); + if (!err) { + flags_orig = cpg->flags; + au_fset_cpup(cpg->flags, RENAME); + err = au_cpup_single(cpg, NULL); + cpg->flags = flags_orig; + if (!err) + return 0; /* success */ + + /* revert */ + au_set_h_dptr(dentry, cpg->bdst, NULL); + au_set_dbtop(dentry, cpg->bsrc); + } + + return err; +} + +struct au_cpup_simple_args { + int *errp; + struct au_cp_generic *cpg; +}; + +static void au_call_cpup_simple(void *args) +{ + struct au_cpup_simple_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_simple(a->cpg); + au_pin_hdir_release(a->cpg->pin); +} + +static int au_do_sio_cpup_simple(struct au_cp_generic *cpg) +{ + int err, wkq_err; + struct dentry *dentry, *parent; + struct file *h_file; + struct inode *h_dir; + + dentry = cpg->dentry; + h_file = NULL; + if (au_ftest_cpup(cpg->flags, HOPEN)) { + AuDebugOn(cpg->bsrc < 0); + h_file = au_h_open_pre(dentry, cpg->bsrc, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + } + + parent = dget_parent(dentry); + h_dir = au_h_iptr(d_inode(parent), cpg->bdst); + if (!au_test_h_perm_sio(h_dir, MAY_EXEC | MAY_WRITE) + && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode)) + err = au_cpup_simple(cpg); + else { + struct au_cpup_simple_args args = { + .errp = &err, + .cpg = cpg + }; + wkq_err = au_wkq_wait(au_call_cpup_simple, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + dput(parent); + if (h_file) + au_h_open_post(dentry, cpg->bsrc, h_file); + +out: + return err; +} + +int au_sio_cpup_simple(struct au_cp_generic *cpg) +{ + aufs_bindex_t bsrc, bbot; + struct dentry *dentry, *h_dentry; + + if (cpg->bsrc < 0) { + dentry = cpg->dentry; + bbot = au_dbbot(dentry); + for (bsrc = cpg->bdst + 1; bsrc <= bbot; bsrc++) { + h_dentry = au_h_dptr(dentry, bsrc); + if (h_dentry) { + AuDebugOn(d_is_negative(h_dentry)); + break; + } + } + AuDebugOn(bsrc > bbot); + cpg->bsrc = bsrc; + } + AuDebugOn(cpg->bsrc <= cpg->bdst); + return au_do_sio_cpup_simple(cpg); +} + +int au_sio_cpdown_simple(struct au_cp_generic *cpg) +{ + AuDebugOn(cpg->bdst <= cpg->bsrc); + return au_do_sio_cpup_simple(cpg); +} + +/* ---------------------------------------------------------------------- */ + +/* + * copyup the deleted file for writing. + */ +static int au_do_cpup_wh(struct au_cp_generic *cpg, struct dentry *wh_dentry, + struct file *file) +{ + int err; + unsigned int flags_orig; + aufs_bindex_t bsrc_orig; + struct au_dinfo *dinfo; + struct { + struct au_hdentry *hd; + struct dentry *h_dentry; + } hdst, hsrc; + + dinfo = au_di(cpg->dentry); + AuRwMustWriteLock(&dinfo->di_rwsem); + + bsrc_orig = cpg->bsrc; + cpg->bsrc = dinfo->di_btop; + hdst.hd = au_hdentry(dinfo, cpg->bdst); + hdst.h_dentry = hdst.hd->hd_dentry; + hdst.hd->hd_dentry = wh_dentry; + dinfo->di_btop = cpg->bdst; + + hsrc.h_dentry = NULL; + if (file) { + hsrc.hd = au_hdentry(dinfo, cpg->bsrc); + hsrc.h_dentry = hsrc.hd->hd_dentry; + hsrc.hd->hd_dentry = au_hf_top(file)->f_path.dentry; + } + flags_orig = cpg->flags; + cpg->flags = !AuCpup_DTIME; + err = au_cpup_single(cpg, /*h_parent*/NULL); + cpg->flags = flags_orig; + if (file) { + if (!err) + err = au_reopen_nondir(file); + hsrc.hd->hd_dentry = hsrc.h_dentry; + } + hdst.hd->hd_dentry = hdst.h_dentry; + dinfo->di_btop = cpg->bsrc; + cpg->bsrc = bsrc_orig; + + return err; +} + +static int au_cpup_wh(struct au_cp_generic *cpg, struct file *file) +{ + int err; + aufs_bindex_t bdst; + struct au_dtime dt; + struct dentry *dentry, *parent, *h_parent, *wh_dentry; + struct au_branch *br; + struct path h_path; + + dentry = cpg->dentry; + bdst = cpg->bdst; + br = au_sbr(dentry->d_sb, bdst); + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + wh_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out; + + h_path.dentry = h_parent; + h_path.mnt = au_br_mnt(br); + au_dtime_store(&dt, parent, &h_path); + err = au_do_cpup_wh(cpg, wh_dentry, file); + if (unlikely(err)) + goto out_wh; + + dget(wh_dentry); + h_path.dentry = wh_dentry; + if (!d_is_dir(wh_dentry)) { + /* no delegation since it is just created */ + err = vfsub_unlink(d_inode(h_parent), &h_path, + /*delegated*/NULL, /*force*/0); + } else + err = vfsub_rmdir(d_inode(h_parent), &h_path); + if (unlikely(err)) { + AuIOErr("failed remove copied-up tmp file %pd(%d)\n", + wh_dentry, err); + err = -EIO; + } + au_dtime_revert(&dt); + au_set_hi_wh(d_inode(dentry), bdst, wh_dentry); + +out_wh: + dput(wh_dentry); +out: + dput(parent); + return err; +} + +struct au_cpup_wh_args { + int *errp; + struct au_cp_generic *cpg; + struct file *file; +}; + +static void au_call_cpup_wh(void *args) +{ + struct au_cpup_wh_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_wh(a->cpg, a->file); + au_pin_hdir_release(a->cpg->pin); +} + +int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file) +{ + int err, wkq_err; + aufs_bindex_t bdst; + struct dentry *dentry, *parent, *h_orph, *h_parent; + struct inode *dir, *h_dir, *h_tmpdir; + struct au_wbr *wbr; + struct au_pin wh_pin, *pin_orig; + + dentry = cpg->dentry; + bdst = cpg->bdst; + parent = dget_parent(dentry); + dir = d_inode(parent); + h_orph = NULL; + h_parent = NULL; + h_dir = au_igrab(au_h_iptr(dir, bdst)); + h_tmpdir = h_dir; + pin_orig = NULL; + if (!h_dir->i_nlink) { + wbr = au_sbr(dentry->d_sb, bdst)->br_wbr; + h_orph = wbr->wbr_orph; + + h_parent = dget(au_h_dptr(parent, bdst)); + au_set_h_dptr(parent, bdst, dget(h_orph)); + h_tmpdir = d_inode(h_orph); + au_set_h_iptr(dir, bdst, au_igrab(h_tmpdir), /*flags*/0); + + inode_lock_nested(h_tmpdir, AuLsc_I_PARENT3); + /* todo: au_h_open_pre()? */ + + pin_orig = cpg->pin; + au_pin_init(&wh_pin, dentry, bdst, AuLsc_DI_PARENT, + AuLsc_I_PARENT3, cpg->pin->udba, AuPin_DI_LOCKED); + cpg->pin = &wh_pin; + } + + if (!au_test_h_perm_sio(h_tmpdir, MAY_EXEC | MAY_WRITE) + && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode)) + err = au_cpup_wh(cpg, file); + else { + struct au_cpup_wh_args args = { + .errp = &err, + .cpg = cpg, + .file = file + }; + wkq_err = au_wkq_wait(au_call_cpup_wh, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + if (h_orph) { + inode_unlock(h_tmpdir); + /* todo: au_h_open_post()? */ + au_set_h_iptr(dir, bdst, au_igrab(h_dir), /*flags*/0); + au_set_h_dptr(parent, bdst, h_parent); + AuDebugOn(!pin_orig); + cpg->pin = pin_orig; + } + iput(h_dir); + dput(parent); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * generic routine for both of copy-up and copy-down. + */ +/* cf. revalidate function in file.c */ +int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst, + int (*cp)(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg), + void *arg) +{ + int err; + struct au_pin pin; + struct dentry *d, *parent, *h_parent, *real_parent, *h_dentry; + + err = 0; + parent = dget_parent(dentry); + if (IS_ROOT(parent)) + goto out; + + au_pin_init(&pin, dentry, bdst, AuLsc_DI_PARENT2, AuLsc_I_PARENT2, + au_opt_udba(dentry->d_sb), AuPin_MNT_WRITE); + + /* do not use au_dpage */ + real_parent = parent; + while (1) { + dput(parent); + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + if (h_parent) + goto out; /* success */ + + /* find top dir which is necessary to cpup */ + do { + d = parent; + dput(parent); + parent = dget_parent(d); + di_read_lock_parent3(parent, !AuLock_IR); + h_parent = au_h_dptr(parent, bdst); + di_read_unlock(parent, !AuLock_IR); + } while (!h_parent); + + if (d != real_parent) + di_write_lock_child3(d); + + /* somebody else might create while we were sleeping */ + h_dentry = au_h_dptr(d, bdst); + if (!h_dentry || d_is_negative(h_dentry)) { + if (h_dentry) + au_update_dbtop(d); + + au_pin_set_dentry(&pin, d); + err = au_do_pin(&pin); + if (!err) { + err = cp(d, bdst, &pin, h_parent, arg); + au_unpin(&pin); + } + } + + if (d != real_parent) + di_write_unlock(d); + if (unlikely(err)) + break; + } + +out: + dput(parent); + return err; +} + +static int au_cpup_dir(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent __maybe_unused, + void *arg __maybe_unused) +{ + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = bdst, + .bsrc = -1, + .len = 0, + .pin = pin, + .flags = AuCpup_DTIME + }; + return au_sio_cpup_simple(&cpg); +} + +int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + return au_cp_dirs(dentry, bdst, au_cpup_dir, NULL); +} + +int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + int err; + struct dentry *parent; + struct inode *dir; + + parent = dget_parent(dentry); + dir = d_inode(parent); + err = 0; + if (au_h_iptr(dir, bdst)) + goto out; + + di_read_unlock(parent, AuLock_IR); + di_write_lock_parent(parent); + /* someone else might change our inode while we were sleeping */ + if (!au_h_iptr(dir, bdst)) + err = au_cpup_dirs(dentry, bdst); + di_downgrade_lock(parent, AuLock_IR); + +out: + dput(parent); + return err; +} diff --git a/fs/aufs/cpup.h b/fs/aufs/cpup.h new file mode 100644 index 0000000000000..c932e6b603e6f --- /dev/null +++ b/fs/aufs/cpup.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * copy-up/down functions + */ + +#ifndef __AUFS_CPUP_H__ +#define __AUFS_CPUP_H__ + +#ifdef __KERNEL__ + +#include + +struct inode; +struct file; +struct au_pin; + +void au_cpup_attr_flags(struct inode *dst, unsigned int iflags); +void au_cpup_attr_timesizes(struct inode *inode); +void au_cpup_attr_nlink(struct inode *inode, int force); +void au_cpup_attr_changeable(struct inode *inode); +void au_cpup_igen(struct inode *inode, struct inode *h_inode); +void au_cpup_attr_all(struct inode *inode, int force); + +/* ---------------------------------------------------------------------- */ + +struct au_cp_generic { + struct dentry *dentry; + aufs_bindex_t bdst, bsrc; + loff_t len; + struct au_pin *pin; + unsigned int flags; +}; + +/* cpup flags */ +#define AuCpup_DTIME 1 /* do dtime_store/revert */ +#define AuCpup_KEEPLINO (1 << 1) /* do not clear the lower xino, + for link(2) */ +#define AuCpup_RENAME (1 << 2) /* rename after cpup */ +#define AuCpup_HOPEN (1 << 3) /* call h_open_pre/post() in + cpup */ +#define AuCpup_OVERWRITE (1 << 4) /* allow overwriting the + existing entry */ +#define AuCpup_RWDST (1 << 5) /* force write target even if + the branch is marked as RO */ + +#ifndef CONFIG_AUFS_BR_HFSPLUS +#undef AuCpup_HOPEN +#define AuCpup_HOPEN 0 +#endif + +#define au_ftest_cpup(flags, name) ((flags) & AuCpup_##name) +#define au_fset_cpup(flags, name) \ + do { (flags) |= AuCpup_##name; } while (0) +#define au_fclr_cpup(flags, name) \ + do { (flags) &= ~AuCpup_##name; } while (0) + +int au_copy_file(struct file *dst, struct file *src, loff_t len); +int au_sio_cpup_simple(struct au_cp_generic *cpg); +int au_sio_cpdown_simple(struct au_cp_generic *cpg); +int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file); + +int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst, + int (*cp)(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg), + void *arg); +int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst); +int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst); + +/* ---------------------------------------------------------------------- */ + +/* keep timestamps when copyup */ +struct au_dtime { + struct dentry *dt_dentry; + struct path dt_h_path; + struct timespec dt_atime, dt_mtime; +}; +void au_dtime_store(struct au_dtime *dt, struct dentry *dentry, + struct path *h_path); +void au_dtime_revert(struct au_dtime *dt); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_CPUP_H__ */ diff --git a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c new file mode 100644 index 0000000000000..e32d91d1f6381 --- /dev/null +++ b/fs/aufs/dbgaufs.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debugfs interface + */ + +#include +#include "aufs.h" + +#ifndef CONFIG_SYSFS +#error DEBUG_FS depends upon SYSFS +#endif + +static struct dentry *dbgaufs; +static const mode_t dbgaufs_mode = 0444; + +/* 20 is max digits length of ulong 64 */ +struct dbgaufs_arg { + int n; + char a[20 * 4]; +}; + +/* + * common function for all XINO files + */ +static int dbgaufs_xi_release(struct inode *inode __maybe_unused, + struct file *file) +{ + void *p; + + p = file->private_data; + if (p) { + /* this is struct dbgaufs_arg */ + AuDebugOn(!au_kfree_sz_test(p)); + au_kfree_do_rcu(p); + } + return 0; +} + +static int dbgaufs_xi_open(struct file *xf, struct file *file, int do_fcnt, + int cnt) +{ + int err; + struct kstat st; + struct dbgaufs_arg *p; + + err = -ENOMEM; + p = kmalloc(sizeof(*p), GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = 0; + p->n = 0; + file->private_data = p; + if (!xf) + goto out; + + err = vfsub_getattr(&xf->f_path, &st); + if (!err) { + if (do_fcnt) + p->n = snprintf + (p->a, sizeof(p->a), "%d, %llux%u %lld\n", + cnt, st.blocks, st.blksize, + (long long)st.size); + else + p->n = snprintf(p->a, sizeof(p->a), "%llux%u %lld\n", + st.blocks, st.blksize, + (long long)st.size); + AuDebugOn(p->n >= sizeof(p->a)); + } else { + p->n = snprintf(p->a, sizeof(p->a), "err %d\n", err); + err = 0; + } + +out: + return err; +} + +static ssize_t dbgaufs_xi_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dbgaufs_arg *p; + + p = file->private_data; + return simple_read_from_buffer(buf, count, ppos, p->a, p->n); +} + +/* ---------------------------------------------------------------------- */ + +struct dbgaufs_plink_arg { + int n; + char a[]; +}; + +static int dbgaufs_plink_release(struct inode *inode __maybe_unused, + struct file *file) +{ + free_page((unsigned long)file->private_data); + return 0; +} + +static int dbgaufs_plink_open(struct inode *inode, struct file *file) +{ + int err, i, limit; + unsigned long n, sum; + struct dbgaufs_plink_arg *p; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct hlist_bl_head *hbl; + + err = -ENOMEM; + p = (void *)get_zeroed_page(GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = -EFBIG; + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + if (au_opt_test(au_mntflags(sb), PLINK)) { + limit = PAGE_SIZE - sizeof(p->n); + + /* the number of buckets */ + n = snprintf(p->a + p->n, limit, "%d\n", AuPlink_NHASH); + p->n += n; + limit -= n; + + sum = 0; + for (i = 0, hbl = sbinfo->si_plink; i < AuPlink_NHASH; + i++, hbl++) { + n = au_hbl_count(hbl); + sum += n; + + n = snprintf(p->a + p->n, limit, "%lu ", n); + p->n += n; + limit -= n; + if (unlikely(limit <= 0)) + goto out_free; + } + p->a[p->n - 1] = '\n'; + + /* the sum of plinks */ + n = snprintf(p->a + p->n, limit, "%lu\n", sum); + p->n += n; + limit -= n; + if (unlikely(limit <= 0)) + goto out_free; + } else { +#define str "1\n0\n0\n" + p->n = sizeof(str) - 1; + strcpy(p->a, str); +#undef str + } + si_read_unlock(sb); + + err = 0; + file->private_data = p; + goto out; /* success */ + +out_free: + free_page((unsigned long)p); +out: + return err; +} + +static ssize_t dbgaufs_plink_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dbgaufs_plink_arg *p; + + p = file->private_data; + return simple_read_from_buffer(buf, count, ppos, p->a, p->n); +} + +static const struct file_operations dbgaufs_plink_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_plink_open, + .release = dbgaufs_plink_release, + .read = dbgaufs_plink_read +}; + +/* ---------------------------------------------------------------------- */ + +static int dbgaufs_xib_open(struct inode *inode, struct file *file) +{ + int err; + struct au_sbinfo *sbinfo; + struct super_block *sb; + + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + err = dbgaufs_xi_open(sbinfo->si_xib, file, /*do_fcnt*/0, /*cnt*/0); + si_read_unlock(sb); + return err; +} + +static const struct file_operations dbgaufs_xib_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xib_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +/* ---------------------------------------------------------------------- */ + +#define DbgaufsXi_PREFIX "xi" + +static int dbgaufs_xino_open(struct inode *inode, struct file *file) +{ + int err, idx; + long l; + aufs_bindex_t bindex; + char *p, a[sizeof(DbgaufsXi_PREFIX) + 8]; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct au_xino *xi; + struct file *xf; + struct qstr *name; + struct au_branch *br; + + err = -ENOENT; + name = &file->f_path.dentry->d_name; + if (unlikely(name->len < sizeof(DbgaufsXi_PREFIX) + || memcmp(name->name, DbgaufsXi_PREFIX, + sizeof(DbgaufsXi_PREFIX) - 1))) + goto out; + + AuDebugOn(name->len >= sizeof(a)); + memcpy(a, name->name, name->len); + a[name->len] = '\0'; + p = strchr(a, '-'); + if (p) + *p = '\0'; + err = kstrtol(a + sizeof(DbgaufsXi_PREFIX) - 1, 10, &l); + if (unlikely(err)) + goto out; + bindex = l; + idx = 0; + if (p) { + err = kstrtol(p + 1, 10, &l); + if (unlikely(err)) + goto out; + idx = l; + } + + err = -ENOENT; + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + if (unlikely(bindex < 0 || bindex > au_sbbot(sb))) + goto out_si; + br = au_sbr(sb, bindex); + xi = br->br_xino; + if (unlikely(idx >= xi->xi_nfile)) + goto out_si; + xf = au_xino_file(xi, idx); + if (xf) + err = dbgaufs_xi_open(xf, file, /*do_fcnt*/1, + au_xino_count(br)); + +out_si: + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +static const struct file_operations dbgaufs_xino_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xino_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +void dbgaufs_xino_del(struct au_branch *br) +{ + struct dentry *dbgaufs; + + dbgaufs = br->br_dbgaufs; + if (!dbgaufs) + return; + + br->br_dbgaufs = NULL; + /* debugfs acquires the parent i_mutex */ + lockdep_off(); + debugfs_remove(dbgaufs); + lockdep_on(); +} + +void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex) +{ + aufs_bindex_t bbot; + struct au_branch *br; + + if (!au_sbi(sb)->si_dbgaufs) + return; + + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + dbgaufs_xino_del(br); + } +} + +static void dbgaufs_br_do_add(struct super_block *sb, aufs_bindex_t bindex, + unsigned int idx, struct dentry *parent, + struct au_sbinfo *sbinfo) +{ + struct au_branch *br; + struct dentry *d; + /* "xi" bindex(5) "-" idx(2) NULL */ + char name[sizeof(DbgaufsXi_PREFIX) + 8]; + + if (!idx) + snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d", bindex); + else + snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d-%u", + bindex, idx); + br = au_sbr(sb, bindex); + if (br->br_dbgaufs) { + struct qstr qstr = QSTR_INIT(name, strlen(name)); + + if (!au_qstreq(&br->br_dbgaufs->d_name, &qstr)) { + /* debugfs acquires the parent i_mutex */ + lockdep_off(); + d = debugfs_rename(parent, br->br_dbgaufs, parent, + name); + lockdep_on(); + if (unlikely(!d)) + pr_warn("failed renaming %pd/%s, ignored.\n", + parent, name); + } + } else { + lockdep_off(); + br->br_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent, + sbinfo, &dbgaufs_xino_fop); + lockdep_on(); + if (unlikely(!br->br_dbgaufs)) + pr_warn("failed creating %pd/%s, ignored.\n", + parent, name); + } +} + +static void dbgaufs_br_add(struct super_block *sb, aufs_bindex_t bindex, + struct dentry *parent, struct au_sbinfo *sbinfo) +{ + struct au_branch *br; + struct au_xino *xi; + unsigned int u; + + br = au_sbr(sb, bindex); + xi = br->br_xino; + for (u = 0; u < xi->xi_nfile; u++) + dbgaufs_br_do_add(sb, bindex, u, parent, sbinfo); +} + +void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown) +{ + struct au_sbinfo *sbinfo; + struct dentry *parent; + aufs_bindex_t bbot; + + if (!au_opt_test(au_mntflags(sb), XINO)) + return; + + sbinfo = au_sbi(sb); + parent = sbinfo->si_dbgaufs; + if (!parent) + return; + + bbot = au_sbbot(sb); + if (topdown) + for (; bindex <= bbot; bindex++) + dbgaufs_br_add(sb, bindex, parent, sbinfo); + else + for (; bbot >= bindex; bbot--) + dbgaufs_br_add(sb, bbot, parent, sbinfo); +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_EXPORT +static int dbgaufs_xigen_open(struct inode *inode, struct file *file) +{ + int err; + struct au_sbinfo *sbinfo; + struct super_block *sb; + + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + err = dbgaufs_xi_open(sbinfo->si_xigen, file, /*do_fcnt*/0, /*cnt*/0); + si_read_unlock(sb); + return err; +} + +static const struct file_operations dbgaufs_xigen_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xigen_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo) +{ + int err; + + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + err = -EIO; + sbinfo->si_dbgaufs_xigen = debugfs_create_file + ("xigen", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_xigen_fop); + if (sbinfo->si_dbgaufs_xigen) + err = 0; + + return err; +} +#else +static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo) +{ + return 0; +} +#endif /* CONFIG_AUFS_EXPORT */ + +/* ---------------------------------------------------------------------- */ + +void dbgaufs_si_fin(struct au_sbinfo *sbinfo) +{ + /* + * This function is a dynamic '__fin' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + debugfs_remove_recursive(sbinfo->si_dbgaufs); + sbinfo->si_dbgaufs = NULL; +} + +int dbgaufs_si_init(struct au_sbinfo *sbinfo) +{ + int err; + char name[SysaufsSiNameLen]; + + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + err = -ENOENT; + if (!dbgaufs) { + AuErr1("/debug/aufs is uninitialized\n"); + goto out; + } + + err = -EIO; + sysaufs_name(sbinfo, name); + sbinfo->si_dbgaufs = debugfs_create_dir(name, dbgaufs); + if (unlikely(!sbinfo->si_dbgaufs)) + goto out; + + /* regardless plink/noplink option */ + sbinfo->si_dbgaufs_plink = debugfs_create_file + ("plink", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_plink_fop); + if (unlikely(!sbinfo->si_dbgaufs_plink)) + goto out_dir; + + /* regardless xino/noxino option */ + sbinfo->si_dbgaufs_xib = debugfs_create_file + ("xib", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_xib_fop); + if (unlikely(!sbinfo->si_dbgaufs_xib)) + goto out_dir; + + err = dbgaufs_xigen_init(sbinfo); + if (!err) + goto out; /* success */ + +out_dir: + dbgaufs_si_fin(sbinfo); +out: + if (unlikely(err)) + pr_err("debugfs/aufs failed\n"); + return err; +} + +/* ---------------------------------------------------------------------- */ + +void dbgaufs_fin(void) +{ + debugfs_remove(dbgaufs); +} + +int __init dbgaufs_init(void) +{ + int err; + + err = -EIO; + dbgaufs = debugfs_create_dir(AUFS_NAME, NULL); + if (dbgaufs) + err = 0; + return err; +} diff --git a/fs/aufs/dbgaufs.h b/fs/aufs/dbgaufs.h new file mode 100644 index 0000000000000..1ea241bf02342 --- /dev/null +++ b/fs/aufs/dbgaufs.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debugfs interface + */ + +#ifndef __DBGAUFS_H__ +#define __DBGAUFS_H__ + +#ifdef __KERNEL__ + +struct super_block; +struct au_sbinfo; +struct au_branch; + +#ifdef CONFIG_DEBUG_FS +/* dbgaufs.c */ +void dbgaufs_xino_del(struct au_branch *br); +void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex); +void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown); +void dbgaufs_si_fin(struct au_sbinfo *sbinfo); +int dbgaufs_si_init(struct au_sbinfo *sbinfo); +void dbgaufs_fin(void); +int __init dbgaufs_init(void); +#else +AuStubVoid(dbgaufs_xino_del, struct au_branch *br) +AuStubVoid(dbgaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(dbgaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex, + int topdown) +AuStubVoid(dbgaufs_si_fin, struct au_sbinfo *sbinfo) +AuStubInt0(dbgaufs_si_init, struct au_sbinfo *sbinfo) +AuStubVoid(dbgaufs_fin, void) +AuStubInt0(__init dbgaufs_init, void) +#endif /* CONFIG_DEBUG_FS */ + +#endif /* __KERNEL__ */ +#endif /* __DBGAUFS_H__ */ diff --git a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c new file mode 100644 index 0000000000000..ec5e2bbac50f5 --- /dev/null +++ b/fs/aufs/dcsub.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for dentry cache + */ + +#include "aufs.h" + +static void au_dpage_free(struct au_dpage *dpage) +{ + int i; + struct dentry **p; + + p = dpage->dentries; + for (i = 0; i < dpage->ndentry; i++) + dput(*p++); + free_page((unsigned long)dpage->dentries); +} + +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) +{ + int err; + void *p; + + err = -ENOMEM; + dpages->dpages = kmalloc(sizeof(*dpages->dpages), gfp); + if (unlikely(!dpages->dpages)) + goto out; + + p = (void *)__get_free_page(gfp); + if (unlikely(!p)) + goto out_dpages; + + dpages->dpages[0].ndentry = 0; + dpages->dpages[0].dentries = p; + dpages->ndpage = 1; + return 0; /* success */ + +out_dpages: + au_kfree_try_rcu(dpages->dpages); +out: + return err; +} + +void au_dpages_free(struct au_dcsub_pages *dpages) +{ + int i; + struct au_dpage *p; + + p = dpages->dpages; + for (i = 0; i < dpages->ndpage; i++) + au_dpage_free(p++); + au_kfree_try_rcu(dpages->dpages); +} + +static int au_dpages_append(struct au_dcsub_pages *dpages, + struct dentry *dentry, gfp_t gfp) +{ + int err, sz; + struct au_dpage *dpage; + void *p; + + dpage = dpages->dpages + dpages->ndpage - 1; + sz = PAGE_SIZE / sizeof(dentry); + if (unlikely(dpage->ndentry >= sz)) { + AuLabel(new dpage); + err = -ENOMEM; + sz = dpages->ndpage * sizeof(*dpages->dpages); + p = au_kzrealloc(dpages->dpages, sz, + sz + sizeof(*dpages->dpages), gfp, + /*may_shrink*/0); + if (unlikely(!p)) + goto out; + + dpages->dpages = p; + dpage = dpages->dpages + dpages->ndpage; + p = (void *)__get_free_page(gfp); + if (unlikely(!p)) + goto out; + + dpage->ndentry = 0; + dpage->dentries = p; + dpages->ndpage++; + } + + AuDebugOn(au_dcount(dentry) <= 0); + dpage->dentries[dpage->ndentry++] = dget_dlock(dentry); + return 0; /* success */ + +out: + return err; +} + +/* todo: BAD approach */ +/* copied from linux/fs/dcache.c */ +enum d_walk_ret { + D_WALK_CONTINUE, + D_WALK_QUIT, + D_WALK_NORETRY, + D_WALK_SKIP, +}; + +extern void d_walk(struct dentry *parent, void *data, + enum d_walk_ret (*enter)(void *, struct dentry *), + void (*finish)(void *)); + +struct ac_dpages_arg { + int err; + struct au_dcsub_pages *dpages; + struct super_block *sb; + au_dpages_test test; + void *arg; +}; + +static enum d_walk_ret au_call_dpages_append(void *_arg, struct dentry *dentry) +{ + enum d_walk_ret ret; + struct ac_dpages_arg *arg = _arg; + + ret = D_WALK_CONTINUE; + if (dentry->d_sb == arg->sb + && !IS_ROOT(dentry) + && au_dcount(dentry) > 0 + && au_di(dentry) + && (!arg->test || arg->test(dentry, arg->arg))) { + arg->err = au_dpages_append(arg->dpages, dentry, GFP_ATOMIC); + if (unlikely(arg->err)) + ret = D_WALK_QUIT; + } + + return ret; +} + +int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root, + au_dpages_test test, void *arg) +{ + struct ac_dpages_arg args = { + .err = 0, + .dpages = dpages, + .sb = root->d_sb, + .test = test, + .arg = arg + }; + + d_walk(root, &args, au_call_dpages_append, NULL); + + return args.err; +} + +int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry, + int do_include, au_dpages_test test, void *arg) +{ + int err; + + err = 0; + write_seqlock(&rename_lock); + spin_lock(&dentry->d_lock); + if (do_include + && au_dcount(dentry) > 0 + && (!test || test(dentry, arg))) + err = au_dpages_append(dpages, dentry, GFP_ATOMIC); + spin_unlock(&dentry->d_lock); + if (unlikely(err)) + goto out; + + /* + * RCU for vfsmount is unnecessary since this is a traverse in a single + * mount + */ + while (!IS_ROOT(dentry)) { + dentry = dentry->d_parent; /* rename_lock is locked */ + spin_lock(&dentry->d_lock); + if (au_dcount(dentry) > 0 + && (!test || test(dentry, arg))) + err = au_dpages_append(dpages, dentry, GFP_ATOMIC); + spin_unlock(&dentry->d_lock); + if (unlikely(err)) + break; + } + +out: + write_sequnlock(&rename_lock); + return err; +} + +static inline int au_dcsub_dpages_aufs(struct dentry *dentry, void *arg) +{ + return au_di(dentry) && dentry->d_sb == arg; +} + +int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages, + struct dentry *dentry, int do_include) +{ + return au_dcsub_pages_rev(dpages, dentry, do_include, + au_dcsub_dpages_aufs, dentry->d_sb); +} + +int au_test_subdir(struct dentry *d1, struct dentry *d2) +{ + struct path path[2] = { + { + .dentry = d1 + }, + { + .dentry = d2 + } + }; + + return path_is_under(path + 0, path + 1); +} diff --git a/fs/aufs/dcsub.h b/fs/aufs/dcsub.h new file mode 100644 index 0000000000000..5e578995c2563 --- /dev/null +++ b/fs/aufs/dcsub.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for dentry cache + */ + +#ifndef __AUFS_DCSUB_H__ +#define __AUFS_DCSUB_H__ + +#ifdef __KERNEL__ + +#include +#include + +struct au_dpage { + int ndentry; + struct dentry **dentries; +}; + +struct au_dcsub_pages { + int ndpage; + struct au_dpage *dpages; +}; + +/* ---------------------------------------------------------------------- */ + +/* dcsub.c */ +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp); +void au_dpages_free(struct au_dcsub_pages *dpages); +typedef int (*au_dpages_test)(struct dentry *dentry, void *arg); +int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root, + au_dpages_test test, void *arg); +int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry, + int do_include, au_dpages_test test, void *arg); +int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages, + struct dentry *dentry, int do_include); +int au_test_subdir(struct dentry *d1, struct dentry *d2); + +/* ---------------------------------------------------------------------- */ + +/* + * todo: in linux-3.13, several similar (but faster) helpers are added to + * include/linux/dcache.h. Try them (in the future). + */ + +static inline int au_d_hashed_positive(struct dentry *d) +{ + int err; + struct inode *inode = d_inode(d); + + err = 0; + if (unlikely(d_unhashed(d) + || d_is_negative(d) + || !inode->i_nlink)) + err = -ENOENT; + return err; +} + +static inline int au_d_linkable(struct dentry *d) +{ + int err; + struct inode *inode = d_inode(d); + + err = au_d_hashed_positive(d); + if (err + && d_is_positive(d) + && (inode->i_state & I_LINKABLE)) + err = 0; + return err; +} + +static inline int au_d_alive(struct dentry *d) +{ + int err; + struct inode *inode; + + err = 0; + if (!IS_ROOT(d)) + err = au_d_hashed_positive(d); + else { + inode = d_inode(d); + if (unlikely(d_unlinked(d) + || d_is_negative(d) + || !inode->i_nlink)) + err = -ENOENT; + } + return err; +} + +static inline int au_alive_dir(struct dentry *d) +{ + int err; + + err = au_d_alive(d); + if (unlikely(err || IS_DEADDIR(d_inode(d)))) + err = -ENOENT; + return err; +} + +static inline int au_qstreq(struct qstr *a, struct qstr *b) +{ + return a->len == b->len + && !memcmp(a->name, b->name, a->len); +} + +/* + * by the commit + * 360f547 2015-01-25 dcache: let the dentry count go down to zero without + * taking d_lock + * the type of d_lockref.count became int, but the inlined function d_count() + * still returns unsigned int. + * I don't know why. Maybe it is for every d_count() users? + * Anyway au_dcount() lives on. + */ +static inline int au_dcount(struct dentry *d) +{ + return (int)d_count(d); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DCSUB_H__ */ diff --git a/fs/aufs/debug.c b/fs/aufs/debug.c new file mode 100644 index 0000000000000..e8b122e350673 --- /dev/null +++ b/fs/aufs/debug.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debug print functions + */ + +#include "aufs.h" + +/* Returns 0, or -errno. arg is in kp->arg. */ +static int param_atomic_t_set(const char *val, const struct kernel_param *kp) +{ + int err, n; + + err = kstrtoint(val, 0, &n); + if (!err) { + if (n > 0) + au_debug_on(); + else + au_debug_off(); + } + return err; +} + +/* Returns length written or -errno. Buffer is 4k (ie. be short!) */ +static int param_atomic_t_get(char *buffer, const struct kernel_param *kp) +{ + atomic_t *a; + + a = kp->arg; + return sprintf(buffer, "%d", atomic_read(a)); +} + +static struct kernel_param_ops param_ops_atomic_t = { + .set = param_atomic_t_set, + .get = param_atomic_t_get + /* void (*free)(void *arg) */ +}; + +atomic_t aufs_debug = ATOMIC_INIT(0); +MODULE_PARM_DESC(debug, "debug print"); +module_param_named(debug, aufs_debug, atomic_t, 0664); + +DEFINE_MUTEX(au_dbg_mtx); /* just to serialize the dbg msgs */ +char *au_plevel = KERN_DEBUG; +#define dpri(fmt, ...) do { \ + if ((au_plevel \ + && strcmp(au_plevel, KERN_DEBUG)) \ + || au_debug_test()) \ + printk("%s" fmt, au_plevel, ##__VA_ARGS__); \ +} while (0) + +/* ---------------------------------------------------------------------- */ + +void au_dpri_whlist(struct au_nhash *whlist) +{ + unsigned long ul, n; + struct hlist_head *head; + struct au_vdir_wh *pos; + + n = whlist->nh_num; + head = whlist->nh_head; + for (ul = 0; ul < n; ul++) { + hlist_for_each_entry(pos, head, wh_hash) + dpri("b%d, %.*s, %d\n", + pos->wh_bindex, + pos->wh_str.len, pos->wh_str.name, + pos->wh_str.len); + head++; + } +} + +void au_dpri_vdir(struct au_vdir *vdir) +{ + unsigned long ul; + union au_vdir_deblk_p p; + unsigned char *o; + + if (!vdir || IS_ERR(vdir)) { + dpri("err %ld\n", PTR_ERR(vdir)); + return; + } + + dpri("deblk %u, nblk %lu, deblk %p, last{%lu, %p}, ver %lu\n", + vdir->vd_deblk_sz, vdir->vd_nblk, vdir->vd_deblk, + vdir->vd_last.ul, vdir->vd_last.p.deblk, vdir->vd_version); + for (ul = 0; ul < vdir->vd_nblk; ul++) { + p.deblk = vdir->vd_deblk[ul]; + o = p.deblk; + dpri("[%lu]: %p\n", ul, o); + } +} + +static int do_pri_inode(aufs_bindex_t bindex, struct inode *inode, int hn, + struct dentry *wh) +{ + char *n = NULL; + int l = 0; + + if (!inode || IS_ERR(inode)) { + dpri("i%d: err %ld\n", bindex, PTR_ERR(inode)); + return -1; + } + + /* the type of i_blocks depends upon CONFIG_LBDAF */ + BUILD_BUG_ON(sizeof(inode->i_blocks) != sizeof(unsigned long) + && sizeof(inode->i_blocks) != sizeof(u64)); + if (wh) { + n = (void *)wh->d_name.name; + l = wh->d_name.len; + } + + dpri("i%d: %p, i%lu, %s, cnt %d, nl %u, 0%o, sz %llu, blk %llu," + " hn %d, ct %lld, np %lu, st 0x%lx, f 0x%x, v %llu, g %x%s%.*s\n", + bindex, inode, + inode->i_ino, inode->i_sb ? au_sbtype(inode->i_sb) : "??", + atomic_read(&inode->i_count), inode->i_nlink, inode->i_mode, + i_size_read(inode), (unsigned long long)inode->i_blocks, + hn, (long long)timespec_to_ns(&inode->i_ctime) & 0x0ffff, + inode->i_mapping ? inode->i_mapping->nrpages : 0, + inode->i_state, inode->i_flags, inode->i_version, + inode->i_generation, + l ? ", wh " : "", l, n); + return 0; +} + +void au_dpri_inode(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_hinode *hi; + aufs_bindex_t bindex; + int err, hn; + + err = do_pri_inode(-1, inode, -1, NULL); + if (err || !au_test_aufs(inode->i_sb) || au_is_bad_inode(inode)) + return; + + iinfo = au_ii(inode); + dpri("i-1: btop %d, bbot %d, gen %d\n", + iinfo->ii_btop, iinfo->ii_bbot, au_iigen(inode, NULL)); + if (iinfo->ii_btop < 0) + return; + hn = 0; + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; bindex++) { + hi = au_hinode(iinfo, bindex); + hn = !!au_hn(hi); + do_pri_inode(bindex, hi->hi_inode, hn, hi->hi_whdentry); + } +} + +void au_dpri_dalias(struct inode *inode) +{ + struct dentry *d; + + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) + au_dpri_dentry(d); + spin_unlock(&inode->i_lock); +} + +static int do_pri_dentry(aufs_bindex_t bindex, struct dentry *dentry) +{ + struct dentry *wh = NULL; + int hn; + struct inode *inode; + struct au_iinfo *iinfo; + struct au_hinode *hi; + + if (!dentry || IS_ERR(dentry)) { + dpri("d%d: err %ld\n", bindex, PTR_ERR(dentry)); + return -1; + } + /* do not call dget_parent() here */ + /* note: access d_xxx without d_lock */ + dpri("d%d: %p, %pd2?, %s, cnt %d, flags 0x%x, %shashed\n", + bindex, dentry, dentry, + dentry->d_sb ? au_sbtype(dentry->d_sb) : "??", + au_dcount(dentry), dentry->d_flags, + d_unhashed(dentry) ? "un" : ""); + hn = -1; + inode = NULL; + if (d_is_positive(dentry)) + inode = d_inode(dentry); + if (inode + && au_test_aufs(dentry->d_sb) + && bindex >= 0 + && !au_is_bad_inode(inode)) { + iinfo = au_ii(inode); + hi = au_hinode(iinfo, bindex); + hn = !!au_hn(hi); + wh = hi->hi_whdentry; + } + do_pri_inode(bindex, inode, hn, wh); + return 0; +} + +void au_dpri_dentry(struct dentry *dentry) +{ + struct au_dinfo *dinfo; + aufs_bindex_t bindex; + int err; + + err = do_pri_dentry(-1, dentry); + if (err || !au_test_aufs(dentry->d_sb)) + return; + + dinfo = au_di(dentry); + if (!dinfo) + return; + dpri("d-1: btop %d, bbot %d, bwh %d, bdiropq %d, gen %d, tmp %d\n", + dinfo->di_btop, dinfo->di_bbot, + dinfo->di_bwh, dinfo->di_bdiropq, au_digen(dentry), + dinfo->di_tmpfile); + if (dinfo->di_btop < 0) + return; + for (bindex = dinfo->di_btop; bindex <= dinfo->di_bbot; bindex++) + do_pri_dentry(bindex, au_hdentry(dinfo, bindex)->hd_dentry); +} + +static int do_pri_file(aufs_bindex_t bindex, struct file *file) +{ + char a[32]; + + if (!file || IS_ERR(file)) { + dpri("f%d: err %ld\n", bindex, PTR_ERR(file)); + return -1; + } + a[0] = 0; + if (bindex < 0 + && !IS_ERR_OR_NULL(file->f_path.dentry) + && au_test_aufs(file->f_path.dentry->d_sb) + && au_fi(file)) + snprintf(a, sizeof(a), ", gen %d, mmapped %d", + au_figen(file), atomic_read(&au_fi(file)->fi_mmapped)); + dpri("f%d: mode 0x%x, flags 0%o, cnt %ld, v %llu, pos %llu%s\n", + bindex, file->f_mode, file->f_flags, (long)file_count(file), + file->f_version, file->f_pos, a); + if (!IS_ERR_OR_NULL(file->f_path.dentry)) + do_pri_dentry(bindex, file->f_path.dentry); + return 0; +} + +void au_dpri_file(struct file *file) +{ + struct au_finfo *finfo; + struct au_fidir *fidir; + struct au_hfile *hfile; + aufs_bindex_t bindex; + int err; + + err = do_pri_file(-1, file); + if (err + || IS_ERR_OR_NULL(file->f_path.dentry) + || !au_test_aufs(file->f_path.dentry->d_sb)) + return; + + finfo = au_fi(file); + if (!finfo) + return; + if (finfo->fi_btop < 0) + return; + fidir = finfo->fi_hdir; + if (!fidir) + do_pri_file(finfo->fi_btop, finfo->fi_htop.hf_file); + else + for (bindex = finfo->fi_btop; + bindex >= 0 && bindex <= fidir->fd_bbot; + bindex++) { + hfile = fidir->fd_hfile + bindex; + do_pri_file(bindex, hfile ? hfile->hf_file : NULL); + } +} + +static int do_pri_br(aufs_bindex_t bindex, struct au_branch *br) +{ + struct vfsmount *mnt; + struct super_block *sb; + + if (!br || IS_ERR(br)) + goto out; + mnt = au_br_mnt(br); + if (!mnt || IS_ERR(mnt)) + goto out; + sb = mnt->mnt_sb; + if (!sb || IS_ERR(sb)) + goto out; + + dpri("s%d: {perm 0x%x, id %d, wbr %p}, " + "%s, dev 0x%02x%02x, flags 0x%lx, cnt %d, active %d, " + "xino %d\n", + bindex, br->br_perm, br->br_id, br->br_wbr, + au_sbtype(sb), MAJOR(sb->s_dev), MINOR(sb->s_dev), + sb->s_flags, sb->s_count, + atomic_read(&sb->s_active), + !!au_xino_file(br->br_xino, /*idx*/-1)); + return 0; + +out: + dpri("s%d: err %ld\n", bindex, PTR_ERR(br)); + return -1; +} + +void au_dpri_sb(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + aufs_bindex_t bindex; + int err; + /* to reduce stack size */ + struct { + struct vfsmount mnt; + struct au_branch fake; + } *a; + + /* this function can be called from magic sysrq */ + a = kzalloc(sizeof(*a), GFP_ATOMIC); + if (unlikely(!a)) { + dpri("no memory\n"); + return; + } + + a->mnt.mnt_sb = sb; + a->fake.br_path.mnt = &a->mnt; + err = do_pri_br(-1, &a->fake); + au_kfree_rcu(a); + dpri("dev 0x%x\n", sb->s_dev); + if (err || !au_test_aufs(sb)) + return; + + sbinfo = au_sbi(sb); + if (!sbinfo) + return; + dpri("nw %d, gen %u, kobj %d\n", + atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation, + kref_read(&sbinfo->si_kobj.kref)); + for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++) + do_pri_br(bindex, sbinfo->si_branch[0 + bindex]); +} + +/* ---------------------------------------------------------------------- */ + +void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line) +{ + struct inode *h_inode, *inode = d_inode(dentry); + struct dentry *h_dentry; + aufs_bindex_t bindex, bbot, bi; + + if (!inode /* || au_di(dentry)->di_lsc == AuLsc_DI_TMP */) + return; + + bbot = au_dbbot(dentry); + bi = au_ibbot(inode); + if (bi < bbot) + bbot = bi; + bindex = au_dbtop(dentry); + bi = au_ibtop(inode); + if (bi > bindex) + bindex = bi; + + for (; bindex <= bbot; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + h_inode = au_h_iptr(inode, bindex); + if (unlikely(h_inode != d_inode(h_dentry))) { + au_debug_on(); + AuDbg("b%d, %s:%d\n", bindex, func, line); + AuDbgDentry(dentry); + AuDbgInode(inode); + au_debug_off(); + BUG(); + } + } +} + +void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen) +{ + int err, i, j; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries; + + err = au_dpages_init(&dpages, GFP_NOFS); + AuDebugOn(err); + err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/1); + AuDebugOn(err); + for (i = dpages.ndpage - 1; !err && i >= 0; i--) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + for (j = dpage->ndentry - 1; !err && j >= 0; j--) + AuDebugOn(au_digen_test(dentries[j], sigen)); + } + au_dpages_free(&dpages); +} + +void au_dbg_verify_kthread(void) +{ + if (au_wkq_test()) { + au_dbg_blocked(); + /* + * It may be recursive, but udba=notify between two aufs mounts, + * where a single ro branch is shared, is not a problem. + */ + /* WARN_ON(1); */ + } +} + +/* ---------------------------------------------------------------------- */ + +int __init au_debug_init(void) +{ + aufs_bindex_t bindex; + struct au_vdir_destr destr; + + bindex = -1; + AuDebugOn(bindex >= 0); + + destr.len = -1; + AuDebugOn(destr.len < NAME_MAX); + +#ifdef CONFIG_4KSTACKS + pr_warn("CONFIG_4KSTACKS is defined.\n"); +#endif + + return 0; +} diff --git a/fs/aufs/debug.h b/fs/aufs/debug.h new file mode 100644 index 0000000000000..9c52470a61fba --- /dev/null +++ b/fs/aufs/debug.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debug print functions + */ + +#ifndef __AUFS_DEBUG_H__ +#define __AUFS_DEBUG_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +#ifdef CONFIG_AUFS_DEBUG +#define AuDebugOn(a) BUG_ON(a) + +/* module parameter */ +extern atomic_t aufs_debug; +static inline void au_debug_on(void) +{ + atomic_inc(&aufs_debug); +} +static inline void au_debug_off(void) +{ + atomic_dec_if_positive(&aufs_debug); +} + +static inline int au_debug_test(void) +{ + return atomic_read(&aufs_debug) > 0; +} +#else +#define AuDebugOn(a) do {} while (0) +AuStubVoid(au_debug_on, void) +AuStubVoid(au_debug_off, void) +AuStubInt0(au_debug_test, void) +#endif /* CONFIG_AUFS_DEBUG */ + +#define param_check_atomic_t(name, p) __param_check(name, p, atomic_t) + +/* ---------------------------------------------------------------------- */ + +/* debug print */ + +#define AuDbg(fmt, ...) do { \ + if (au_debug_test()) \ + pr_debug("DEBUG: " fmt, ##__VA_ARGS__); \ +} while (0) +#define AuLabel(l) AuDbg(#l "\n") +#define AuIOErr(fmt, ...) pr_err("I/O Error, " fmt, ##__VA_ARGS__) +#define AuWarn1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + pr_warn(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuErr1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + pr_err(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuIOErr1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + AuIOErr(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuUnsupportMsg "This operation is not supported." \ + " Please report this application to aufs-users ML." +#define AuUnsupport(fmt, ...) do { \ + pr_err(AuUnsupportMsg "\n" fmt, ##__VA_ARGS__); \ + dump_stack(); \ +} while (0) + +#define AuTraceErr(e) do { \ + if (unlikely((e) < 0)) \ + AuDbg("err %d\n", (int)(e)); \ +} while (0) + +#define AuTraceErrPtr(p) do { \ + if (IS_ERR(p)) \ + AuDbg("err %ld\n", PTR_ERR(p)); \ +} while (0) + +/* dirty macros for debug print, use with "%.*s" and caution */ +#define AuLNPair(qstr) (qstr)->len, (qstr)->name + +/* ---------------------------------------------------------------------- */ + +struct dentry; +#ifdef CONFIG_AUFS_DEBUG +extern struct mutex au_dbg_mtx; +extern char *au_plevel; +struct au_nhash; +void au_dpri_whlist(struct au_nhash *whlist); +struct au_vdir; +void au_dpri_vdir(struct au_vdir *vdir); +struct inode; +void au_dpri_inode(struct inode *inode); +void au_dpri_dalias(struct inode *inode); +void au_dpri_dentry(struct dentry *dentry); +struct file; +void au_dpri_file(struct file *filp); +struct super_block; +void au_dpri_sb(struct super_block *sb); + +#define au_dbg_verify_dinode(d) __au_dbg_verify_dinode(d, __func__, __LINE__) +void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line); +void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen); +void au_dbg_verify_kthread(void); + +int __init au_debug_init(void); + +#define AuDbgWhlist(w) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#w "\n"); \ + au_dpri_whlist(w); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgVdir(v) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#v "\n"); \ + au_dpri_vdir(v); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgInode(i) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#i "\n"); \ + au_dpri_inode(i); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgDAlias(i) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#i "\n"); \ + au_dpri_dalias(i); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgDentry(d) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#d "\n"); \ + au_dpri_dentry(d); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgFile(f) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#f "\n"); \ + au_dpri_file(f); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgSb(sb) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#sb "\n"); \ + au_dpri_sb(sb); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgSym(addr) do { \ + char sym[KSYM_SYMBOL_LEN]; \ + sprint_symbol(sym, (unsigned long)addr); \ + AuDbg("%s\n", sym); \ +} while (0) +#else +AuStubVoid(au_dbg_verify_dinode, struct dentry *dentry) +AuStubVoid(au_dbg_verify_gen, struct dentry *parent, unsigned int sigen) +AuStubVoid(au_dbg_verify_kthread, void) +AuStubInt0(__init au_debug_init, void) + +#define AuDbgWhlist(w) do {} while (0) +#define AuDbgVdir(v) do {} while (0) +#define AuDbgInode(i) do {} while (0) +#define AuDbgDAlias(i) do {} while (0) +#define AuDbgDentry(d) do {} while (0) +#define AuDbgFile(f) do {} while (0) +#define AuDbgSb(sb) do {} while (0) +#define AuDbgSym(addr) do {} while (0) +#endif /* CONFIG_AUFS_DEBUG */ + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_MAGIC_SYSRQ +int __init au_sysrq_init(void); +void au_sysrq_fin(void); + +#ifdef CONFIG_HW_CONSOLE +#define au_dbg_blocked() do { \ + WARN_ON(1); \ + handle_sysrq('w'); \ +} while (0) +#else +AuStubVoid(au_dbg_blocked, void) +#endif + +#else +AuStubInt0(__init au_sysrq_init, void) +AuStubVoid(au_sysrq_fin, void) +AuStubVoid(au_dbg_blocked, void) +#endif /* CONFIG_AUFS_MAGIC_SYSRQ */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DEBUG_H__ */ diff --git a/fs/aufs/dentry.c b/fs/aufs/dentry.c new file mode 100644 index 0000000000000..55bb0dad2a62a --- /dev/null +++ b/fs/aufs/dentry.c @@ -0,0 +1,1153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * lookup and dentry operations + */ + +#include +#include "aufs.h" + +/* + * returns positive/negative dentry, NULL or an error. + * NULL means whiteout-ed or not-found. + */ +static struct dentry* +au_do_lookup(struct dentry *h_parent, struct dentry *dentry, + aufs_bindex_t bindex, struct au_do_lookup_args *args) +{ + struct dentry *h_dentry; + struct inode *h_inode; + struct au_branch *br; + int wh_found, opq; + unsigned char wh_able; + const unsigned char allow_neg = !!au_ftest_lkup(args->flags, ALLOW_NEG); + const unsigned char ignore_perm = !!au_ftest_lkup(args->flags, + IGNORE_PERM); + + wh_found = 0; + br = au_sbr(dentry->d_sb, bindex); + wh_able = !!au_br_whable(br->br_perm); + if (wh_able) + wh_found = au_wh_test(h_parent, &args->whname, ignore_perm); + h_dentry = ERR_PTR(wh_found); + if (!wh_found) + goto real_lookup; + if (unlikely(wh_found < 0)) + goto out; + + /* We found a whiteout */ + /* au_set_dbbot(dentry, bindex); */ + au_set_dbwh(dentry, bindex); + if (!allow_neg) + return NULL; /* success */ + +real_lookup: + if (!ignore_perm) + h_dentry = vfsub_lkup_one(args->name, h_parent); + else + h_dentry = au_sio_lkup_one(args->name, h_parent); + if (IS_ERR(h_dentry)) { + if (PTR_ERR(h_dentry) == -ENAMETOOLONG + && !allow_neg) + h_dentry = NULL; + goto out; + } + + h_inode = d_inode(h_dentry); + if (d_is_negative(h_dentry)) { + if (!allow_neg) + goto out_neg; + } else if (wh_found + || (args->type && args->type != (h_inode->i_mode & S_IFMT))) + goto out_neg; + else if (au_ftest_lkup(args->flags, DIRREN) + /* && h_inode */ + && !au_dr_lkup_h_ino(args, bindex, h_inode->i_ino)) { + AuDbg("b%d %pd ignored hi%llu\n", bindex, h_dentry, + (unsigned long long)h_inode->i_ino); + goto out_neg; + } + + if (au_dbbot(dentry) <= bindex) + au_set_dbbot(dentry, bindex); + if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry)) + au_set_dbtop(dentry, bindex); + au_set_h_dptr(dentry, bindex, h_dentry); + + if (!d_is_dir(h_dentry) + || !wh_able + || (d_really_is_positive(dentry) && !d_is_dir(dentry))) + goto out; /* success */ + + vfsub_inode_lock_shared_nested(h_inode, AuLsc_I_CHILD); + opq = au_diropq_test(h_dentry); + inode_unlock_shared(h_inode); + if (opq > 0) + au_set_dbdiropq(dentry, bindex); + else if (unlikely(opq < 0)) { + au_set_h_dptr(dentry, bindex, NULL); + h_dentry = ERR_PTR(opq); + } + goto out; + +out_neg: + dput(h_dentry); + h_dentry = NULL; +out: + return h_dentry; +} + +static int au_test_shwh(struct super_block *sb, const struct qstr *name) +{ + if (unlikely(!au_opt_test(au_mntflags(sb), SHWH) + && !strncmp(name->name, AUFS_WH_PFX, AUFS_WH_PFX_LEN))) + return -EPERM; + return 0; +} + +/* + * returns the number of lower positive dentries, + * otherwise an error. + * can be called at unlinking with @type is zero. + */ +int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop, + unsigned int flags) +{ + int npositive, err; + aufs_bindex_t bindex, btail, bdiropq; + unsigned char isdir, dirperm1, dirren; + struct au_do_lookup_args args = { + .flags = flags, + .name = &dentry->d_name + }; + struct dentry *parent; + struct super_block *sb; + + sb = dentry->d_sb; + err = au_test_shwh(sb, args.name); + if (unlikely(err)) + goto out; + + err = au_wh_name_alloc(&args.whname, args.name); + if (unlikely(err)) + goto out; + + isdir = !!d_is_dir(dentry); + dirperm1 = !!au_opt_test(au_mntflags(sb), DIRPERM1); + dirren = !!au_opt_test(au_mntflags(sb), DIRREN); + if (dirren) + au_fset_lkup(args.flags, DIRREN); + + npositive = 0; + parent = dget_parent(dentry); + btail = au_dbtaildir(parent); + for (bindex = btop; bindex <= btail; bindex++) { + struct dentry *h_parent, *h_dentry; + struct inode *h_inode, *h_dir; + struct au_branch *br; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) { + if (d_is_positive(h_dentry)) + npositive++; + break; + } + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || !d_is_dir(h_parent)) + continue; + + if (dirren) { + /* if the inum matches, then use the prepared name */ + err = au_dr_lkup_name(&args, bindex); + if (unlikely(err)) + goto out_parent; + } + + h_dir = d_inode(h_parent); + vfsub_inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + h_dentry = au_do_lookup(h_parent, dentry, bindex, &args); + inode_unlock_shared(h_dir); + err = PTR_ERR(h_dentry); + if (IS_ERR(h_dentry)) + goto out_parent; + if (h_dentry) + au_fclr_lkup(args.flags, ALLOW_NEG); + if (dirperm1) + au_fset_lkup(args.flags, IGNORE_PERM); + + if (au_dbwh(dentry) == bindex) + break; + if (!h_dentry) + continue; + if (d_is_negative(h_dentry)) + continue; + h_inode = d_inode(h_dentry); + npositive++; + if (!args.type) + args.type = h_inode->i_mode & S_IFMT; + if (args.type != S_IFDIR) + break; + else if (isdir) { + /* the type of lower may be different */ + bdiropq = au_dbdiropq(dentry); + if (bdiropq >= 0 && bdiropq <= bindex) + break; + } + br = au_sbr(sb, bindex); + if (dirren + && au_dr_hino_test_add(&br->br_dirren, h_inode->i_ino, + /*add_ent*/NULL)) { + /* prepare next name to lookup */ + err = au_dr_lkup(&args, dentry, bindex); + if (unlikely(err)) + goto out_parent; + } + } + + if (npositive) { + AuLabel(positive); + au_update_dbtop(dentry); + } + err = npositive; + if (unlikely(!au_opt_test(au_mntflags(sb), UDBA_NONE) + && au_dbtop(dentry) < 0)) { + err = -EIO; + AuIOErr("both of real entry and whiteout found, %pd, err %d\n", + dentry, err); + } + +out_parent: + dput(parent); + au_kfree_try_rcu(args.whname.name); + if (dirren) + au_dr_lkup_fin(&args); +out: + return err; +} + +struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent) +{ + struct dentry *dentry; + int wkq_err; + + if (!au_test_h_perm_sio(d_inode(parent), MAY_EXEC)) + dentry = vfsub_lkup_one(name, parent); + else { + struct vfsub_lkup_one_args args = { + .errp = &dentry, + .name = name, + .parent = parent + }; + + wkq_err = au_wkq_wait(vfsub_call_lkup_one, &args); + if (unlikely(wkq_err)) + dentry = ERR_PTR(wkq_err); + } + + return dentry; +} + +/* + * lookup @dentry on @bindex which should be negative. + */ +int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh) +{ + int err; + struct dentry *parent, *h_parent, *h_dentry; + struct au_branch *br; + + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bindex); + br = au_sbr(dentry->d_sb, bindex); + if (wh) + h_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name); + else + h_dentry = au_sio_lkup_one(&dentry->d_name, h_parent); + err = PTR_ERR(h_dentry); + if (IS_ERR(h_dentry)) + goto out; + if (unlikely(d_is_positive(h_dentry))) { + err = -EIO; + AuIOErr("%pd should be negative on b%d.\n", h_dentry, bindex); + dput(h_dentry); + goto out; + } + + err = 0; + if (bindex < au_dbtop(dentry)) + au_set_dbtop(dentry, bindex); + if (au_dbbot(dentry) < bindex) + au_set_dbbot(dentry, bindex); + au_set_h_dptr(dentry, bindex, h_dentry); + +out: + dput(parent); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* subset of struct inode */ +struct au_iattr { + unsigned long i_ino; + /* unsigned int i_nlink; */ + kuid_t i_uid; + kgid_t i_gid; + u64 i_version; +/* + loff_t i_size; + blkcnt_t i_blocks; +*/ + umode_t i_mode; +}; + +static void au_iattr_save(struct au_iattr *ia, struct inode *h_inode) +{ + ia->i_ino = h_inode->i_ino; + /* ia->i_nlink = h_inode->i_nlink; */ + ia->i_uid = h_inode->i_uid; + ia->i_gid = h_inode->i_gid; + ia->i_version = h_inode->i_version; +/* + ia->i_size = h_inode->i_size; + ia->i_blocks = h_inode->i_blocks; +*/ + ia->i_mode = (h_inode->i_mode & S_IFMT); +} + +static int au_iattr_test(struct au_iattr *ia, struct inode *h_inode) +{ + return ia->i_ino != h_inode->i_ino + /* || ia->i_nlink != h_inode->i_nlink */ + || !uid_eq(ia->i_uid, h_inode->i_uid) + || !gid_eq(ia->i_gid, h_inode->i_gid) + || ia->i_version != h_inode->i_version +/* + || ia->i_size != h_inode->i_size + || ia->i_blocks != h_inode->i_blocks +*/ + || ia->i_mode != (h_inode->i_mode & S_IFMT); +} + +static int au_h_verify_dentry(struct dentry *h_dentry, struct dentry *h_parent, + struct au_branch *br) +{ + int err; + struct au_iattr ia; + struct inode *h_inode; + struct dentry *h_d; + struct super_block *h_sb; + + err = 0; + memset(&ia, -1, sizeof(ia)); + h_sb = h_dentry->d_sb; + h_inode = NULL; + if (d_is_positive(h_dentry)) { + h_inode = d_inode(h_dentry); + au_iattr_save(&ia, h_inode); + } else if (au_test_nfs(h_sb) || au_test_fuse(h_sb)) + /* nfs d_revalidate may return 0 for negative dentry */ + /* fuse d_revalidate always return 0 for negative dentry */ + goto out; + + /* main purpose is namei.c:cached_lookup() and d_revalidate */ + h_d = vfsub_lkup_one(&h_dentry->d_name, h_parent); + err = PTR_ERR(h_d); + if (IS_ERR(h_d)) + goto out; + + err = 0; + if (unlikely(h_d != h_dentry + || d_inode(h_d) != h_inode + || (h_inode && au_iattr_test(&ia, h_inode)))) + err = au_busy_or_stale(); + dput(h_d); + +out: + AuTraceErr(err); + return err; +} + +int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir, + struct dentry *h_parent, struct au_branch *br) +{ + int err; + + err = 0; + if (udba == AuOpt_UDBA_REVAL + && !au_test_fs_remote(h_dentry->d_sb)) { + IMustLock(h_dir); + err = (d_inode(h_dentry->d_parent) != h_dir); + } else if (udba != AuOpt_UDBA_NONE) + err = au_h_verify_dentry(h_dentry, h_parent, br); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_refresh_hdentry(struct dentry *dentry, struct dentry *parent) +{ + int err; + aufs_bindex_t new_bindex, bindex, bbot, bwh, bdiropq; + struct au_hdentry tmp, *p, *q; + struct au_dinfo *dinfo; + struct super_block *sb; + + DiMustWriteLock(dentry); + + sb = dentry->d_sb; + dinfo = au_di(dentry); + bbot = dinfo->di_bbot; + bwh = dinfo->di_bwh; + bdiropq = dinfo->di_bdiropq; + bindex = dinfo->di_btop; + p = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, p++) { + if (!p->hd_dentry) + continue; + + new_bindex = au_br_index(sb, p->hd_id); + if (new_bindex == bindex) + continue; + + if (dinfo->di_bwh == bindex) + bwh = new_bindex; + if (dinfo->di_bdiropq == bindex) + bdiropq = new_bindex; + if (new_bindex < 0) { + au_hdput(p); + p->hd_dentry = NULL; + continue; + } + + /* swap two lower dentries, and loop again */ + q = au_hdentry(dinfo, new_bindex); + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hd_dentry) { + bindex--; + p--; + } + } + + dinfo->di_bwh = -1; + if (bwh >= 0 && bwh <= au_sbbot(sb) && au_sbr_whable(sb, bwh)) + dinfo->di_bwh = bwh; + + dinfo->di_bdiropq = -1; + if (bdiropq >= 0 + && bdiropq <= au_sbbot(sb) + && au_sbr_whable(sb, bdiropq)) + dinfo->di_bdiropq = bdiropq; + + err = -EIO; + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + bbot = au_dbbot(parent); + bindex = 0; + p = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, p++) + if (p->hd_dentry) { + dinfo->di_btop = bindex; + break; + } + + if (dinfo->di_btop >= 0) { + bindex = bbot; + p = au_hdentry(dinfo, bindex); + for (; bindex >= 0; bindex--, p--) + if (p->hd_dentry) { + dinfo->di_bbot = bindex; + err = 0; + break; + } + } + + return err; +} + +static void au_do_hide(struct dentry *dentry) +{ + struct inode *inode; + + if (d_really_is_positive(dentry)) { + inode = d_inode(dentry); + if (!d_is_dir(dentry)) { + if (inode->i_nlink && !d_unhashed(dentry)) + drop_nlink(inode); + } else { + clear_nlink(inode); + /* stop next lookup */ + inode->i_flags |= S_DEAD; + } + smp_mb(); /* necessary? */ + } + d_drop(dentry); +} + +static int au_hide_children(struct dentry *parent) +{ + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry *dentry; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, parent, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + /* in reverse order */ + for (i = dpages.ndpage - 1; i >= 0; i--) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = ndentry - 1; j >= 0; j--) { + dentry = dpage->dentries[j]; + if (dentry != parent) + au_do_hide(dentry); + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static void au_hide(struct dentry *dentry) +{ + int err; + + AuDbgDentry(dentry); + if (d_is_dir(dentry)) { + /* shrink_dcache_parent(dentry); */ + err = au_hide_children(dentry); + if (unlikely(err)) + AuIOErr("%pd, failed hiding children, ignored %d\n", + dentry, err); + } + au_do_hide(dentry); +} + +/* + * By adding a dirty branch, a cached dentry may be affected in various ways. + * + * a dirty branch is added + * - on the top of layers + * - in the middle of layers + * - to the bottom of layers + * + * on the added branch there exists + * - a whiteout + * - a diropq + * - a same named entry + * + exist + * * negative --> positive + * * positive --> positive + * - type is unchanged + * - type is changed + * + doesn't exist + * * negative --> negative + * * positive --> negative (rejected by au_br_del() for non-dir case) + * - none + */ +static int au_refresh_by_dinfo(struct dentry *dentry, struct au_dinfo *dinfo, + struct au_dinfo *tmp) +{ + int err; + aufs_bindex_t bindex, bbot; + struct { + struct dentry *dentry; + struct inode *inode; + mode_t mode; + } orig_h, tmp_h = { + .dentry = NULL + }; + struct au_hdentry *hd; + struct inode *inode, *h_inode; + struct dentry *h_dentry; + + err = 0; + AuDebugOn(dinfo->di_btop < 0); + orig_h.mode = 0; + orig_h.dentry = au_hdentry(dinfo, dinfo->di_btop)->hd_dentry; + orig_h.inode = NULL; + if (d_is_positive(orig_h.dentry)) { + orig_h.inode = d_inode(orig_h.dentry); + orig_h.mode = orig_h.inode->i_mode & S_IFMT; + } + if (tmp->di_btop >= 0) { + tmp_h.dentry = au_hdentry(tmp, tmp->di_btop)->hd_dentry; + if (d_is_positive(tmp_h.dentry)) { + tmp_h.inode = d_inode(tmp_h.dentry); + tmp_h.mode = tmp_h.inode->i_mode & S_IFMT; + } + } + + inode = NULL; + if (d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (!orig_h.inode) { + AuDbg("negative originally\n"); + if (inode) { + au_hide(dentry); + goto out; + } + AuDebugOn(inode); + AuDebugOn(dinfo->di_btop != dinfo->di_bbot); + AuDebugOn(dinfo->di_bdiropq != -1); + + if (!tmp_h.inode) { + AuDbg("negative --> negative\n"); + /* should have only one negative lower */ + if (tmp->di_btop >= 0 + && tmp->di_btop < dinfo->di_btop) { + AuDebugOn(tmp->di_btop != tmp->di_bbot); + AuDebugOn(dinfo->di_btop != dinfo->di_bbot); + au_set_h_dptr(dentry, dinfo->di_btop, NULL); + au_di_cp(dinfo, tmp); + hd = au_hdentry(tmp, tmp->di_btop); + au_set_h_dptr(dentry, tmp->di_btop, + dget(hd->hd_dentry)); + } + au_dbg_verify_dinode(dentry); + } else { + AuDbg("negative --> positive\n"); + /* + * similar to the behaviour of creating with bypassing + * aufs. + * unhash it in order to force an error in the + * succeeding create operation. + * we should not set S_DEAD here. + */ + d_drop(dentry); + /* au_di_swap(tmp, dinfo); */ + au_dbg_verify_dinode(dentry); + } + } else { + AuDbg("positive originally\n"); + /* inode may be NULL */ + AuDebugOn(inode && (inode->i_mode & S_IFMT) != orig_h.mode); + if (!tmp_h.inode) { + AuDbg("positive --> negative\n"); + /* or bypassing aufs */ + au_hide(dentry); + if (tmp->di_bwh >= 0 && tmp->di_bwh <= dinfo->di_btop) + dinfo->di_bwh = tmp->di_bwh; + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } else if (orig_h.mode == tmp_h.mode) { + AuDbg("positive --> positive, same type\n"); + if (!S_ISDIR(orig_h.mode) + && dinfo->di_btop > tmp->di_btop) { + /* + * similar to the behaviour of removing and + * creating. + */ + au_hide(dentry); + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } else { + /* fill empty slots */ + if (dinfo->di_btop > tmp->di_btop) + dinfo->di_btop = tmp->di_btop; + if (dinfo->di_bbot < tmp->di_bbot) + dinfo->di_bbot = tmp->di_bbot; + dinfo->di_bwh = tmp->di_bwh; + dinfo->di_bdiropq = tmp->di_bdiropq; + bbot = dinfo->di_bbot; + bindex = tmp->di_btop; + hd = au_hdentry(tmp, bindex); + for (; bindex <= bbot; bindex++, hd++) { + if (au_h_dptr(dentry, bindex)) + continue; + h_dentry = hd->hd_dentry; + if (!h_dentry) + continue; + AuDebugOn(d_is_negative(h_dentry)); + h_inode = d_inode(h_dentry); + AuDebugOn(orig_h.mode + != (h_inode->i_mode + & S_IFMT)); + au_set_h_dptr(dentry, bindex, + dget(h_dentry)); + } + if (inode) + err = au_refresh_hinode(inode, dentry); + au_dbg_verify_dinode(dentry); + } + } else { + AuDbg("positive --> positive, different type\n"); + /* similar to the behaviour of removing and creating */ + au_hide(dentry); + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } + } + +out: + return err; +} + +void au_refresh_dop(struct dentry *dentry, int force_reval) +{ + const struct dentry_operations *dop + = force_reval ? &aufs_dop : dentry->d_sb->s_d_op; + static const unsigned int mask + = DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE; + + BUILD_BUG_ON(sizeof(mask) != sizeof(dentry->d_flags)); + + if (dentry->d_op == dop) + return; + + AuDbg("%pd\n", dentry); + spin_lock(&dentry->d_lock); + if (dop == &aufs_dop) + dentry->d_flags |= mask; + else + dentry->d_flags &= ~mask; + dentry->d_op = dop; + spin_unlock(&dentry->d_lock); +} + +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent) +{ + int err, ebrange, nbr; + unsigned int sigen; + struct au_dinfo *dinfo, *tmp; + struct super_block *sb; + struct inode *inode; + + DiMustWriteLock(dentry); + AuDebugOn(IS_ROOT(dentry)); + AuDebugOn(d_really_is_negative(parent)); + + sb = dentry->d_sb; + sigen = au_sigen(sb); + err = au_digen_test(parent, sigen); + if (unlikely(err)) + goto out; + + nbr = au_sbbot(sb) + 1; + dinfo = au_di(dentry); + err = au_di_realloc(dinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + ebrange = au_dbrange_test(dentry); + if (!ebrange) + ebrange = au_do_refresh_hdentry(dentry, parent); + + if (d_unhashed(dentry) || ebrange /* || dinfo->di_tmpfile */) { + AuDebugOn(au_dbtop(dentry) < 0 && au_dbbot(dentry) >= 0); + if (d_really_is_positive(dentry)) { + inode = d_inode(dentry); + err = au_refresh_hinode_self(inode); + } + au_dbg_verify_dinode(dentry); + if (!err) + goto out_dgen; /* success */ + goto out; + } + + /* temporary dinfo */ + AuDbgDentry(dentry); + err = -ENOMEM; + tmp = au_di_alloc(sb, AuLsc_DI_TMP); + if (unlikely(!tmp)) + goto out; + au_di_swap(tmp, dinfo); + /* returns the number of positive dentries */ + /* + * if current working dir is removed, it returns an error. + * but the dentry is legal. + */ + err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG); + AuDbgDentry(dentry); + au_di_swap(tmp, dinfo); + if (err == -ENOENT) + err = 0; + if (err >= 0) { + /* compare/refresh by dinfo */ + AuDbgDentry(dentry); + err = au_refresh_by_dinfo(dentry, dinfo, tmp); + au_dbg_verify_dinode(dentry); + AuTraceErr(err); + } + au_di_realloc(dinfo, nbr, /*may_shrink*/1); /* harmless if err */ + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err)) + goto out; + +out_dgen: + au_update_digen(dentry); +out: + if (unlikely(err && !(dentry->d_flags & DCACHE_NFSFS_RENAMED))) { + AuIOErr("failed refreshing %pd, %d\n", dentry, err); + AuDbgDentry(dentry); + } + AuTraceErr(err); + return err; +} + +static int au_do_h_d_reval(struct dentry *h_dentry, unsigned int flags, + struct dentry *dentry, aufs_bindex_t bindex) +{ + int err, valid; + + err = 0; + if (!(h_dentry->d_flags & DCACHE_OP_REVALIDATE)) + goto out; + + AuDbg("b%d\n", bindex); + /* + * gave up supporting LOOKUP_CREATE/OPEN for lower fs, + * due to whiteout and branch permission. + */ + flags &= ~(/*LOOKUP_PARENT |*/ LOOKUP_OPEN | LOOKUP_CREATE + | LOOKUP_FOLLOW | LOOKUP_EXCL); + /* it may return tri-state */ + valid = h_dentry->d_op->d_revalidate(h_dentry, flags); + + if (unlikely(valid < 0)) + err = valid; + else if (!valid) + err = -EINVAL; + +out: + AuTraceErr(err); + return err; +} + +/* todo: remove this */ +static int h_d_revalidate(struct dentry *dentry, struct inode *inode, + unsigned int flags, int do_udba, int dirren) +{ + int err; + umode_t mode, h_mode; + aufs_bindex_t bindex, btail, btop, ibs, ibe; + unsigned char plus, unhashed, is_root, h_plus, h_nfs, tmpfile; + struct inode *h_inode, *h_cached_inode; + struct dentry *h_dentry; + struct qstr *name, *h_name; + + err = 0; + plus = 0; + mode = 0; + ibs = -1; + ibe = -1; + unhashed = !!d_unhashed(dentry); + is_root = !!IS_ROOT(dentry); + name = &dentry->d_name; + tmpfile = au_di(dentry)->di_tmpfile; + + /* + * Theoretically, REVAL test should be unnecessary in case of + * {FS,I}NOTIFY. + * But {fs,i}notify doesn't fire some necessary events, + * IN_ATTRIB for atime/nlink/pageio + * Let's do REVAL test too. + */ + if (do_udba && inode) { + mode = (inode->i_mode & S_IFMT); + plus = (inode->i_nlink > 0); + ibs = au_ibtop(inode); + ibe = au_ibbot(inode); + } + + btop = au_dbtop(dentry); + btail = btop; + if (inode && S_ISDIR(inode->i_mode)) + btail = au_dbtaildir(dentry); + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + + AuDbg("b%d, %pd\n", bindex, h_dentry); + h_nfs = !!au_test_nfs(h_dentry->d_sb); + spin_lock(&h_dentry->d_lock); + h_name = &h_dentry->d_name; + if (unlikely(do_udba + && !is_root + && ((!h_nfs + && (unhashed != !!d_unhashed(h_dentry) + || (!tmpfile && !dirren + && !au_qstreq(name, h_name)) + )) + || (h_nfs + && !(flags & LOOKUP_OPEN) + && (h_dentry->d_flags + & DCACHE_NFSFS_RENAMED))) + )) { + int h_unhashed; + + h_unhashed = d_unhashed(h_dentry); + spin_unlock(&h_dentry->d_lock); + AuDbg("unhash 0x%x 0x%x, %pd %pd\n", + unhashed, h_unhashed, dentry, h_dentry); + goto err; + } + spin_unlock(&h_dentry->d_lock); + + err = au_do_h_d_reval(h_dentry, flags, dentry, bindex); + if (unlikely(err)) + /* do not goto err, to keep the errno */ + break; + + /* todo: plink too? */ + if (!do_udba) + continue; + + /* UDBA tests */ + if (unlikely(!!inode != d_is_positive(h_dentry))) + goto err; + + h_inode = NULL; + if (d_is_positive(h_dentry)) + h_inode = d_inode(h_dentry); + h_plus = plus; + h_mode = mode; + h_cached_inode = h_inode; + if (h_inode) { + h_mode = (h_inode->i_mode & S_IFMT); + h_plus = (h_inode->i_nlink > 0); + } + if (inode && ibs <= bindex && bindex <= ibe) + h_cached_inode = au_h_iptr(inode, bindex); + + if (!h_nfs) { + if (unlikely(plus != h_plus && !tmpfile)) + goto err; + } else { + if (unlikely(!(h_dentry->d_flags & DCACHE_NFSFS_RENAMED) + && !is_root + && !IS_ROOT(h_dentry) + && unhashed != d_unhashed(h_dentry))) + goto err; + } + if (unlikely(mode != h_mode + || h_cached_inode != h_inode)) + goto err; + continue; + +err: + err = -EINVAL; + break; + } + + AuTraceErr(err); + return err; +} + +/* todo: consolidate with do_refresh() and au_reval_for_attr() */ +static int simple_reval_dpath(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *parent; + + if (!au_digen_test(dentry, sigen)) + return 0; + + parent = dget_parent(dentry); + di_read_lock_parent(parent, AuLock_IR); + AuDebugOn(au_digen_test(parent, sigen)); + au_dbg_verify_gen(parent, sigen); + err = au_refresh_dentry(dentry, parent); + di_read_unlock(parent, AuLock_IR); + dput(parent); + AuTraceErr(err); + return err; +} + +int au_reval_dpath(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *d, *parent; + + if (!au_ftest_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR)) + return simple_reval_dpath(dentry, sigen); + + /* slow loop, keep it simple and stupid */ + /* cf: au_cpup_dirs() */ + err = 0; + parent = NULL; + while (au_digen_test(dentry, sigen)) { + d = dentry; + while (1) { + dput(parent); + parent = dget_parent(d); + if (!au_digen_test(parent, sigen)) + break; + d = parent; + } + + if (d != dentry) + di_write_lock_child2(d); + + /* someone might update our dentry while we were sleeping */ + if (au_digen_test(d, sigen)) { + /* + * todo: consolidate with simple_reval_dpath(), + * do_refresh() and au_reval_for_attr(). + */ + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(d, parent); + di_read_unlock(parent, AuLock_IR); + } + + if (d != dentry) + di_write_unlock(d); + dput(parent); + if (unlikely(err)) + break; + } + + return err; +} + +/* + * if valid returns 1, otherwise 0. + */ +static int aufs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + int valid, err; + unsigned int sigen; + unsigned char do_udba, dirren; + struct super_block *sb; + struct inode *inode; + + /* todo: support rcu-walk? */ + if (flags & LOOKUP_RCU) + return -ECHILD; + + valid = 0; + if (unlikely(!au_di(dentry))) + goto out; + + valid = 1; + sb = dentry->d_sb; + /* + * todo: very ugly + * i_mutex of parent dir may be held, + * but we should not return 'invalid' due to busy. + */ + err = aufs_read_lock(dentry, AuLock_FLUSH | AuLock_DW | AuLock_NOPLM); + if (unlikely(err)) { + valid = err; + AuTraceErr(err); + goto out; + } + inode = NULL; + if (d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (unlikely(inode && au_is_bad_inode(inode))) { + err = -EINVAL; + AuTraceErr(err); + goto out_dgrade; + } + if (unlikely(au_dbrange_test(dentry))) { + err = -EINVAL; + AuTraceErr(err); + goto out_dgrade; + } + + sigen = au_sigen(sb); + if (au_digen_test(dentry, sigen)) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_dpath(dentry, sigen); + if (unlikely(err)) { + AuTraceErr(err); + goto out_dgrade; + } + } + di_downgrade_lock(dentry, AuLock_IR); + + err = -EINVAL; + if (!(flags & (LOOKUP_OPEN | LOOKUP_EMPTY)) + && inode + && !(inode->i_state && I_LINKABLE) + && (IS_DEADDIR(inode) || !inode->i_nlink)) { + AuTraceErr(err); + goto out_inval; + } + + do_udba = !au_opt_test(au_mntflags(sb), UDBA_NONE); + if (do_udba && inode) { + aufs_bindex_t btop = au_ibtop(inode); + struct inode *h_inode; + + if (btop >= 0) { + h_inode = au_h_iptr(inode, btop); + if (h_inode && au_test_higen(inode, h_inode)) { + AuTraceErr(err); + goto out_inval; + } + } + } + + dirren = !!au_opt_test(au_mntflags(sb), DIRREN); + err = h_d_revalidate(dentry, inode, flags, do_udba, dirren); + if (unlikely(!err && do_udba && au_dbtop(dentry) < 0)) { + err = -EIO; + AuDbg("both of real entry and whiteout found, %p, err %d\n", + dentry, err); + } + goto out_inval; + +out_dgrade: + di_downgrade_lock(dentry, AuLock_IR); +out_inval: + aufs_read_unlock(dentry, AuLock_IR); + AuTraceErr(err); + valid = !err; +out: + if (!valid) { + AuDbg("%pd invalid, %d\n", dentry, valid); + d_drop(dentry); + } + return valid; +} + +static void aufs_d_release(struct dentry *dentry) +{ + if (au_di(dentry)) { + au_di_fin(dentry); + au_hn_di_reinit(dentry); + } +} + +const struct dentry_operations aufs_dop = { + .d_revalidate = aufs_d_revalidate, + .d_weak_revalidate = aufs_d_revalidate, + .d_release = aufs_d_release +}; + +/* aufs_dop without d_revalidate */ +const struct dentry_operations aufs_dop_noreval = { + .d_release = aufs_d_release +}; diff --git a/fs/aufs/dentry.h b/fs/aufs/dentry.h new file mode 100644 index 0000000000000..3b443fc7eef0a --- /dev/null +++ b/fs/aufs/dentry.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * lookup and dentry operations + */ + +#ifndef __AUFS_DENTRY_H__ +#define __AUFS_DENTRY_H__ + +#ifdef __KERNEL__ + +#include +#include "dirren.h" +#include "rwsem.h" + +struct au_hdentry { + struct dentry *hd_dentry; + aufs_bindex_t hd_id; +}; + +struct au_dinfo { + atomic_t di_generation; + + struct au_rwsem di_rwsem; + aufs_bindex_t di_btop, di_bbot, di_bwh, di_bdiropq; + unsigned char di_tmpfile; /* to allow the different name */ + struct au_hdentry *di_hdentry; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* flags for au_lkup_dentry() */ +#define AuLkup_ALLOW_NEG 1 +#define AuLkup_IGNORE_PERM (1 << 1) +#define AuLkup_DIRREN (1 << 2) +#define au_ftest_lkup(flags, name) ((flags) & AuLkup_##name) +#define au_fset_lkup(flags, name) \ + do { (flags) |= AuLkup_##name; } while (0) +#define au_fclr_lkup(flags, name) \ + do { (flags) &= ~AuLkup_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuLkup_DIRREN +#define AuLkup_DIRREN 0 +#endif + +struct au_do_lookup_args { + unsigned int flags; + mode_t type; + struct qstr whname, *name; + struct au_dr_lookup dirren; +}; + +/* ---------------------------------------------------------------------- */ + +/* dentry.c */ +extern const struct dentry_operations aufs_dop, aufs_dop_noreval; +struct au_branch; +struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent); +int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir, + struct dentry *h_parent, struct au_branch *br); + +int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop, + unsigned int flags); +int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh); +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent); +int au_reval_dpath(struct dentry *dentry, unsigned int sigen); +void au_refresh_dop(struct dentry *dentry, int force_reval); + +/* dinfo.c */ +void au_di_init_once(void *_di); +struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc); +void au_di_free(struct au_dinfo *dinfo); +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b); +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src); +int au_di_init(struct dentry *dentry); +void au_di_fin(struct dentry *dentry); +int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink); + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc); +void di_read_unlock(struct dentry *d, int flags); +void di_downgrade_lock(struct dentry *d, int flags); +void di_write_lock(struct dentry *d, unsigned int lsc); +void di_write_unlock(struct dentry *d); +void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir); +void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir); +void di_write_unlock2(struct dentry *d1, struct dentry *d2); + +struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex); +struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex); +aufs_bindex_t au_dbtail(struct dentry *dentry); +aufs_bindex_t au_dbtaildir(struct dentry *dentry); + +void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_dentry); +int au_digen_test(struct dentry *dentry, unsigned int sigen); +int au_dbrange_test(struct dentry *dentry); +void au_update_digen(struct dentry *dentry); +void au_update_dbrange(struct dentry *dentry, int do_put_zero); +void au_update_dbtop(struct dentry *dentry); +void au_update_dbbot(struct dentry *dentry); +int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry); + +/* ---------------------------------------------------------------------- */ + +static inline struct au_dinfo *au_di(struct dentry *dentry) +{ + return dentry->d_fsdata; +} + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for dinfo */ +enum { + AuLsc_DI_CHILD, /* child first */ + AuLsc_DI_CHILD2, /* rename(2), link(2), and cpup at hnotify */ + AuLsc_DI_CHILD3, /* copyup dirs */ + AuLsc_DI_PARENT, + AuLsc_DI_PARENT2, + AuLsc_DI_PARENT3, + AuLsc_DI_TMP /* temp for replacing dinfo */ +}; + +/* + * di_read_lock_child, di_write_lock_child, + * di_read_lock_child2, di_write_lock_child2, + * di_read_lock_child3, di_write_lock_child3, + * di_read_lock_parent, di_write_lock_parent, + * di_read_lock_parent2, di_write_lock_parent2, + * di_read_lock_parent3, di_write_lock_parent3, + */ +#define AuReadLockFunc(name, lsc) \ +static inline void di_read_lock_##name(struct dentry *d, int flags) \ +{ di_read_lock(d, flags, AuLsc_DI_##lsc); } + +#define AuWriteLockFunc(name, lsc) \ +static inline void di_write_lock_##name(struct dentry *d) \ +{ di_write_lock(d, AuLsc_DI_##lsc); } + +#define AuRWLockFuncs(name, lsc) \ + AuReadLockFunc(name, lsc) \ + AuWriteLockFunc(name, lsc) + +AuRWLockFuncs(child, CHILD); +AuRWLockFuncs(child2, CHILD2); +AuRWLockFuncs(child3, CHILD3); +AuRWLockFuncs(parent, PARENT); +AuRWLockFuncs(parent2, PARENT2); +AuRWLockFuncs(parent3, PARENT3); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define DiMustNoWaiters(d) AuRwMustNoWaiters(&au_di(d)->di_rwsem) +#define DiMustAnyLock(d) AuRwMustAnyLock(&au_di(d)->di_rwsem) +#define DiMustWriteLock(d) AuRwMustWriteLock(&au_di(d)->di_rwsem) + +/* ---------------------------------------------------------------------- */ + +/* todo: memory barrier? */ +static inline unsigned int au_digen(struct dentry *d) +{ + return atomic_read(&au_di(d)->di_generation); +} + +static inline void au_h_dentry_init(struct au_hdentry *hdentry) +{ + hdentry->hd_dentry = NULL; +} + +static inline struct au_hdentry *au_hdentry(struct au_dinfo *di, + aufs_bindex_t bindex) +{ + return di->di_hdentry + bindex; +} + +static inline void au_hdput(struct au_hdentry *hd) +{ + if (hd) + dput(hd->hd_dentry); +} + +static inline aufs_bindex_t au_dbtop(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_btop; +} + +static inline aufs_bindex_t au_dbbot(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bbot; +} + +static inline aufs_bindex_t au_dbwh(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bwh; +} + +static inline aufs_bindex_t au_dbdiropq(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bdiropq; +} + +/* todo: hard/soft set? */ +static inline void au_set_dbtop(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_btop = bindex; +} + +static inline void au_set_dbbot(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_bbot = bindex; +} + +static inline void au_set_dbwh(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + /* dbwh can be outside of btop - bbot range */ + au_di(dentry)->di_bwh = bindex; +} + +static inline void au_set_dbdiropq(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_bdiropq = bindex; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_HNOTIFY +static inline void au_digen_dec(struct dentry *d) +{ + atomic_dec(&au_di(d)->di_generation); +} + +static inline void au_hn_di_reinit(struct dentry *dentry) +{ + dentry->d_fsdata = NULL; +} +#else +AuStubVoid(au_hn_di_reinit, struct dentry *dentry __maybe_unused) +#endif /* CONFIG_AUFS_HNOTIFY */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DENTRY_H__ */ diff --git a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c new file mode 100644 index 0000000000000..86140548f0463 --- /dev/null +++ b/fs/aufs/dinfo.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dentry private data + */ + +#include "aufs.h" + +void au_di_init_once(void *_dinfo) +{ + struct au_dinfo *dinfo = _dinfo; + + au_rw_init(&dinfo->di_rwsem); +} + +struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc) +{ + struct au_dinfo *dinfo; + int nbr, i; + + dinfo = au_cache_alloc_dinfo(); + if (unlikely(!dinfo)) + goto out; + + nbr = au_sbbot(sb) + 1; + if (nbr <= 0) + nbr = 1; + dinfo->di_hdentry = kcalloc(nbr, sizeof(*dinfo->di_hdentry), GFP_NOFS); + if (dinfo->di_hdentry) { + au_rw_write_lock_nested(&dinfo->di_rwsem, lsc); + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + dinfo->di_bwh = -1; + dinfo->di_bdiropq = -1; + dinfo->di_tmpfile = 0; + for (i = 0; i < nbr; i++) + dinfo->di_hdentry[i].hd_id = -1; + goto out; + } + + au_cache_free_dinfo(dinfo); + dinfo = NULL; + +out: + return dinfo; +} + +void au_di_free(struct au_dinfo *dinfo) +{ + struct au_hdentry *p; + aufs_bindex_t bbot, bindex; + + /* dentry may not be revalidated */ + bindex = dinfo->di_btop; + if (bindex >= 0) { + bbot = dinfo->di_bbot; + p = au_hdentry(dinfo, bindex); + while (bindex++ <= bbot) + au_hdput(p++); + } + au_kfree_try_rcu(dinfo->di_hdentry); + au_cache_free_dinfo(dinfo); +} + +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b) +{ + struct au_hdentry *p; + aufs_bindex_t bi; + + AuRwMustWriteLock(&a->di_rwsem); + AuRwMustWriteLock(&b->di_rwsem); + +#define DiSwap(v, name) \ + do { \ + v = a->di_##name; \ + a->di_##name = b->di_##name; \ + b->di_##name = v; \ + } while (0) + + DiSwap(p, hdentry); + DiSwap(bi, btop); + DiSwap(bi, bbot); + DiSwap(bi, bwh); + DiSwap(bi, bdiropq); + /* smp_mb(); */ + +#undef DiSwap +} + +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src) +{ + AuRwMustWriteLock(&dst->di_rwsem); + AuRwMustWriteLock(&src->di_rwsem); + + dst->di_btop = src->di_btop; + dst->di_bbot = src->di_bbot; + dst->di_bwh = src->di_bwh; + dst->di_bdiropq = src->di_bdiropq; + /* smp_mb(); */ +} + +int au_di_init(struct dentry *dentry) +{ + int err; + struct super_block *sb; + struct au_dinfo *dinfo; + + err = 0; + sb = dentry->d_sb; + dinfo = au_di_alloc(sb, AuLsc_DI_CHILD); + if (dinfo) { + atomic_set(&dinfo->di_generation, au_sigen(sb)); + /* smp_mb(); */ /* atomic_set */ + dentry->d_fsdata = dinfo; + } else + err = -ENOMEM; + + return err; +} + +void au_di_fin(struct dentry *dentry) +{ + struct au_dinfo *dinfo; + + dinfo = au_di(dentry); + AuRwDestroy(&dinfo->di_rwsem); + au_di_free(dinfo); +} + +int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_hdentry *hdp; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + err = -ENOMEM; + sz = sizeof(*hdp) * (dinfo->di_bbot + 1); + if (!sz) + sz = sizeof(*hdp); + hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS, + may_shrink); + if (hdp) { + dinfo->di_hdentry = hdp; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void do_ii_write_lock(struct inode *inode, unsigned int lsc) +{ + switch (lsc) { + case AuLsc_DI_CHILD: + ii_write_lock_child(inode); + break; + case AuLsc_DI_CHILD2: + ii_write_lock_child2(inode); + break; + case AuLsc_DI_CHILD3: + ii_write_lock_child3(inode); + break; + case AuLsc_DI_PARENT: + ii_write_lock_parent(inode); + break; + case AuLsc_DI_PARENT2: + ii_write_lock_parent2(inode); + break; + case AuLsc_DI_PARENT3: + ii_write_lock_parent3(inode); + break; + default: + BUG(); + } +} + +static void do_ii_read_lock(struct inode *inode, unsigned int lsc) +{ + switch (lsc) { + case AuLsc_DI_CHILD: + ii_read_lock_child(inode); + break; + case AuLsc_DI_CHILD2: + ii_read_lock_child2(inode); + break; + case AuLsc_DI_CHILD3: + ii_read_lock_child3(inode); + break; + case AuLsc_DI_PARENT: + ii_read_lock_parent(inode); + break; + case AuLsc_DI_PARENT2: + ii_read_lock_parent2(inode); + break; + case AuLsc_DI_PARENT3: + ii_read_lock_parent3(inode); + break; + default: + BUG(); + } +} + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc) +{ + struct inode *inode; + + au_rw_read_lock_nested(&au_di(d)->di_rwsem, lsc); + if (d_really_is_positive(d)) { + inode = d_inode(d); + if (au_ftest_lock(flags, IW)) + do_ii_write_lock(inode, lsc); + else if (au_ftest_lock(flags, IR)) + do_ii_read_lock(inode, lsc); + } +} + +void di_read_unlock(struct dentry *d, int flags) +{ + struct inode *inode; + + if (d_really_is_positive(d)) { + inode = d_inode(d); + if (au_ftest_lock(flags, IW)) { + au_dbg_verify_dinode(d); + ii_write_unlock(inode); + } else if (au_ftest_lock(flags, IR)) { + au_dbg_verify_dinode(d); + ii_read_unlock(inode); + } + } + au_rw_read_unlock(&au_di(d)->di_rwsem); +} + +void di_downgrade_lock(struct dentry *d, int flags) +{ + if (d_really_is_positive(d) && au_ftest_lock(flags, IR)) + ii_downgrade_lock(d_inode(d)); + au_rw_dgrade_lock(&au_di(d)->di_rwsem); +} + +void di_write_lock(struct dentry *d, unsigned int lsc) +{ + au_rw_write_lock_nested(&au_di(d)->di_rwsem, lsc); + if (d_really_is_positive(d)) + do_ii_write_lock(d_inode(d), lsc); +} + +void di_write_unlock(struct dentry *d) +{ + au_dbg_verify_dinode(d); + if (d_really_is_positive(d)) + ii_write_unlock(d_inode(d)); + au_rw_write_unlock(&au_di(d)->di_rwsem); +} + +void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir) +{ + AuDebugOn(d1 == d2 + || d_inode(d1) == d_inode(d2) + || d1->d_sb != d2->d_sb); + + if ((isdir && au_test_subdir(d1, d2)) + || d1 < d2) { + di_write_lock_child(d1); + di_write_lock_child2(d2); + } else { + di_write_lock_child(d2); + di_write_lock_child2(d1); + } +} + +void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir) +{ + AuDebugOn(d1 == d2 + || d_inode(d1) == d_inode(d2) + || d1->d_sb != d2->d_sb); + + if ((isdir && au_test_subdir(d1, d2)) + || d1 < d2) { + di_write_lock_parent(d1); + di_write_lock_parent2(d2); + } else { + di_write_lock_parent(d2); + di_write_lock_parent2(d1); + } +} + +void di_write_unlock2(struct dentry *d1, struct dentry *d2) +{ + di_write_unlock(d1); + if (d_inode(d1) == d_inode(d2)) + au_rw_write_unlock(&au_di(d2)->di_rwsem); + else + di_write_unlock(d2); +} + +/* ---------------------------------------------------------------------- */ + +struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex) +{ + struct dentry *d; + + DiMustAnyLock(dentry); + + if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry)) + return NULL; + AuDebugOn(bindex < 0); + d = au_hdentry(au_di(dentry), bindex)->hd_dentry; + AuDebugOn(d && au_dcount(d) <= 0); + return d; +} + +/* + * extended version of au_h_dptr(). + * returns a hashed and positive (or linkable) h_dentry in bindex, NULL, or + * error. + */ +struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex) +{ + struct dentry *h_dentry; + struct inode *inode, *h_inode; + + AuDebugOn(d_really_is_negative(dentry)); + + h_dentry = NULL; + if (au_dbtop(dentry) <= bindex + && bindex <= au_dbbot(dentry)) + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && !au_d_linkable(h_dentry)) { + dget(h_dentry); + goto out; /* success */ + } + + inode = d_inode(dentry); + AuDebugOn(bindex < au_ibtop(inode)); + AuDebugOn(au_ibbot(inode) < bindex); + h_inode = au_h_iptr(inode, bindex); + h_dentry = d_find_alias(h_inode); + if (h_dentry) { + if (!IS_ERR(h_dentry)) { + if (!au_d_linkable(h_dentry)) + goto out; /* success */ + dput(h_dentry); + } else + goto out; + } + + if (au_opt_test(au_mntflags(dentry->d_sb), PLINK)) { + h_dentry = au_plink_lkup(inode, bindex); + AuDebugOn(!h_dentry); + if (!IS_ERR(h_dentry)) { + if (!au_d_hashed_positive(h_dentry)) + goto out; /* success */ + dput(h_dentry); + h_dentry = NULL; + } + } + +out: + AuDbgDentry(h_dentry); + return h_dentry; +} + +aufs_bindex_t au_dbtail(struct dentry *dentry) +{ + aufs_bindex_t bbot, bwh; + + bbot = au_dbbot(dentry); + if (0 <= bbot) { + bwh = au_dbwh(dentry); + if (!bwh) + return bwh; + if (0 < bwh && bwh < bbot) + return bwh - 1; + } + return bbot; +} + +aufs_bindex_t au_dbtaildir(struct dentry *dentry) +{ + aufs_bindex_t bbot, bopq; + + bbot = au_dbtail(dentry); + if (0 <= bbot) { + bopq = au_dbdiropq(dentry); + if (0 <= bopq && bopq < bbot) + bbot = bopq; + } + return bbot; +} + +/* ---------------------------------------------------------------------- */ + +void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_dentry) +{ + struct au_dinfo *dinfo; + struct au_hdentry *hd; + struct au_branch *br; + + DiMustWriteLock(dentry); + + dinfo = au_di(dentry); + hd = au_hdentry(dinfo, bindex); + au_hdput(hd); + hd->hd_dentry = h_dentry; + if (h_dentry) { + br = au_sbr(dentry->d_sb, bindex); + hd->hd_id = br->br_id; + } +} + +int au_dbrange_test(struct dentry *dentry) +{ + int err; + aufs_bindex_t btop, bbot; + + err = 0; + btop = au_dbtop(dentry); + bbot = au_dbbot(dentry); + if (btop >= 0) + AuDebugOn(bbot < 0 && btop > bbot); + else { + err = -EIO; + AuDebugOn(bbot >= 0); + } + + return err; +} + +int au_digen_test(struct dentry *dentry, unsigned int sigen) +{ + int err; + + err = 0; + if (unlikely(au_digen(dentry) != sigen + || au_iigen_test(d_inode(dentry), sigen))) + err = -EIO; + + return err; +} + +void au_update_digen(struct dentry *dentry) +{ + atomic_set(&au_di(dentry)->di_generation, au_sigen(dentry->d_sb)); + /* smp_mb(); */ /* atomic_set */ +} + +void au_update_dbrange(struct dentry *dentry, int do_put_zero) +{ + struct au_dinfo *dinfo; + struct dentry *h_d; + struct au_hdentry *hdp; + aufs_bindex_t bindex, bbot; + + DiMustWriteLock(dentry); + + dinfo = au_di(dentry); + if (!dinfo || dinfo->di_btop < 0) + return; + + if (do_put_zero) { + bbot = dinfo->di_bbot; + bindex = dinfo->di_btop; + hdp = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, hdp++) { + h_d = hdp->hd_dentry; + if (h_d && d_is_negative(h_d)) + au_set_h_dptr(dentry, bindex, NULL); + } + } + + dinfo->di_btop = 0; + hdp = au_hdentry(dinfo, dinfo->di_btop); + for (; dinfo->di_btop <= dinfo->di_bbot; dinfo->di_btop++, hdp++) + if (hdp->hd_dentry) + break; + if (dinfo->di_btop > dinfo->di_bbot) { + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + return; + } + + hdp = au_hdentry(dinfo, dinfo->di_bbot); + for (; dinfo->di_bbot >= 0; dinfo->di_bbot--, hdp--) + if (hdp->hd_dentry) + break; + AuDebugOn(dinfo->di_btop > dinfo->di_bbot || dinfo->di_bbot < 0); +} + +void au_update_dbtop(struct dentry *dentry) +{ + aufs_bindex_t bindex, bbot; + struct dentry *h_dentry; + + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + if (d_is_positive(h_dentry)) { + au_set_dbtop(dentry, bindex); + return; + } + au_set_h_dptr(dentry, bindex, NULL); + } +} + +void au_update_dbbot(struct dentry *dentry) +{ + aufs_bindex_t bindex, btop; + struct dentry *h_dentry; + + btop = au_dbtop(dentry); + for (bindex = au_dbbot(dentry); bindex >= btop; bindex--) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + if (d_is_positive(h_dentry)) { + au_set_dbbot(dentry, bindex); + return; + } + au_set_h_dptr(dentry, bindex, NULL); + } +} + +int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry) +{ + aufs_bindex_t bindex, bbot; + + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) + if (au_h_dptr(dentry, bindex) == h_dentry) + return bindex; + return -1; +} diff --git a/fs/aufs/dir.c b/fs/aufs/dir.c new file mode 100644 index 0000000000000..ab117dfd41893 --- /dev/null +++ b/fs/aufs/dir.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * directory operations + */ + +#include +#include "aufs.h" + +void au_add_nlink(struct inode *dir, struct inode *h_dir) +{ + unsigned int nlink; + + AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode)); + + nlink = dir->i_nlink; + nlink += h_dir->i_nlink - 2; + if (h_dir->i_nlink < 2) + nlink += 2; + smp_mb(); /* for i_nlink */ + /* 0 can happen in revaliding */ + set_nlink(dir, nlink); +} + +void au_sub_nlink(struct inode *dir, struct inode *h_dir) +{ + unsigned int nlink; + + AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode)); + + nlink = dir->i_nlink; + nlink -= h_dir->i_nlink - 2; + if (h_dir->i_nlink < 2) + nlink -= 2; + smp_mb(); /* for i_nlink */ + /* nlink == 0 means the branch-fs is broken */ + set_nlink(dir, nlink); +} + +loff_t au_dir_size(struct file *file, struct dentry *dentry) +{ + loff_t sz; + aufs_bindex_t bindex, bbot; + struct file *h_file; + struct dentry *h_dentry; + + sz = 0; + if (file) { + AuDebugOn(!d_is_dir(file->f_path.dentry)); + + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); + bindex <= bbot && sz < KMALLOC_MAX_SIZE; + bindex++) { + h_file = au_hf_dir(file, bindex); + if (h_file && file_inode(h_file)) + sz += vfsub_f_size_read(h_file); + } + } else { + AuDebugOn(!dentry); + AuDebugOn(!d_is_dir(dentry)); + + bbot = au_dbtaildir(dentry); + for (bindex = au_dbtop(dentry); + bindex <= bbot && sz < KMALLOC_MAX_SIZE; + bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) + sz += i_size_read(d_inode(h_dentry)); + } + } + if (sz < KMALLOC_MAX_SIZE) + sz = roundup_pow_of_two(sz); + if (sz > KMALLOC_MAX_SIZE) + sz = KMALLOC_MAX_SIZE; + else if (sz < NAME_MAX) { + BUILD_BUG_ON(AUFS_RDBLK_DEF < NAME_MAX); + sz = AUFS_RDBLK_DEF; + } + return sz; +} + +struct au_dir_ts_arg { + struct dentry *dentry; + aufs_bindex_t brid; +}; + +static void au_do_dir_ts(void *arg) +{ + struct au_dir_ts_arg *a = arg; + struct au_dtime dt; + struct path h_path; + struct inode *dir, *h_dir; + struct super_block *sb; + struct au_branch *br; + struct au_hinode *hdir; + int err; + aufs_bindex_t btop, bindex; + + sb = a->dentry->d_sb; + if (d_really_is_negative(a->dentry)) + goto out; + /* no dir->i_mutex lock */ + aufs_read_lock(a->dentry, AuLock_DW); /* noflush */ + + dir = d_inode(a->dentry); + btop = au_ibtop(dir); + bindex = au_br_index(sb, a->brid); + if (bindex < btop) + goto out_unlock; + + br = au_sbr(sb, bindex); + h_path.dentry = au_h_dptr(a->dentry, bindex); + if (!h_path.dentry) + goto out_unlock; + h_path.mnt = au_br_mnt(br); + au_dtime_store(&dt, a->dentry, &h_path); + + br = au_sbr(sb, btop); + if (!au_br_writable(br->br_perm)) + goto out_unlock; + h_path.dentry = au_h_dptr(a->dentry, btop); + h_path.mnt = au_br_mnt(br); + err = vfsub_mnt_want_write(h_path.mnt); + if (err) + goto out_unlock; + hdir = au_hi(dir, btop); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + h_dir = au_h_iptr(dir, btop); + if (h_dir->i_nlink + && timespec_compare(&h_dir->i_mtime, &dt.dt_mtime) < 0) { + dt.dt_h_path = h_path; + au_dtime_revert(&dt); + } + au_hn_inode_unlock(hdir); + vfsub_mnt_drop_write(h_path.mnt); + au_cpup_attr_timesizes(dir); + +out_unlock: + aufs_read_unlock(a->dentry, AuLock_DW); +out: + dput(a->dentry); + au_nwt_done(&au_sbi(sb)->si_nowait); + au_kfree_try_rcu(arg); +} + +void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) +{ + int perm, wkq_err; + aufs_bindex_t btop; + struct au_dir_ts_arg *arg; + struct dentry *dentry; + struct super_block *sb; + + IMustLock(dir); + + dentry = d_find_any_alias(dir); + AuDebugOn(!dentry); + sb = dentry->d_sb; + btop = au_ibtop(dir); + if (btop == bindex) { + au_cpup_attr_timesizes(dir); + goto out; + } + + perm = au_sbr_perm(sb, btop); + if (!au_br_writable(perm)) + goto out; + + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (!arg) + goto out; + + arg->dentry = dget(dentry); /* will be dput-ted by au_do_dir_ts() */ + arg->brid = au_sbr_id(sb, bindex); + wkq_err = au_wkq_nowait(au_do_dir_ts, arg, sb, /*flags*/0); + if (unlikely(wkq_err)) { + pr_err("wkq %d\n", wkq_err); + dput(dentry); + au_kfree_try_rcu(arg); + } + +out: + dput(dentry); +} + +/* ---------------------------------------------------------------------- */ + +static int reopen_dir(struct file *file) +{ + int err; + unsigned int flags; + aufs_bindex_t bindex, btail, btop; + struct dentry *dentry, *h_dentry; + struct file *h_file; + + /* open all lower dirs */ + dentry = file->f_path.dentry; + btop = au_dbtop(dentry); + for (bindex = au_fbtop(file); bindex < btop; bindex++) + au_set_h_fptr(file, bindex, NULL); + au_set_fbtop(file, btop); + + btail = au_dbtaildir(dentry); + for (bindex = au_fbbot_dir(file); btail < bindex; bindex--) + au_set_h_fptr(file, bindex, NULL); + au_set_fbbot_dir(file, btail); + + flags = vfsub_file_flags(file); + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + h_file = au_hf_dir(file, bindex); + if (h_file) + continue; + + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; /* close all? */ + au_set_h_fptr(file, bindex, h_file); + } + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + err = 0; + +out: + return err; +} + +static int do_open_dir(struct file *file, int flags, struct file *h_file) +{ + int err; + aufs_bindex_t bindex, btail; + struct dentry *dentry, *h_dentry; + struct vfsmount *mnt; + + FiMustWriteLock(file); + AuDebugOn(h_file); + + err = 0; + mnt = file->f_path.mnt; + dentry = file->f_path.dentry; + file->f_version = d_inode(dentry)->i_version; + bindex = au_dbtop(dentry); + au_set_fbtop(file, bindex); + btail = au_dbtaildir(dentry); + au_set_fbbot_dir(file, btail); + for (; !err && bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + + err = vfsub_test_mntns(mnt, h_dentry->d_sb); + if (unlikely(err)) + break; + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + if (IS_ERR(h_file)) { + err = PTR_ERR(h_file); + break; + } + au_set_h_fptr(file, bindex, h_file); + } + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + if (!err) + return 0; /* success */ + + /* close all */ + for (bindex = au_fbtop(file); bindex <= btail; bindex++) + au_set_h_fptr(file, bindex, NULL); + au_set_fbtop(file, -1); + au_set_fbbot_dir(file, -1); + + return err; +} + +static int aufs_open_dir(struct inode *inode __maybe_unused, + struct file *file) +{ + int err; + struct super_block *sb; + struct au_fidir *fidir; + + err = -ENOMEM; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + fidir = au_fidir_alloc(sb); + if (fidir) { + struct au_do_open_args args = { + .open = do_open_dir, + .fidir = fidir + }; + err = au_do_open(file, &args); + if (unlikely(err)) + au_kfree_rcu(fidir); + } + si_read_unlock(sb); + return err; +} + +static int aufs_release_dir(struct inode *inode __maybe_unused, + struct file *file) +{ + struct au_vdir *vdir_cache; + struct au_finfo *finfo; + struct au_fidir *fidir; + struct au_hfile *hf; + aufs_bindex_t bindex, bbot; + + finfo = au_fi(file); + fidir = finfo->fi_hdir; + if (fidir) { + au_hbl_del(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + vdir_cache = fidir->fd_vdir_cache; /* lock-free */ + if (vdir_cache) + au_vdir_free(vdir_cache); + + bindex = finfo->fi_btop; + if (bindex >= 0) { + hf = fidir->fd_hfile + bindex; + /* + * calls fput() instead of filp_close(), + * since no dnotify or lock for the lower file. + */ + bbot = fidir->fd_bbot; + for (; bindex <= bbot; bindex++, hf++) + if (hf->hf_file) + au_hfput(hf, /*execed*/0); + } + au_kfree_rcu(fidir); + finfo->fi_hdir = NULL; + } + au_finfo_fin(file); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_flush_dir(struct file *file, fl_owner_t id) +{ + int err; + aufs_bindex_t bindex, bbot; + struct file *h_file; + + err = 0; + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) { + h_file = au_hf_dir(file, bindex); + if (h_file) + err = vfsub_flush(h_file, id); + } + return err; +} + +static int aufs_flush_dir(struct file *file, fl_owner_t id) +{ + return au_do_flush(file, id, au_do_flush_dir); +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_fsync_dir_no_file(struct dentry *dentry, int datasync) +{ + int err; + aufs_bindex_t bbot, bindex; + struct inode *inode; + struct super_block *sb; + + err = 0; + sb = dentry->d_sb; + inode = d_inode(dentry); + IMustLock(inode); + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); !err && bindex <= bbot; bindex++) { + struct path h_path; + + if (au_test_ro(sb, bindex, inode)) + continue; + h_path.dentry = au_h_dptr(dentry, bindex); + if (!h_path.dentry) + continue; + + h_path.mnt = au_sbr_mnt(sb, bindex); + err = vfsub_fsync(NULL, &h_path, datasync); + } + + return err; +} + +static int au_do_fsync_dir(struct file *file, int datasync) +{ + int err; + aufs_bindex_t bbot, bindex; + struct file *h_file; + struct super_block *sb; + struct inode *inode; + + err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0); + if (unlikely(err)) + goto out; + + inode = file_inode(file); + sb = inode->i_sb; + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) { + h_file = au_hf_dir(file, bindex); + if (!h_file || au_test_ro(sb, bindex, inode)) + continue; + + err = vfsub_fsync(h_file, &h_file->f_path, datasync); + } + +out: + return err; +} + +/* + * @file may be NULL + */ +static int aufs_fsync_dir(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err; + struct dentry *dentry; + struct inode *inode; + struct super_block *sb; + + err = 0; + dentry = file->f_path.dentry; + inode = d_inode(dentry); + inode_lock(inode); + sb = dentry->d_sb; + si_noflush_read_lock(sb); + if (file) + err = au_do_fsync_dir(file, datasync); + else { + di_write_lock_child(dentry); + err = au_do_fsync_dir_no_file(dentry, datasync); + } + au_cpup_attr_timesizes(inode); + di_write_unlock(dentry); + if (file) + fi_write_unlock(file); + + si_read_unlock(sb); + inode_unlock(inode); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_iterate_shared(struct file *file, struct dir_context *ctx) +{ + int err; + struct dentry *dentry; + struct inode *inode, *h_inode; + struct super_block *sb; + + AuDbg("%pD, ctx{%pf, %llu}\n", file, ctx->actor, ctx->pos); + + dentry = file->f_path.dentry; + inode = d_inode(dentry); + IMustLock(inode); + + sb = dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0); + if (unlikely(err)) + goto out; + err = au_alive_dir(dentry); + if (!err) + err = au_vdir_init(file); + di_downgrade_lock(dentry, AuLock_IR); + if (unlikely(err)) + goto out_unlock; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + if (!au_test_nfsd()) { + err = au_vdir_fill_de(file, ctx); + fsstack_copy_attr_atime(inode, h_inode); + } else { + /* + * nfsd filldir may call lookup_one_len(), vfs_getattr(), + * encode_fh() and others. + */ + atomic_inc(&h_inode->i_count); + di_read_unlock(dentry, AuLock_IR); + si_read_unlock(sb); + err = au_vdir_fill_de(file, ctx); + fsstack_copy_attr_atime(inode, h_inode); + fi_write_unlock(file); + iput(h_inode); + + AuTraceErr(err); + return err; + } + +out_unlock: + di_read_unlock(dentry, AuLock_IR); + fi_write_unlock(file); +out: + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +#define AuTestEmpty_WHONLY 1 +#define AuTestEmpty_CALLED (1 << 1) +#define AuTestEmpty_SHWH (1 << 2) +#define au_ftest_testempty(flags, name) ((flags) & AuTestEmpty_##name) +#define au_fset_testempty(flags, name) \ + do { (flags) |= AuTestEmpty_##name; } while (0) +#define au_fclr_testempty(flags, name) \ + do { (flags) &= ~AuTestEmpty_##name; } while (0) + +#ifndef CONFIG_AUFS_SHWH +#undef AuTestEmpty_SHWH +#define AuTestEmpty_SHWH 0 +#endif + +struct test_empty_arg { + struct dir_context ctx; + struct au_nhash *whlist; + unsigned int flags; + int err; + aufs_bindex_t bindex; +}; + +static int test_empty_cb(struct dir_context *ctx, const char *__name, + int namelen, loff_t offset __maybe_unused, u64 ino, + unsigned int d_type) +{ + struct test_empty_arg *arg = container_of(ctx, struct test_empty_arg, + ctx); + char *name = (void *)__name; + + arg->err = 0; + au_fset_testempty(arg->flags, CALLED); + /* smp_mb(); */ + if (name[0] == '.' + && (namelen == 1 || (name[1] == '.' && namelen == 2))) + goto out; /* success */ + + if (namelen <= AUFS_WH_PFX_LEN + || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + if (au_ftest_testempty(arg->flags, WHONLY) + && !au_nhash_test_known_wh(arg->whlist, name, namelen)) + arg->err = -ENOTEMPTY; + goto out; + } + + name += AUFS_WH_PFX_LEN; + namelen -= AUFS_WH_PFX_LEN; + if (!au_nhash_test_known_wh(arg->whlist, name, namelen)) + arg->err = au_nhash_append_wh + (arg->whlist, name, namelen, ino, d_type, arg->bindex, + au_ftest_testempty(arg->flags, SHWH)); + +out: + /* smp_mb(); */ + AuTraceErr(arg->err); + return arg->err; +} + +static int do_test_empty(struct dentry *dentry, struct test_empty_arg *arg) +{ + int err; + struct file *h_file; + struct au_branch *br; + + h_file = au_h_open(dentry, arg->bindex, + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_LARGEFILE, + /*file*/NULL, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = 0; + if (!au_opt_test(au_mntflags(dentry->d_sb), UDBA_NONE) + && !file_inode(h_file)->i_nlink) + goto out_put; + + do { + arg->err = 0; + au_fclr_testempty(arg->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(h_file, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err && au_ftest_testempty(arg->flags, CALLED)); + +out_put: + fput(h_file); + br = au_sbr(dentry->d_sb, arg->bindex); + au_lcnt_dec(&br->br_nfiles); +out: + return err; +} + +struct do_test_empty_args { + int *errp; + struct dentry *dentry; + struct test_empty_arg *arg; +}; + +static void call_do_test_empty(void *args) +{ + struct do_test_empty_args *a = args; + *a->errp = do_test_empty(a->dentry, a->arg); +} + +static int sio_test_empty(struct dentry *dentry, struct test_empty_arg *arg) +{ + int err, wkq_err; + struct dentry *h_dentry; + struct inode *h_inode; + + h_dentry = au_h_dptr(dentry, arg->bindex); + h_inode = d_inode(h_dentry); + /* todo: i_mode changes anytime? */ + vfsub_inode_lock_shared_nested(h_inode, AuLsc_I_CHILD); + err = au_test_h_perm_sio(h_inode, MAY_EXEC | MAY_READ); + inode_unlock_shared(h_inode); + if (!err) + err = do_test_empty(dentry, arg); + else { + struct do_test_empty_args args = { + .errp = &err, + .dentry = dentry, + .arg = arg + }; + unsigned int flags = arg->flags; + + wkq_err = au_wkq_wait(call_do_test_empty, &args); + if (unlikely(wkq_err)) + err = wkq_err; + arg->flags = flags; + } + + return err; +} + +int au_test_empty_lower(struct dentry *dentry) +{ + int err; + unsigned int rdhash; + aufs_bindex_t bindex, btop, btail; + struct au_nhash whlist; + struct test_empty_arg arg = { + .ctx = { + .actor = test_empty_cb + } + }; + int (*test_empty)(struct dentry *dentry, struct test_empty_arg *arg); + + SiMustAnyLock(dentry->d_sb); + + rdhash = au_sbi(dentry->d_sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, dentry)); + err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + + arg.flags = 0; + arg.whlist = &whlist; + btop = au_dbtop(dentry); + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)) + au_fset_testempty(arg.flags, SHWH); + test_empty = do_test_empty; + if (au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1)) + test_empty = sio_test_empty; + arg.bindex = btop; + err = test_empty(dentry, &arg); + if (unlikely(err)) + goto out_whlist; + + au_fset_testempty(arg.flags, WHONLY); + btail = au_dbtaildir(dentry); + for (bindex = btop + 1; !err && bindex <= btail; bindex++) { + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) { + arg.bindex = bindex; + err = test_empty(dentry, &arg); + } + } + +out_whlist: + au_nhash_wh_free(&whlist); +out: + return err; +} + +int au_test_empty(struct dentry *dentry, struct au_nhash *whlist) +{ + int err; + struct test_empty_arg arg = { + .ctx = { + .actor = test_empty_cb + } + }; + aufs_bindex_t bindex, btail; + + err = 0; + arg.whlist = whlist; + arg.flags = AuTestEmpty_WHONLY; + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)) + au_fset_testempty(arg.flags, SHWH); + btail = au_dbtaildir(dentry); + for (bindex = au_dbtop(dentry); !err && bindex <= btail; bindex++) { + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) { + arg.bindex = bindex; + err = sio_test_empty(dentry, &arg); + } + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +const struct file_operations aufs_dir_fop = { + .owner = THIS_MODULE, + .llseek = default_llseek, + .read = generic_read_dir, + .iterate_shared = aufs_iterate_shared, + .unlocked_ioctl = aufs_ioctl_dir, +#ifdef CONFIG_COMPAT + .compat_ioctl = aufs_compat_ioctl_dir, +#endif + .open = aufs_open_dir, + .release = aufs_release_dir, + .flush = aufs_flush_dir, + .fsync = aufs_fsync_dir +}; diff --git a/fs/aufs/dir.h b/fs/aufs/dir.h new file mode 100644 index 0000000000000..ff578fc5eec5f --- /dev/null +++ b/fs/aufs/dir.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * directory operations + */ + +#ifndef __AUFS_DIR_H__ +#define __AUFS_DIR_H__ + +#ifdef __KERNEL__ + +#include + +/* ---------------------------------------------------------------------- */ + +/* need to be faster and smaller */ + +struct au_nhash { + unsigned int nh_num; + struct hlist_head *nh_head; +}; + +struct au_vdir_destr { + unsigned char len; + unsigned char name[0]; +} __packed; + +struct au_vdir_dehstr { + struct hlist_node hash; + struct au_vdir_destr *str; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +struct au_vdir_de { + ino_t de_ino; + unsigned char de_type; + /* caution: packed */ + struct au_vdir_destr de_str; +} __packed; + +struct au_vdir_wh { + struct hlist_node wh_hash; +#ifdef CONFIG_AUFS_SHWH + ino_t wh_ino; + aufs_bindex_t wh_bindex; + unsigned char wh_type; +#else + aufs_bindex_t wh_bindex; +#endif + /* caution: packed */ + struct au_vdir_destr wh_str; +} __packed; + +union au_vdir_deblk_p { + unsigned char *deblk; + struct au_vdir_de *de; +}; + +struct au_vdir { + unsigned char **vd_deblk; + unsigned long vd_nblk; + struct { + unsigned long ul; + union au_vdir_deblk_p p; + } vd_last; + + unsigned long vd_version; + unsigned int vd_deblk_sz; + unsigned long vd_jiffy; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* dir.c */ +extern const struct file_operations aufs_dir_fop; +void au_add_nlink(struct inode *dir, struct inode *h_dir); +void au_sub_nlink(struct inode *dir, struct inode *h_dir); +loff_t au_dir_size(struct file *file, struct dentry *dentry); +void au_dir_ts(struct inode *dir, aufs_bindex_t bsrc); +int au_test_empty_lower(struct dentry *dentry); +int au_test_empty(struct dentry *dentry, struct au_nhash *whlist); + +/* vdir.c */ +unsigned int au_rdhash_est(loff_t sz); +int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp); +void au_nhash_wh_free(struct au_nhash *whlist); +int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt, + int limit); +int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen); +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh); +void au_vdir_free(struct au_vdir *vdir); +int au_vdir_init(struct file *file); +int au_vdir_fill_de(struct file *file, struct dir_context *ctx); + +/* ioctl.c */ +long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg); + +#ifdef CONFIG_AUFS_RDU +/* rdu.c */ +long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_rdu_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg); +#endif +#else +AuStub(long, au_rdu_ioctl, return -EINVAL, struct file *file, + unsigned int cmd, unsigned long arg) +#ifdef CONFIG_COMPAT +AuStub(long, au_rdu_compat_ioctl, return -EINVAL, struct file *file, + unsigned int cmd, unsigned long arg) +#endif +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DIR_H__ */ diff --git a/fs/aufs/dirren.c b/fs/aufs/dirren.c new file mode 100644 index 0000000000000..0c19b9c731f90 --- /dev/null +++ b/fs/aufs/dirren.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * special handling in renaming a directory + * in order to support looking-up the before-renamed name on the lower readonly + * branches + */ + +#include +#include "aufs.h" + +static void au_dr_hino_del(struct au_dr_br *dr, struct au_dr_hino *ent) +{ + int idx; + + idx = au_dr_ihash(ent->dr_h_ino); + au_hbl_del(&ent->dr_hnode, dr->dr_h_ino + idx); +} + +static int au_dr_hino_test_empty(struct au_dr_br *dr) +{ + int ret, i; + struct hlist_bl_head *hbl; + + ret = 1; + for (i = 0; ret && i < AuDirren_NHASH; i++) { + hbl = dr->dr_h_ino + i; + hlist_bl_lock(hbl); + ret &= hlist_bl_empty(hbl); + hlist_bl_unlock(hbl); + } + + return ret; +} + +static struct au_dr_hino *au_dr_hino_find(struct au_dr_br *dr, ino_t ino) +{ + struct au_dr_hino *found, *ent; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + int idx; + + found = NULL; + idx = au_dr_ihash(ino); + hbl = dr->dr_h_ino + idx; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode) + if (ent->dr_h_ino == ino) { + found = ent; + break; + } + hlist_bl_unlock(hbl); + + return found; +} + +int au_dr_hino_test_add(struct au_dr_br *dr, ino_t ino, + struct au_dr_hino *add_ent) +{ + int found, idx; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_dr_hino *ent; + + found = 0; + idx = au_dr_ihash(ino); + hbl = dr->dr_h_ino + idx; +#if 0 + { + struct hlist_bl_node *tmp; + + hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode) + AuDbg("hi%llu\n", (unsigned long long)ent->dr_h_ino); + } +#endif + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode) + if (ent->dr_h_ino == ino) { + found = 1; + break; + } + if (!found && add_ent) + hlist_bl_add_head(&add_ent->dr_hnode, hbl); + hlist_bl_unlock(hbl); + + if (!found && add_ent) + AuDbg("i%llu added\n", (unsigned long long)add_ent->dr_h_ino); + + return found; +} + +void au_dr_hino_free(struct au_dr_br *dr) +{ + int i; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_dr_hino *ent; + + /* SiMustWriteLock(sb); */ + + for (i = 0; i < AuDirren_NHASH; i++) { + hbl = dr->dr_h_ino + i; + /* no spinlock since sbinfo must be write-locked */ + hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode) + au_kfree_rcu(ent); + INIT_HLIST_BL_HEAD(hbl); + } +} + +/* returns the number of inodes or an error */ +static int au_dr_hino_store(struct super_block *sb, struct au_branch *br, + struct file *hinofile) +{ + int err, i; + ssize_t ssz; + loff_t pos, oldsize; + __be64 u64; + struct inode *hinoinode; + struct hlist_bl_head *hbl; + struct hlist_bl_node *n1, *n2; + struct au_dr_hino *ent; + + SiMustWriteLock(sb); + AuDebugOn(!au_br_writable(br->br_perm)); + + hinoinode = file_inode(hinofile); + oldsize = i_size_read(hinoinode); + + err = 0; + pos = 0; + hbl = br->br_dirren.dr_h_ino; + for (i = 0; !err && i < AuDirren_NHASH; i++, hbl++) { + /* no bit-lock since sbinfo must be write-locked */ + hlist_bl_for_each_entry_safe(ent, n1, n2, hbl, dr_hnode) { + AuDbg("hi%llu, %pD2\n", + (unsigned long long)ent->dr_h_ino, hinofile); + u64 = cpu_to_be64(ent->dr_h_ino); + ssz = vfsub_write_k(hinofile, &u64, sizeof(u64), &pos); + if (ssz == sizeof(u64)) + continue; + + /* write error */ + pr_err("ssz %zd, %pD2\n", ssz, hinofile); + err = -ENOSPC; + if (ssz < 0) + err = ssz; + break; + } + } + /* regardless the error */ + if (pos < oldsize) { + err = vfsub_trunc(&hinofile->f_path, pos, /*attr*/0, hinofile); + AuTraceErr(err); + } + + AuTraceErr(err); + return err; +} + +static int au_dr_hino_load(struct au_dr_br *dr, struct file *hinofile) +{ + int err, hidx; + ssize_t ssz; + size_t sz, n; + loff_t pos; + uint64_t u64; + struct au_dr_hino *ent; + struct inode *hinoinode; + struct hlist_bl_head *hbl; + + err = 0; + pos = 0; + hbl = dr->dr_h_ino; + hinoinode = file_inode(hinofile); + sz = i_size_read(hinoinode); + AuDebugOn(sz % sizeof(u64)); + n = sz / sizeof(u64); + while (n--) { + ssz = vfsub_read_k(hinofile, &u64, sizeof(u64), &pos); + if (unlikely(ssz != sizeof(u64))) { + pr_err("ssz %zd, %pD2\n", ssz, hinofile); + err = -EINVAL; + if (ssz < 0) + err = ssz; + goto out_free; + } + + ent = kmalloc(sizeof(*ent), GFP_NOFS); + if (!ent) { + err = -ENOMEM; + AuTraceErr(err); + goto out_free; + } + ent->dr_h_ino = be64_to_cpu((__force __be64)u64); + AuDbg("hi%llu, %pD2\n", + (unsigned long long)ent->dr_h_ino, hinofile); + hidx = au_dr_ihash(ent->dr_h_ino); + au_hbl_add(&ent->dr_hnode, hbl + hidx); + } + goto out; /* success */ + +out_free: + au_dr_hino_free(dr); +out: + AuTraceErr(err); + return err; +} + +/* + * @bindex/@br is a switch to distinguish whether suspending hnotify or not. + * @path is a switch to distinguish load and store. + */ +static int au_dr_hino(struct super_block *sb, aufs_bindex_t bindex, + struct au_branch *br, const struct path *path) +{ + int err, flags; + unsigned char load, suspend; + struct file *hinofile; + struct au_hinode *hdir; + struct inode *dir, *delegated; + struct path hinopath; + struct qstr hinoname = QSTR_INIT(AUFS_WH_DR_BRHINO, + sizeof(AUFS_WH_DR_BRHINO) - 1); + + AuDebugOn(bindex < 0 && !br); + AuDebugOn(bindex >= 0 && br); + + err = -EINVAL; + suspend = !br; + if (suspend) + br = au_sbr(sb, bindex); + load = !!path; + if (!load) { + path = &br->br_path; + AuDebugOn(!au_br_writable(br->br_perm)); + if (unlikely(!au_br_writable(br->br_perm))) + goto out; + } + + hdir = NULL; + if (suspend) { + dir = d_inode(sb->s_root); + hdir = au_hinode(au_ii(dir), bindex); + dir = hdir->hi_inode; + au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD); + } else { + dir = d_inode(path->dentry); + inode_lock_nested(dir, AuLsc_I_CHILD); + } + hinopath.dentry = vfsub_lkup_one(&hinoname, path->dentry); + err = PTR_ERR(hinopath.dentry); + if (IS_ERR(hinopath.dentry)) + goto out_unlock; + + err = 0; + flags = O_RDONLY; + if (load) { + if (d_is_negative(hinopath.dentry)) + goto out_dput; /* success */ + } else { + if (au_dr_hino_test_empty(&br->br_dirren)) { + if (d_is_positive(hinopath.dentry)) { + delegated = NULL; + err = vfsub_unlink(dir, &hinopath, &delegated, + /*force*/0); + AuTraceErr(err); + if (unlikely(err)) + pr_err("ignored err %d, %pd2\n", + err, hinopath.dentry); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + err = 0; + } + goto out_dput; + } else if (!d_is_positive(hinopath.dentry)) { + err = vfsub_create(dir, &hinopath, 0600, + /*want_excl*/false); + AuTraceErr(err); + if (unlikely(err)) + goto out_dput; + } + flags = O_WRONLY; + } + hinopath.mnt = path->mnt; + hinofile = vfsub_dentry_open(&hinopath, flags); + if (suspend) + au_hn_inode_unlock(hdir); + else + inode_unlock(dir); + dput(hinopath.dentry); + AuTraceErrPtr(hinofile); + if (IS_ERR(hinofile)) { + err = PTR_ERR(hinofile); + goto out; + } + + if (load) + err = au_dr_hino_load(&br->br_dirren, hinofile); + else + err = au_dr_hino_store(sb, br, hinofile); + fput(hinofile); + goto out; + +out_dput: + dput(hinopath.dentry); +out_unlock: + if (suspend) + au_hn_inode_unlock(hdir); + else + inode_unlock(dir); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_dr_brid_init(struct au_dr_brid *brid, const struct path *path) +{ + int err; + struct kstatfs kstfs; + dev_t dev; + struct dentry *dentry; + struct super_block *sb; + + err = vfs_statfs((void *)path, &kstfs); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + /* todo: support for UUID */ + + if (kstfs.f_fsid.val[0] || kstfs.f_fsid.val[1]) { + brid->type = AuBrid_FSID; + brid->fsid = kstfs.f_fsid; + } else { + dentry = path->dentry; + sb = dentry->d_sb; + dev = sb->s_dev; + if (dev) { + brid->type = AuBrid_DEV; + brid->dev = dev; + } + } + +out: + return err; +} + +int au_dr_br_init(struct super_block *sb, struct au_branch *br, + const struct path *path) +{ + int err, i; + struct au_dr_br *dr; + struct hlist_bl_head *hbl; + + dr = &br->br_dirren; + hbl = dr->dr_h_ino; + for (i = 0; i < AuDirren_NHASH; i++, hbl++) + INIT_HLIST_BL_HEAD(hbl); + + err = au_dr_brid_init(&dr->dr_brid, path); + if (unlikely(err)) + goto out; + + if (au_opt_test(au_mntflags(sb), DIRREN)) + err = au_dr_hino(sb, /*bindex*/-1, br, path); + +out: + AuTraceErr(err); + return err; +} + +int au_dr_br_fin(struct super_block *sb, struct au_branch *br) +{ + int err; + + err = 0; + if (au_br_writable(br->br_perm)) + err = au_dr_hino(sb, /*bindex*/-1, br, /*path*/NULL); + if (!err) + au_dr_hino_free(&br->br_dirren); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_brid_str(struct au_dr_brid *brid, struct inode *h_inode, + char *buf, size_t sz) +{ + int err; + unsigned int major, minor; + char *p; + + p = buf; + err = snprintf(p, sz, "%d_", brid->type); + AuDebugOn(err > sz); + p += err; + sz -= err; + switch (brid->type) { + case AuBrid_Unset: + return -EINVAL; + case AuBrid_UUID: + err = snprintf(p, sz, "%pU", brid->uuid.b); + break; + case AuBrid_FSID: + err = snprintf(p, sz, "%08x-%08x", + brid->fsid.val[0], brid->fsid.val[1]); + break; + case AuBrid_DEV: + major = MAJOR(brid->dev); + minor = MINOR(brid->dev); + if (major <= 0xff && minor <= 0xff) + err = snprintf(p, sz, "%02x%02x", major, minor); + else + err = snprintf(p, sz, "%03x:%05x", major, minor); + break; + } + AuDebugOn(err > sz); + p += err; + sz -= err; + err = snprintf(p, sz, "_%llu", (unsigned long long)h_inode->i_ino); + AuDebugOn(err > sz); + p += err; + sz -= err; + + return p - buf; +} + +static int au_drinfo_name(struct au_branch *br, char *name, int len) +{ + int rlen; + struct dentry *br_dentry; + struct inode *br_inode; + + br_dentry = au_br_dentry(br); + br_inode = d_inode(br_dentry); + rlen = au_brid_str(&br->br_dirren.dr_brid, br_inode, name, len); + AuDebugOn(rlen >= AUFS_DIRREN_ENV_VAL_SZ); + AuDebugOn(rlen > len); + + return rlen; +} + +/* ---------------------------------------------------------------------- */ + +/* + * from the given @h_dentry, construct drinfo at @*fdata. + * when the size of @*fdata is not enough, reallocate and return new @fdata and + * @allocated. + */ +static int au_drinfo_construct(struct au_drinfo_fdata **fdata, + struct dentry *h_dentry, + unsigned char *allocated) +{ + int err, v; + struct au_drinfo_fdata *f, *p; + struct au_drinfo *drinfo; + struct inode *h_inode; + struct qstr *qname; + + err = 0; + f = *fdata; + h_inode = d_inode(h_dentry); + qname = &h_dentry->d_name; + drinfo = &f->drinfo; + drinfo->ino = (__force uint64_t)cpu_to_be64(h_inode->i_ino); + drinfo->oldnamelen = qname->len; + if (*allocated < sizeof(*f) + qname->len) { + v = roundup_pow_of_two(*allocated + qname->len); + p = au_krealloc(f, v, GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) { + err = -ENOMEM; + AuTraceErr(err); + goto out; + } + f = p; + *fdata = f; + *allocated = v; + drinfo = &f->drinfo; + } + memcpy(drinfo->oldname, qname->name, qname->len); + AuDbg("i%llu, %.*s\n", + be64_to_cpu((__force __be64)drinfo->ino), drinfo->oldnamelen, + drinfo->oldname); + +out: + AuTraceErr(err); + return err; +} + +/* callers have to free the return value */ +static struct au_drinfo *au_drinfo_read_k(struct file *file, ino_t h_ino) +{ + struct au_drinfo *ret, *drinfo; + struct au_drinfo_fdata fdata; + int len; + loff_t pos; + ssize_t ssz; + + ret = ERR_PTR(-EIO); + pos = 0; + ssz = vfsub_read_k(file, &fdata, sizeof(fdata), &pos); + if (unlikely(ssz != sizeof(fdata))) { + AuIOErr("ssz %zd, %u, %pD2\n", + ssz, (unsigned int)sizeof(fdata), file); + goto out; + } + + fdata.magic = ntohl((__force __be32)fdata.magic); + switch (fdata.magic) { + case AUFS_DRINFO_MAGIC_V1: + break; + default: + AuIOErr("magic-num 0x%x, 0x%x, %pD2\n", + fdata.magic, AUFS_DRINFO_MAGIC_V1, file); + goto out; + } + + drinfo = &fdata.drinfo; + len = drinfo->oldnamelen; + if (!len) { + AuIOErr("broken drinfo %pD2\n", file); + goto out; + } + + ret = NULL; + drinfo->ino = be64_to_cpu((__force __be64)drinfo->ino); + if (unlikely(h_ino && drinfo->ino != h_ino)) { + AuDbg("ignored i%llu, i%llu, %pD2\n", + (unsigned long long)drinfo->ino, + (unsigned long long)h_ino, file); + goto out; /* success */ + } + + ret = kmalloc(sizeof(*ret) + len, GFP_NOFS); + if (unlikely(!ret)) { + ret = ERR_PTR(-ENOMEM); + AuTraceErrPtr(ret); + goto out; + } + + *ret = *drinfo; + ssz = vfsub_read_k(file, (void *)ret->oldname, len, &pos); + if (unlikely(ssz != len)) { + au_kfree_rcu(ret); + ret = ERR_PTR(-EIO); + AuIOErr("ssz %zd, %u, %pD2\n", ssz, len, file); + goto out; + } + + AuDbg("oldname %.*s\n", ret->oldnamelen, ret->oldname); + +out: + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* in order to be revertible */ +struct au_drinfo_rev_elm { + int created; + struct dentry *info_dentry; + struct au_drinfo *info_last; +}; + +struct au_drinfo_rev { + unsigned char already; + aufs_bindex_t nelm; + struct au_drinfo_rev_elm elm[0]; +}; + +/* todo: isn't it too large? */ +struct au_drinfo_store { + struct path h_ppath; + struct dentry *h_dentry; + struct au_drinfo_fdata *fdata; + char *infoname; /* inside of whname, just after PFX */ + char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ]; + aufs_bindex_t btgt, btail; + unsigned char no_sio, + allocated, /* current size of *fdata */ + infonamelen, /* room size for p */ + whnamelen, /* length of the generated name */ + renameback; /* renamed back */ +}; + +/* on rename(2) error, the caller should revert it using @elm */ +static int au_drinfo_do_store(struct au_drinfo_store *w, + struct au_drinfo_rev_elm *elm) +{ + int err, len; + ssize_t ssz; + loff_t pos; + struct path infopath = { + .mnt = w->h_ppath.mnt + }; + struct inode *h_dir, *h_inode, *delegated; + struct file *infofile; + struct qstr *qname; + + AuDebugOn(elm + && memcmp(elm, page_address(ZERO_PAGE(0)), sizeof(*elm))); + + infopath.dentry = vfsub_lookup_one_len(w->whname, w->h_ppath.dentry, + w->whnamelen); + AuTraceErrPtr(infopath.dentry); + if (IS_ERR(infopath.dentry)) { + err = PTR_ERR(infopath.dentry); + goto out; + } + + err = 0; + h_dir = d_inode(w->h_ppath.dentry); + if (elm && d_is_negative(infopath.dentry)) { + err = vfsub_create(h_dir, &infopath, 0600, /*want_excl*/true); + AuTraceErr(err); + if (unlikely(err)) + goto out_dput; + elm->created = 1; + elm->info_dentry = dget(infopath.dentry); + } + + infofile = vfsub_dentry_open(&infopath, O_RDWR); + AuTraceErrPtr(infofile); + if (IS_ERR(infofile)) { + err = PTR_ERR(infofile); + goto out_dput; + } + + h_inode = d_inode(infopath.dentry); + if (elm && i_size_read(h_inode)) { + h_inode = d_inode(w->h_dentry); + elm->info_last = au_drinfo_read_k(infofile, h_inode->i_ino); + AuTraceErrPtr(elm->info_last); + if (IS_ERR(elm->info_last)) { + err = PTR_ERR(elm->info_last); + elm->info_last = NULL; + AuDebugOn(elm->info_dentry); + goto out_fput; + } + } + + if (elm && w->renameback) { + delegated = NULL; + err = vfsub_unlink(h_dir, &infopath, &delegated, /*force*/0); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + goto out_fput; + } + + pos = 0; + qname = &w->h_dentry->d_name; + len = sizeof(*w->fdata) + qname->len; + if (!elm) + len = sizeof(*w->fdata) + w->fdata->drinfo.oldnamelen; + ssz = vfsub_write_k(infofile, w->fdata, len, &pos); + if (ssz == len) { + AuDbg("hi%llu, %.*s\n", w->fdata->drinfo.ino, + w->fdata->drinfo.oldnamelen, w->fdata->drinfo.oldname); + goto out_fput; /* success */ + } else { + err = -EIO; + if (ssz < 0) + err = ssz; + /* the caller should revert it using @elm */ + } + +out_fput: + fput(infofile); +out_dput: + dput(infopath.dentry); +out: + AuTraceErr(err); + return err; +} + +struct au_call_drinfo_do_store_args { + int *errp; + struct au_drinfo_store *w; + struct au_drinfo_rev_elm *elm; +}; + +static void au_call_drinfo_do_store(void *args) +{ + struct au_call_drinfo_do_store_args *a = args; + + *a->errp = au_drinfo_do_store(a->w, a->elm); +} + +static int au_drinfo_store_sio(struct au_drinfo_store *w, + struct au_drinfo_rev_elm *elm) +{ + int err, wkq_err; + + if (w->no_sio) + err = au_drinfo_do_store(w, elm); + else { + struct au_call_drinfo_do_store_args a = { + .errp = &err, + .w = w, + .elm = elm + }; + wkq_err = au_wkq_wait(au_call_drinfo_do_store, &a); + if (unlikely(wkq_err)) + err = wkq_err; + } + AuTraceErr(err); + + return err; +} + +static int au_drinfo_store_work_init(struct au_drinfo_store *w, + aufs_bindex_t btgt) +{ + int err; + + memset(w, 0, sizeof(*w)); + w->allocated = roundup_pow_of_two(sizeof(*w->fdata) + 40); + strcpy(w->whname, AUFS_WH_DR_INFO_PFX); + w->infoname = w->whname + sizeof(AUFS_WH_DR_INFO_PFX) - 1; + w->infonamelen = sizeof(w->whname) - sizeof(AUFS_WH_DR_INFO_PFX); + w->btgt = btgt; + w->no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID); + + err = -ENOMEM; + w->fdata = kcalloc(1, w->allocated, GFP_NOFS); + if (unlikely(!w->fdata)) { + AuTraceErr(err); + goto out; + } + w->fdata->magic = (__force uint32_t)htonl(AUFS_DRINFO_MAGIC_V1); + err = 0; + +out: + return err; +} + +static void au_drinfo_store_work_fin(struct au_drinfo_store *w) +{ + au_kfree_rcu(w->fdata); +} + +static void au_drinfo_store_rev(struct au_drinfo_rev *rev, + struct au_drinfo_store *w) +{ + struct au_drinfo_rev_elm *elm; + struct inode *h_dir, *delegated; + int err, nelm; + struct path infopath = { + .mnt = w->h_ppath.mnt + }; + + h_dir = d_inode(w->h_ppath.dentry); + IMustLock(h_dir); + + err = 0; + elm = rev->elm; + for (nelm = rev->nelm; nelm > 0; nelm--, elm++) { + AuDebugOn(elm->created && elm->info_last); + if (elm->created) { + AuDbg("here\n"); + delegated = NULL; + infopath.dentry = elm->info_dentry; + err = vfsub_unlink(h_dir, &infopath, &delegated, + !w->no_sio); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + dput(elm->info_dentry); + } else if (elm->info_last) { + AuDbg("here\n"); + w->fdata->drinfo = *elm->info_last; + memcpy(w->fdata->drinfo.oldname, + elm->info_last->oldname, + elm->info_last->oldnamelen); + err = au_drinfo_store_sio(w, /*elm*/NULL); + au_kfree_rcu(elm->info_last); + } + if (unlikely(err)) + AuIOErr("%d, %s\n", err, w->whname); + /* go on even if err */ + } +} + +/* caller has to call au_dr_rename_fin() later */ +static int au_drinfo_store(struct dentry *dentry, aufs_bindex_t btgt, + struct qstr *dst_name, void *_rev) +{ + int err, sz, nelm; + aufs_bindex_t bindex, btail; + struct au_drinfo_store work; + struct au_drinfo_rev *rev, **p; + struct au_drinfo_rev_elm *elm; + struct super_block *sb; + struct au_branch *br; + struct au_hinode *hdir; + + err = au_drinfo_store_work_init(&work, btgt); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + err = -ENOMEM; + btail = au_dbtaildir(dentry); + nelm = btail - btgt; + sz = sizeof(*rev) + sizeof(*elm) * nelm; + rev = kcalloc(1, sz, GFP_NOFS); + if (unlikely(!rev)) { + AuTraceErr(err); + goto out_args; + } + rev->nelm = nelm; + elm = rev->elm; + p = _rev; + *p = rev; + + err = 0; + sb = dentry->d_sb; + work.h_ppath.dentry = au_h_dptr(dentry, btgt); + work.h_ppath.mnt = au_sbr_mnt(sb, btgt); + hdir = au_hi(d_inode(dentry), btgt); + au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD); + for (bindex = btgt + 1; bindex <= btail; bindex++, elm++) { + work.h_dentry = au_h_dptr(dentry, bindex); + if (!work.h_dentry) + continue; + + err = au_drinfo_construct(&work.fdata, work.h_dentry, + &work.allocated); + AuTraceErr(err); + if (unlikely(err)) + break; + + work.renameback = au_qstreq(&work.h_dentry->d_name, dst_name); + br = au_sbr(sb, bindex); + work.whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1; + work.whnamelen += au_drinfo_name(br, work.infoname, + work.infonamelen); + AuDbg("whname %.*s, i%llu, %.*s\n", + work.whnamelen, work.whname, + be64_to_cpu((__force __be64)work.fdata->drinfo.ino), + work.fdata->drinfo.oldnamelen, + work.fdata->drinfo.oldname); + + err = au_drinfo_store_sio(&work, elm); + AuTraceErr(err); + if (unlikely(err)) + break; + } + if (unlikely(err)) { + /* revert all drinfo */ + au_drinfo_store_rev(rev, &work); + au_kfree_try_rcu(rev); + *p = NULL; + } + au_hn_inode_unlock(hdir); + +out_args: + au_drinfo_store_work_fin(&work); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_dr_rename(struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev) +{ + int err, already; + ino_t ino; + struct super_block *sb; + struct au_branch *br; + struct au_dr_br *dr; + struct dentry *h_dentry; + struct inode *h_inode; + struct au_dr_hino *ent; + struct au_drinfo_rev *rev, **p; + + AuDbg("bindex %d\n", bindex); + + err = -ENOMEM; + ent = kmalloc(sizeof(*ent), GFP_NOFS); + if (unlikely(!ent)) + goto out; + + sb = src->d_sb; + br = au_sbr(sb, bindex); + dr = &br->br_dirren; + h_dentry = au_h_dptr(src, bindex); + h_inode = d_inode(h_dentry); + ino = h_inode->i_ino; + ent->dr_h_ino = ino; + already = au_dr_hino_test_add(dr, ino, ent); + AuDbg("b%d, hi%llu, already %d\n", + bindex, (unsigned long long)ino, already); + + err = au_drinfo_store(src, bindex, dst_name, _rev); + AuTraceErr(err); + if (!err) { + p = _rev; + rev = *p; + rev->already = already; + goto out; /* success */ + } + + /* revert */ + if (!already) + au_dr_hino_del(dr, ent); + au_kfree_rcu(ent); + +out: + AuTraceErr(err); + return err; +} + +void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *_rev) +{ + struct au_drinfo_rev *rev; + struct au_drinfo_rev_elm *elm; + int nelm; + + rev = _rev; + elm = rev->elm; + for (nelm = rev->nelm; nelm > 0; nelm--, elm++) { + dput(elm->info_dentry); + au_kfree_rcu(elm->info_last); + } + au_kfree_try_rcu(rev); +} + +void au_dr_rename_rev(struct dentry *src, aufs_bindex_t btgt, void *_rev) +{ + int err; + struct au_drinfo_store work; + struct au_drinfo_rev *rev = _rev; + struct super_block *sb; + struct au_branch *br; + struct inode *h_inode; + struct au_dr_br *dr; + struct au_dr_hino *ent; + + err = au_drinfo_store_work_init(&work, btgt); + if (unlikely(err)) + goto out; + + sb = src->d_sb; + br = au_sbr(sb, btgt); + work.h_ppath.dentry = au_h_dptr(src, btgt); + work.h_ppath.mnt = au_br_mnt(br); + au_drinfo_store_rev(rev, &work); + au_drinfo_store_work_fin(&work); + if (rev->already) + goto out; + + dr = &br->br_dirren; + h_inode = d_inode(work.h_ppath.dentry); + ent = au_dr_hino_find(dr, h_inode->i_ino); + BUG_ON(!ent); + au_dr_hino_del(dr, ent); + au_kfree_rcu(ent); + +out: + au_kfree_try_rcu(rev); + if (unlikely(err)) + pr_err("failed to remove dirren info\n"); +} + +/* ---------------------------------------------------------------------- */ + +static struct au_drinfo *au_drinfo_do_load(struct path *h_ppath, + char *whname, int whnamelen, + struct dentry **info_dentry) +{ + struct au_drinfo *drinfo; + struct file *f; + struct inode *h_dir; + struct path infopath; + int unlocked; + + AuDbg("%pd/%.*s\n", h_ppath->dentry, whnamelen, whname); + + *info_dentry = NULL; + drinfo = NULL; + unlocked = 0; + h_dir = d_inode(h_ppath->dentry); + vfsub_inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + infopath.dentry = vfsub_lookup_one_len(whname, h_ppath->dentry, + whnamelen); + if (IS_ERR(infopath.dentry)) { + drinfo = (void *)infopath.dentry; + goto out; + } + + if (d_is_negative(infopath.dentry)) + goto out_dput; /* success */ + + infopath.mnt = h_ppath->mnt; + f = vfsub_dentry_open(&infopath, O_RDONLY); + inode_unlock_shared(h_dir); + unlocked = 1; + if (IS_ERR(f)) { + drinfo = (void *)f; + goto out_dput; + } + + drinfo = au_drinfo_read_k(f, /*h_ino*/0); + if (IS_ERR_OR_NULL(drinfo)) + goto out_fput; + + AuDbg("oldname %.*s\n", drinfo->oldnamelen, drinfo->oldname); + *info_dentry = dget(infopath.dentry); /* keep it alive */ + +out_fput: + fput(f); +out_dput: + dput(infopath.dentry); +out: + if (!unlocked) + inode_unlock_shared(h_dir); + AuTraceErrPtr(drinfo); + return drinfo; +} + +struct au_drinfo_do_load_args { + struct au_drinfo **drinfop; + struct path *h_ppath; + char *whname; + int whnamelen; + struct dentry **info_dentry; +}; + +static void au_call_drinfo_do_load(void *args) +{ + struct au_drinfo_do_load_args *a = args; + + *a->drinfop = au_drinfo_do_load(a->h_ppath, a->whname, a->whnamelen, + a->info_dentry); +} + +struct au_drinfo_load { + struct path h_ppath; + struct qstr *qname; + unsigned char no_sio; + + aufs_bindex_t ninfo; + struct au_drinfo **drinfo; +}; + +static int au_drinfo_load(struct au_drinfo_load *w, aufs_bindex_t bindex, + struct au_branch *br) +{ + int err, wkq_err, whnamelen, e; + char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ] + = AUFS_WH_DR_INFO_PFX; + struct au_drinfo *drinfo; + struct qstr oldname; + struct inode *h_dir, *delegated; + struct dentry *info_dentry; + struct path infopath; + + whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1; + whnamelen += au_drinfo_name(br, whname + whnamelen, + sizeof(whname) - whnamelen); + if (w->no_sio) + drinfo = au_drinfo_do_load(&w->h_ppath, whname, whnamelen, + &info_dentry); + else { + struct au_drinfo_do_load_args args = { + .drinfop = &drinfo, + .h_ppath = &w->h_ppath, + .whname = whname, + .whnamelen = whnamelen, + .info_dentry = &info_dentry + }; + wkq_err = au_wkq_wait(au_call_drinfo_do_load, &args); + if (unlikely(wkq_err)) + drinfo = ERR_PTR(wkq_err); + } + err = PTR_ERR(drinfo); + if (IS_ERR_OR_NULL(drinfo)) + goto out; + + err = 0; + oldname.len = drinfo->oldnamelen; + oldname.name = drinfo->oldname; + if (au_qstreq(w->qname, &oldname)) { + /* the name is renamed back */ + au_kfree_rcu(drinfo); + drinfo = NULL; + + infopath.dentry = info_dentry; + infopath.mnt = w->h_ppath.mnt; + h_dir = d_inode(w->h_ppath.dentry); + delegated = NULL; + inode_lock_nested(h_dir, AuLsc_I_PARENT); + e = vfsub_unlink(h_dir, &infopath, &delegated, !w->no_sio); + inode_unlock(h_dir); + if (unlikely(e)) + AuIOErr("ignored %d, %pd2\n", e, &infopath.dentry); + if (unlikely(e == -EWOULDBLOCK)) + iput(delegated); + } + au_kfree_rcu(w->drinfo[bindex]); + w->drinfo[bindex] = drinfo; + dput(info_dentry); + +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_dr_lkup_free(struct au_drinfo **drinfo, int n) +{ + struct au_drinfo **p = drinfo; + + while (n-- > 0) + au_kfree_rcu(*drinfo++); + au_kfree_try_rcu(p); +} + +int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t btgt) +{ + int err, ninfo; + struct au_drinfo_load w; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + struct inode *h_dir; + struct au_dr_hino *ent; + struct super_block *sb; + + AuDbg("%.*s, name %.*s, whname %.*s, b%d\n", + AuLNPair(&dentry->d_name), AuLNPair(&lkup->dirren.dr_name), + AuLNPair(&lkup->whname), btgt); + + sb = dentry->d_sb; + bbot = au_sbbot(sb); + w.ninfo = bbot + 1; + if (!lkup->dirren.drinfo) { + lkup->dirren.drinfo = kcalloc(w.ninfo, + sizeof(*lkup->dirren.drinfo), + GFP_NOFS); + if (unlikely(!lkup->dirren.drinfo)) { + err = -ENOMEM; + goto out; + } + lkup->dirren.ninfo = w.ninfo; + } + w.drinfo = lkup->dirren.drinfo; + w.no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID); + w.h_ppath.dentry = au_h_dptr(dentry, btgt); + AuDebugOn(!w.h_ppath.dentry); + w.h_ppath.mnt = au_sbr_mnt(sb, btgt); + w.qname = &dentry->d_name; + + ninfo = 0; + for (bindex = btgt + 1; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_drinfo_load(&w, bindex, br); + if (unlikely(err)) + goto out_free; + if (w.drinfo[bindex]) + ninfo++; + } + if (!ninfo) { + br = au_sbr(sb, btgt); + h_dir = d_inode(w.h_ppath.dentry); + ent = au_dr_hino_find(&br->br_dirren, h_dir->i_ino); + AuDebugOn(!ent); + au_dr_hino_del(&br->br_dirren, ent); + au_kfree_rcu(ent); + } + goto out; /* success */ + +out_free: + au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo); + lkup->dirren.ninfo = 0; + lkup->dirren.drinfo = NULL; +out: + AuTraceErr(err); + return err; +} + +void au_dr_lkup_fin(struct au_do_lookup_args *lkup) +{ + au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo); +} + +int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt) +{ + int err; + struct au_drinfo *drinfo; + + err = 0; + if (!lkup->dirren.drinfo) + goto out; + AuDebugOn(lkup->dirren.ninfo < btgt + 1); + drinfo = lkup->dirren.drinfo[btgt + 1]; + if (!drinfo) + goto out; + + au_kfree_try_rcu(lkup->whname.name); + lkup->whname.name = NULL; + lkup->dirren.dr_name.len = drinfo->oldnamelen; + lkup->dirren.dr_name.name = drinfo->oldname; + lkup->name = &lkup->dirren.dr_name; + err = au_wh_name_alloc(&lkup->whname, lkup->name); + if (!err) + AuDbg("name %.*s, whname %.*s, b%d\n", + AuLNPair(lkup->name), AuLNPair(&lkup->whname), + btgt); + +out: + AuTraceErr(err); + return err; +} + +int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex, + ino_t h_ino) +{ + int match; + struct au_drinfo *drinfo; + + match = 1; + if (!lkup->dirren.drinfo) + goto out; + AuDebugOn(lkup->dirren.ninfo < bindex + 1); + drinfo = lkup->dirren.drinfo[bindex + 1]; + if (!drinfo) + goto out; + + match = (drinfo->ino == h_ino); + AuDbg("match %d\n", match); + +out: + return match; +} + +/* ---------------------------------------------------------------------- */ + +int au_dr_opt_set(struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_dr_hino(sb, bindex, /*br*/NULL, &br->br_path); + } + + return err; +} + +int au_dr_opt_flush(struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm)) + err = au_dr_hino(sb, bindex, /*br*/NULL, /*path*/NULL); + } + + return err; +} + +int au_dr_opt_clr(struct super_block *sb, int no_flush) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + if (!no_flush) { + err = au_dr_opt_flush(sb); + if (unlikely(err)) + goto out; + } + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + au_dr_hino_free(&br->br_dirren); + } + +out: + return err; +} diff --git a/fs/aufs/dirren.h b/fs/aufs/dirren.h new file mode 100644 index 0000000000000..a986736462c3a --- /dev/null +++ b/fs/aufs/dirren.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2017-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * renamed dir info + */ + +#ifndef __AUFS_DIRREN_H__ +#define __AUFS_DIRREN_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include "hbl.h" + +#define AuDirren_NHASH 100 + +#ifdef CONFIG_AUFS_DIRREN +enum au_brid_type { + AuBrid_Unset, + AuBrid_UUID, + AuBrid_FSID, + AuBrid_DEV +}; + +struct au_dr_brid { + enum au_brid_type type; + union { + uuid_t uuid; /* unimplemented yet */ + fsid_t fsid; + dev_t dev; + }; +}; + +/* 20 is the max digits length of ulong 64 */ +/* brid-type "_" uuid "_" inum */ +#define AUFS_DIRREN_FNAME_SZ (1 + 1 + UUID_STRING_LEN + 20) +#define AUFS_DIRREN_ENV_VAL_SZ (AUFS_DIRREN_FNAME_SZ + 1 + 20) + +struct au_dr_hino { + struct hlist_bl_node dr_hnode; + ino_t dr_h_ino; +}; + +struct au_dr_br { + struct hlist_bl_head dr_h_ino[AuDirren_NHASH]; + struct au_dr_brid dr_brid; +}; + +struct au_dr_lookup { + /* dr_name is pointed by struct au_do_lookup_args.name */ + struct qstr dr_name; /* subset of dr_info */ + aufs_bindex_t ninfo; + struct au_drinfo **drinfo; +}; +#else +struct au_dr_hino; +/* empty */ +struct au_dr_br { }; +struct au_dr_lookup { }; +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_branch; +struct au_do_lookup_args; +struct au_hinode; +#ifdef CONFIG_AUFS_DIRREN +int au_dr_hino_test_add(struct au_dr_br *dr, ino_t h_ino, + struct au_dr_hino *add_ent); +void au_dr_hino_free(struct au_dr_br *dr); +int au_dr_br_init(struct super_block *sb, struct au_branch *br, + const struct path *path); +int au_dr_br_fin(struct super_block *sb, struct au_branch *br); +int au_dr_rename(struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev); +void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *rev); +void au_dr_rename_rev(struct dentry *src, aufs_bindex_t bindex, void *rev); +int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t bindex); +int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt); +int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex, + ino_t h_ino); +void au_dr_lkup_fin(struct au_do_lookup_args *lkup); +int au_dr_opt_set(struct super_block *sb); +int au_dr_opt_flush(struct super_block *sb); +int au_dr_opt_clr(struct super_block *sb, int no_flush); +#else +AuStubInt0(au_dr_hino_test_add, struct au_dr_br *dr, ino_t h_ino, + struct au_dr_hino *add_ent); +AuStubVoid(au_dr_hino_free, struct au_dr_br *dr); +AuStubInt0(au_dr_br_init, struct super_block *sb, struct au_branch *br, + const struct path *path); +AuStubInt0(au_dr_br_fin, struct super_block *sb, struct au_branch *br); +AuStubInt0(au_dr_rename, struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev); +AuStubVoid(au_dr_rename_fin, struct dentry *src, aufs_bindex_t btgt, void *rev); +AuStubVoid(au_dr_rename_rev, struct dentry *src, aufs_bindex_t bindex, + void *rev); +AuStubInt0(au_dr_lkup, struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t bindex); +AuStubInt0(au_dr_lkup_name, struct au_do_lookup_args *lkup, aufs_bindex_t btgt); +AuStubInt0(au_dr_lkup_h_ino, struct au_do_lookup_args *lkup, + aufs_bindex_t bindex, ino_t h_ino); +AuStubVoid(au_dr_lkup_fin, struct au_do_lookup_args *lkup); +AuStubInt0(au_dr_opt_set, struct super_block *sb); +AuStubInt0(au_dr_opt_flush, struct super_block *sb); +AuStubInt0(au_dr_opt_clr, struct super_block *sb, int no_flush); +#endif + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_DIRREN +static inline int au_dr_ihash(ino_t h_ino) +{ + return h_ino % AuDirren_NHASH; +} +#else +AuStubInt0(au_dr_ihash, ino_t h_ino); +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DIRREN_H__ */ diff --git a/fs/aufs/dynop.c b/fs/aufs/dynop.c new file mode 100644 index 0000000000000..7c7d8432d4226 --- /dev/null +++ b/fs/aufs/dynop.c @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dynamically customizable operations for regular files + */ + +#include "aufs.h" + +#define DyPrSym(key) AuDbgSym(key->dk_op.dy_hop) + +/* + * How large will these lists be? + * Usually just a few elements, 20-30 at most for each, I guess. + */ +static struct hlist_bl_head dynop[AuDyLast]; + +static struct au_dykey *dy_gfind_get(struct hlist_bl_head *hbl, + const void *h_op) +{ + struct au_dykey *key, *tmp; + struct hlist_bl_node *pos; + + key = NULL; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + if (kref_get_unless_zero(&tmp->dk_kref)) + key = tmp; + break; + } + hlist_bl_unlock(hbl); + + return key; +} + +static struct au_dykey *dy_bradd(struct au_branch *br, struct au_dykey *key) +{ + struct au_dykey **k, *found; + const void *h_op = key->dk_op.dy_hop; + int i; + + found = NULL; + k = br->br_dykey; + for (i = 0; i < AuBrDynOp; i++) + if (k[i]) { + if (k[i]->dk_op.dy_hop == h_op) { + found = k[i]; + break; + } + } else + break; + if (!found) { + spin_lock(&br->br_dykey_lock); + for (; i < AuBrDynOp; i++) + if (k[i]) { + if (k[i]->dk_op.dy_hop == h_op) { + found = k[i]; + break; + } + } else { + k[i] = key; + break; + } + spin_unlock(&br->br_dykey_lock); + BUG_ON(i == AuBrDynOp); /* expand the array */ + } + + return found; +} + +/* kref_get() if @key is already added */ +static struct au_dykey *dy_gadd(struct hlist_bl_head *hbl, struct au_dykey *key) +{ + struct au_dykey *tmp, *found; + struct hlist_bl_node *pos; + const void *h_op = key->dk_op.dy_hop; + + found = NULL; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + if (kref_get_unless_zero(&tmp->dk_kref)) + found = tmp; + break; + } + if (!found) + hlist_bl_add_head(&key->dk_hnode, hbl); + hlist_bl_unlock(hbl); + + if (!found) + DyPrSym(key); + return found; +} + +static void dy_free_rcu(struct rcu_head *rcu) +{ + struct au_dykey *key; + + key = container_of(rcu, struct au_dykey, dk_rcu); + DyPrSym(key); + kfree(key); +} + +static void dy_free(struct kref *kref) +{ + struct au_dykey *key; + struct hlist_bl_head *hbl; + + key = container_of(kref, struct au_dykey, dk_kref); + hbl = dynop + key->dk_op.dy_type; + au_hbl_del(&key->dk_hnode, hbl); + call_rcu(&key->dk_rcu, dy_free_rcu); +} + +void au_dy_put(struct au_dykey *key) +{ + kref_put(&key->dk_kref, dy_free); +} + +/* ---------------------------------------------------------------------- */ + +#define DyDbgSize(cnt, op) AuDebugOn(cnt != sizeof(op)/sizeof(void *)) + +#ifdef CONFIG_AUFS_DEBUG +#define DyDbgDeclare(cnt) unsigned int cnt = 0 +#define DyDbgInc(cnt) do { cnt++; } while (0) +#else +#define DyDbgDeclare(cnt) do {} while (0) +#define DyDbgInc(cnt) do {} while (0) +#endif + +#define DySet(func, dst, src, h_op, h_sb) do { \ + DyDbgInc(cnt); \ + if (h_op->func) { \ + if (src.func) \ + dst.func = src.func; \ + else \ + AuDbg("%s %s\n", au_sbtype(h_sb), #func); \ + } \ +} while (0) + +#define DySetForce(func, dst, src) do { \ + AuDebugOn(!src.func); \ + DyDbgInc(cnt); \ + dst.func = src.func; \ +} while (0) + +#define DySetAop(func) \ + DySet(func, dyaop->da_op, aufs_aop, h_aop, h_sb) +#define DySetAopForce(func) \ + DySetForce(func, dyaop->da_op, aufs_aop) + +static void dy_aop(struct au_dykey *key, const void *h_op, + struct super_block *h_sb __maybe_unused) +{ + struct au_dyaop *dyaop = (void *)key; + const struct address_space_operations *h_aop = h_op; + DyDbgDeclare(cnt); + + AuDbg("%s\n", au_sbtype(h_sb)); + + DySetAop(writepage); + DySetAopForce(readpage); /* force */ + DySetAop(writepages); + DySetAop(set_page_dirty); + DySetAop(readpages); + DySetAop(write_begin); + DySetAop(write_end); + DySetAop(bmap); + DySetAop(invalidatepage); + DySetAop(releasepage); + DySetAop(freepage); + /* this one will be changed according to an aufs mount option */ + DySetAop(direct_IO); + DySetAop(migratepage); + DySetAop(isolate_page); + DySetAop(putback_page); + DySetAop(launder_page); + DySetAop(is_partially_uptodate); + DySetAop(is_dirty_writeback); + DySetAop(error_remove_page); + DySetAop(swap_activate); + DySetAop(swap_deactivate); + + DyDbgSize(cnt, *h_aop); +} + +/* ---------------------------------------------------------------------- */ + +static void dy_bug(struct kref *kref) +{ + BUG(); +} + +static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) +{ + struct au_dykey *key, *old; + struct hlist_bl_head *hbl; + struct op { + unsigned int sz; + void (*set)(struct au_dykey *key, const void *h_op, + struct super_block *h_sb __maybe_unused); + }; + static const struct op a[] = { + [AuDy_AOP] = { + .sz = sizeof(struct au_dyaop), + .set = dy_aop + } + }; + const struct op *p; + + hbl = dynop + op->dy_type; + key = dy_gfind_get(hbl, op->dy_hop); + if (key) + goto out_add; /* success */ + + p = a + op->dy_type; + key = kzalloc(p->sz, GFP_NOFS); + if (unlikely(!key)) { + key = ERR_PTR(-ENOMEM); + goto out; + } + + key->dk_op.dy_hop = op->dy_hop; + kref_init(&key->dk_kref); + p->set(key, op->dy_hop, au_br_sb(br)); + old = dy_gadd(hbl, key); + if (old) { + au_kfree_rcu(key); + key = old; + } + +out_add: + old = dy_bradd(br, key); + if (old) + /* its ref-count should never be zero here */ + kref_put(&key->dk_kref, dy_bug); +out: + return key; +} + +/* ---------------------------------------------------------------------- */ +/* + * Aufs prohibits O_DIRECT by default even if the branch supports it. + * This behaviour is necessary to return an error from open(O_DIRECT) instead + * of the succeeding I/O. The dio mount option enables O_DIRECT and makes + * open(O_DIRECT) always succeed, but the succeeding I/O may return an error. + * See the aufs manual in detail. + */ +static void dy_adx(struct au_dyaop *dyaop, int do_dx) +{ + if (!do_dx) + dyaop->da_op.direct_IO = NULL; + else + dyaop->da_op.direct_IO = aufs_aop.direct_IO; +} + +static struct au_dyaop *dy_aget(struct au_branch *br, + const struct address_space_operations *h_aop, + int do_dx) +{ + struct au_dyaop *dyaop; + struct au_dynop op; + + op.dy_type = AuDy_AOP; + op.dy_haop = h_aop; + dyaop = (void *)dy_get(&op, br); + if (IS_ERR(dyaop)) + goto out; + dy_adx(dyaop, do_dx); + +out: + return dyaop; +} + +int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode) +{ + int err, do_dx; + struct super_block *sb; + struct au_branch *br; + struct au_dyaop *dyaop; + + AuDebugOn(!S_ISREG(h_inode->i_mode)); + IiMustWriteLock(inode); + + sb = inode->i_sb; + br = au_sbr(sb, bindex); + do_dx = !!au_opt_test(au_mntflags(sb), DIO); + dyaop = dy_aget(br, h_inode->i_mapping->a_ops, do_dx); + err = PTR_ERR(dyaop); + if (IS_ERR(dyaop)) + /* unnecessary to call dy_fput() */ + goto out; + + err = 0; + inode->i_mapping->a_ops = &dyaop->da_op; + +out: + return err; +} + +/* + * Is it safe to replace a_ops during the inode/file is in operation? + * Yes, I hope so. + */ +int au_dy_irefresh(struct inode *inode) +{ + int err; + aufs_bindex_t btop; + struct inode *h_inode; + + err = 0; + if (S_ISREG(inode->i_mode)) { + btop = au_ibtop(inode); + h_inode = au_h_iptr(inode, btop); + err = au_dy_iaop(inode, btop, h_inode); + } + return err; +} + +void au_dy_arefresh(int do_dx) +{ + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_dykey *key; + + hbl = dynop + AuDy_AOP; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(key, pos, hbl, dk_hnode) + dy_adx((void *)key, do_dx); + hlist_bl_unlock(hbl); +} + +/* ---------------------------------------------------------------------- */ + +void __init au_dy_init(void) +{ + int i; + + /* make sure that 'struct au_dykey *' can be any type */ + BUILD_BUG_ON(offsetof(struct au_dyaop, da_key)); + + for (i = 0; i < AuDyLast; i++) + INIT_HLIST_BL_HEAD(dynop + i); +} + +void au_dy_fin(void) +{ + int i; + + for (i = 0; i < AuDyLast; i++) + WARN_ON(!hlist_bl_empty(dynop + i)); +} diff --git a/fs/aufs/dynop.h b/fs/aufs/dynop.h new file mode 100644 index 0000000000000..f3301c2a6a9bf --- /dev/null +++ b/fs/aufs/dynop.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2010-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dynamically customizable operations (for regular files only) + */ + +#ifndef __AUFS_DYNOP_H__ +#define __AUFS_DYNOP_H__ + +#ifdef __KERNEL__ + +#include +#include + +enum {AuDy_AOP, AuDyLast}; + +struct au_dynop { + int dy_type; + union { + const void *dy_hop; + const struct address_space_operations *dy_haop; + }; +}; + +struct au_dykey { + union { + struct hlist_bl_node dk_hnode; + struct rcu_head dk_rcu; + }; + struct au_dynop dk_op; + + /* + * during I am in the branch local array, kref is gotten. when the + * branch is removed, kref is put. + */ + struct kref dk_kref; +}; + +/* stop unioning since their sizes are very different from each other */ +struct au_dyaop { + struct au_dykey da_key; + struct address_space_operations da_op; /* not const */ +}; + +/* ---------------------------------------------------------------------- */ + +/* dynop.c */ +struct au_branch; +void au_dy_put(struct au_dykey *key); +int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode); +int au_dy_irefresh(struct inode *inode); +void au_dy_arefresh(int do_dio); + +void __init au_dy_init(void); +void au_dy_fin(void); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DYNOP_H__ */ diff --git a/fs/aufs/export.c b/fs/aufs/export.c new file mode 100644 index 0000000000000..1da2ca0a969bc --- /dev/null +++ b/fs/aufs/export.c @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * export via nfs + */ + +#include +#include +#include +#include +#include +#include +#include "aufs.h" + +union conv { +#ifdef CONFIG_AUFS_INO_T_64 + __u32 a[2]; +#else + __u32 a[1]; +#endif + ino_t ino; +}; + +static ino_t decode_ino(__u32 *a) +{ + union conv u; + + BUILD_BUG_ON(sizeof(u.ino) != sizeof(u.a)); + u.a[0] = a[0]; +#ifdef CONFIG_AUFS_INO_T_64 + u.a[1] = a[1]; +#endif + return u.ino; +} + +static void encode_ino(__u32 *a, ino_t ino) +{ + union conv u; + + u.ino = ino; + a[0] = u.a[0]; +#ifdef CONFIG_AUFS_INO_T_64 + a[1] = u.a[1]; +#endif +} + +/* NFS file handle */ +enum { + Fh_br_id, + Fh_sigen, +#ifdef CONFIG_AUFS_INO_T_64 + /* support 64bit inode number */ + Fh_ino1, + Fh_ino2, + Fh_dir_ino1, + Fh_dir_ino2, +#else + Fh_ino1, + Fh_dir_ino1, +#endif + Fh_igen, + Fh_h_type, + Fh_tail, + + Fh_ino = Fh_ino1, + Fh_dir_ino = Fh_dir_ino1 +}; + +static int au_test_anon(struct dentry *dentry) +{ + /* note: read d_flags without d_lock */ + return !!(dentry->d_flags & DCACHE_DISCONNECTED); +} + +int au_test_nfsd(void) +{ + int ret; + struct task_struct *tsk = current; + char comm[sizeof(tsk->comm)]; + + ret = 0; + if (tsk->flags & PF_KTHREAD) { + get_task_comm(comm, tsk); + ret = !strcmp(comm, "nfsd"); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ +/* inode generation external table */ + +void au_xigen_inc(struct inode *inode) +{ + loff_t pos; + ssize_t sz; + __u32 igen; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + sb = inode->i_sb; + AuDebugOn(!au_opt_test(au_mntflags(sb), XINO)); + + sbinfo = au_sbi(sb); + pos = inode->i_ino; + pos *= sizeof(igen); + igen = inode->i_generation + 1; + sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xigen, &igen, + sizeof(igen), &pos); + if (sz == sizeof(igen)) + return; /* success */ + + if (unlikely(sz >= 0)) + AuIOErr("xigen error (%zd)\n", sz); +} + +int au_xigen_new(struct inode *inode) +{ + int err; + loff_t pos; + ssize_t sz; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct file *file; + + err = 0; + /* todo: dirty, at mount time */ + if (inode->i_ino == AUFS_ROOT_INO) + goto out; + sb = inode->i_sb; + SiMustAnyLock(sb); + if (unlikely(!au_opt_test(au_mntflags(sb), XINO))) + goto out; + + err = -EFBIG; + pos = inode->i_ino; + if (unlikely(au_loff_max / sizeof(inode->i_generation) - 1 < pos)) { + AuIOErr1("too large i%lld\n", pos); + goto out; + } + pos *= sizeof(inode->i_generation); + + err = 0; + sbinfo = au_sbi(sb); + file = sbinfo->si_xigen; + BUG_ON(!file); + + if (vfsub_f_size_read(file) + < pos + sizeof(inode->i_generation)) { + inode->i_generation = atomic_inc_return(&sbinfo->si_xigen_next); + sz = xino_fwrite(sbinfo->si_xwrite, file, &inode->i_generation, + sizeof(inode->i_generation), &pos); + } else + sz = xino_fread(sbinfo->si_xread, file, &inode->i_generation, + sizeof(inode->i_generation), &pos); + if (sz == sizeof(inode->i_generation)) + goto out; /* success */ + + err = sz; + if (unlikely(sz >= 0)) { + err = -EIO; + AuIOErr("xigen error (%zd)\n", sz); + } + +out: + return err; +} + +int au_xigen_set(struct super_block *sb, struct path *path) +{ + int err; + struct au_sbinfo *sbinfo; + struct file *file; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + file = au_xino_create2(sb, path, sbinfo->si_xigen); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + err = 0; + if (sbinfo->si_xigen) + fput(sbinfo->si_xigen); + sbinfo->si_xigen = file; + +out: + AuTraceErr(err); + return err; +} + +void au_xigen_clr(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + if (sbinfo->si_xigen) { + fput(sbinfo->si_xigen); + sbinfo->si_xigen = NULL; + } +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *decode_by_ino(struct super_block *sb, ino_t ino, + ino_t dir_ino) +{ + struct dentry *dentry, *d; + struct inode *inode; + unsigned int sigen; + + dentry = NULL; + inode = ilookup(sb, ino); + if (!inode) + goto out; + + dentry = ERR_PTR(-ESTALE); + sigen = au_sigen(sb); + if (unlikely(au_is_bad_inode(inode) + || IS_DEADDIR(inode) + || sigen != au_iigen(inode, NULL))) + goto out_iput; + + dentry = NULL; + if (!dir_ino || S_ISDIR(inode->i_mode)) + dentry = d_find_alias(inode); + else { + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) { + spin_lock(&d->d_lock); + if (!au_test_anon(d) + && d_inode(d->d_parent)->i_ino == dir_ino) { + dentry = dget_dlock(d); + spin_unlock(&d->d_lock); + break; + } + spin_unlock(&d->d_lock); + } + spin_unlock(&inode->i_lock); + } + if (unlikely(dentry && au_digen_test(dentry, sigen))) { + /* need to refresh */ + dput(dentry); + dentry = NULL; + } + +out_iput: + iput(inode); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +/* todo: dirty? */ +/* if exportfs_decode_fh() passed vfsmount*, we could be happy */ + +struct au_compare_mnt_args { + /* input */ + struct super_block *sb; + + /* output */ + struct vfsmount *mnt; +}; + +static int au_compare_mnt(struct vfsmount *mnt, void *arg) +{ + struct au_compare_mnt_args *a = arg; + + if (mnt->mnt_sb != a->sb) + return 0; + a->mnt = mntget(mnt); + return 1; +} + +static struct vfsmount *au_mnt_get(struct super_block *sb) +{ + int err; + struct path root; + struct au_compare_mnt_args args = { + .sb = sb + }; + + get_fs_root(current->fs, &root); + rcu_read_lock(); + err = iterate_mounts(au_compare_mnt, &args, root.mnt); + rcu_read_unlock(); + path_put(&root); + AuDebugOn(!err); + AuDebugOn(!args.mnt); + return args.mnt; +} + +struct au_nfsd_si_lock { + unsigned int sigen; + aufs_bindex_t bindex, br_id; + unsigned char force_lock; +}; + +static int si_nfsd_read_lock(struct super_block *sb, + struct au_nfsd_si_lock *nsi_lock) +{ + int err; + aufs_bindex_t bindex; + + si_read_lock(sb, AuLock_FLUSH); + + /* branch id may be wrapped around */ + err = 0; + bindex = au_br_index(sb, nsi_lock->br_id); + if (bindex >= 0 && nsi_lock->sigen + AUFS_BRANCH_MAX > au_sigen(sb)) + goto out; /* success */ + + err = -ESTALE; + bindex = -1; + if (!nsi_lock->force_lock) + si_read_unlock(sb); + +out: + nsi_lock->bindex = bindex; + return err; +} + +struct find_name_by_ino { + struct dir_context ctx; + int called, found; + ino_t ino; + char *name; + int namelen; +}; + +static int +find_name_by_ino(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct find_name_by_ino *a = container_of(ctx, struct find_name_by_ino, + ctx); + + a->called++; + if (a->ino != ino) + return 0; + + memcpy(a->name, name, namelen); + a->namelen = namelen; + a->found = 1; + return 1; +} + +static struct dentry *au_lkup_by_ino(struct path *path, ino_t ino, + struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry, *parent; + struct file *file; + struct inode *dir; + struct find_name_by_ino arg = { + .ctx = { + .actor = find_name_by_ino + } + }; + int err; + + parent = path->dentry; + if (nsi_lock) + si_read_unlock(parent->d_sb); + file = vfsub_dentry_open(path, au_dir_roflags); + dentry = (void *)file; + if (IS_ERR(file)) + goto out; + + dentry = ERR_PTR(-ENOMEM); + arg.name = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!arg.name)) + goto out_file; + arg.ino = ino; + arg.found = 0; + do { + arg.called = 0; + /* smp_mb(); */ + err = vfsub_iterate_dir(file, &arg.ctx); + } while (!err && !arg.found && arg.called); + dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_name; + /* instead of ENOENT */ + dentry = ERR_PTR(-ESTALE); + if (!arg.found) + goto out_name; + + /* do not call vfsub_lkup_one() */ + dir = d_inode(parent); + dentry = vfsub_lookup_one_len_unlocked(arg.name, parent, arg.namelen); + AuTraceErrPtr(dentry); + if (IS_ERR(dentry)) + goto out_name; + AuDebugOn(au_test_anon(dentry)); + if (unlikely(d_really_is_negative(dentry))) { + dput(dentry); + dentry = ERR_PTR(-ENOENT); + } + +out_name: + free_page((unsigned long)arg.name); +out_file: + fput(file); +out: + if (unlikely(nsi_lock + && si_nfsd_read_lock(parent->d_sb, nsi_lock) < 0)) + if (!IS_ERR(dentry)) { + dput(dentry); + dentry = ERR_PTR(-ESTALE); + } + AuTraceErrPtr(dentry); + return dentry; +} + +static struct dentry *decode_by_dir_ino(struct super_block *sb, ino_t ino, + ino_t dir_ino, + struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry; + struct path path; + + if (dir_ino != AUFS_ROOT_INO) { + path.dentry = decode_by_ino(sb, dir_ino, 0); + dentry = path.dentry; + if (!path.dentry || IS_ERR(path.dentry)) + goto out; + AuDebugOn(au_test_anon(path.dentry)); + } else + path.dentry = dget(sb->s_root); + + path.mnt = au_mnt_get(sb); + dentry = au_lkup_by_ino(&path, ino, nsi_lock); + path_put(&path); + +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +static int h_acceptable(void *expv, struct dentry *dentry) +{ + return 1; +} + +static char *au_build_path(struct dentry *h_parent, struct path *h_rootpath, + char *buf, int len, struct super_block *sb) +{ + char *p; + int n; + struct path path; + + p = d_path(h_rootpath, buf, len); + if (IS_ERR(p)) + goto out; + n = strlen(p); + + path.mnt = h_rootpath->mnt; + path.dentry = h_parent; + p = d_path(&path, buf, len); + if (IS_ERR(p)) + goto out; + if (n != 1) + p += n; + + path.mnt = au_mnt_get(sb); + path.dentry = sb->s_root; + p = d_path(&path, buf, len - strlen(p)); + mntput(path.mnt); + if (IS_ERR(p)) + goto out; + if (n != 1) + p[strlen(p)] = '/'; + +out: + AuTraceErrPtr(p); + return p; +} + +static +struct dentry *decode_by_path(struct super_block *sb, ino_t ino, __u32 *fh, + int fh_len, struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry, *h_parent, *root; + struct super_block *h_sb; + char *pathname, *p; + struct vfsmount *h_mnt; + struct au_branch *br; + int err; + struct path path; + + br = au_sbr(sb, nsi_lock->bindex); + h_mnt = au_br_mnt(br); + h_sb = h_mnt->mnt_sb; + /* todo: call lower fh_to_dentry()? fh_to_parent()? */ + lockdep_off(); + h_parent = exportfs_decode_fh(h_mnt, (void *)(fh + Fh_tail), + fh_len - Fh_tail, fh[Fh_h_type], + h_acceptable, /*context*/NULL); + lockdep_on(); + dentry = h_parent; + if (unlikely(!h_parent || IS_ERR(h_parent))) { + AuWarn1("%s decode_fh failed, %ld\n", + au_sbtype(h_sb), PTR_ERR(h_parent)); + goto out; + } + dentry = NULL; + if (unlikely(au_test_anon(h_parent))) { + AuWarn1("%s decode_fh returned a disconnected dentry\n", + au_sbtype(h_sb)); + goto out_h_parent; + } + + dentry = ERR_PTR(-ENOMEM); + pathname = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!pathname)) + goto out_h_parent; + + root = sb->s_root; + path.mnt = h_mnt; + di_read_lock_parent(root, !AuLock_IR); + path.dentry = au_h_dptr(root, nsi_lock->bindex); + di_read_unlock(root, !AuLock_IR); + p = au_build_path(h_parent, &path, pathname, PAGE_SIZE, sb); + dentry = (void *)p; + if (IS_ERR(p)) + goto out_pathname; + + si_read_unlock(sb); + err = vfsub_kern_path(p, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_relock; + + dentry = ERR_PTR(-ENOENT); + AuDebugOn(au_test_anon(path.dentry)); + if (unlikely(d_really_is_negative(path.dentry))) + goto out_path; + + if (ino != d_inode(path.dentry)->i_ino) + dentry = au_lkup_by_ino(&path, ino, /*nsi_lock*/NULL); + else + dentry = dget(path.dentry); + +out_path: + path_put(&path); +out_relock: + if (unlikely(si_nfsd_read_lock(sb, nsi_lock) < 0)) + if (!IS_ERR(dentry)) { + dput(dentry); + dentry = ERR_PTR(-ESTALE); + } +out_pathname: + free_page((unsigned long)pathname); +out_h_parent: + dput(h_parent); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry * +aufs_fh_to_dentry(struct super_block *sb, struct fid *fid, int fh_len, + int fh_type) +{ + struct dentry *dentry; + __u32 *fh = fid->raw; + struct au_branch *br; + ino_t ino, dir_ino; + struct au_nfsd_si_lock nsi_lock = { + .force_lock = 0 + }; + + dentry = ERR_PTR(-ESTALE); + /* it should never happen, but the file handle is unreliable */ + if (unlikely(fh_len < Fh_tail)) + goto out; + nsi_lock.sigen = fh[Fh_sigen]; + nsi_lock.br_id = fh[Fh_br_id]; + + /* branch id may be wrapped around */ + br = NULL; + if (unlikely(si_nfsd_read_lock(sb, &nsi_lock))) + goto out; + nsi_lock.force_lock = 1; + + /* is this inode still cached? */ + ino = decode_ino(fh + Fh_ino); + /* it should never happen */ + if (unlikely(ino == AUFS_ROOT_INO)) + goto out_unlock; + + dir_ino = decode_ino(fh + Fh_dir_ino); + dentry = decode_by_ino(sb, ino, dir_ino); + if (IS_ERR(dentry)) + goto out_unlock; + if (dentry) + goto accept; + + /* is the parent dir cached? */ + br = au_sbr(sb, nsi_lock.bindex); + au_lcnt_inc(&br->br_nfiles); + dentry = decode_by_dir_ino(sb, ino, dir_ino, &nsi_lock); + if (IS_ERR(dentry)) + goto out_unlock; + if (dentry) + goto accept; + + /* lookup path */ + dentry = decode_by_path(sb, ino, fh, fh_len, &nsi_lock); + if (IS_ERR(dentry)) + goto out_unlock; + if (unlikely(!dentry)) + /* todo?: make it ESTALE */ + goto out_unlock; + +accept: + if (!au_digen_test(dentry, au_sigen(sb)) + && d_inode(dentry)->i_generation == fh[Fh_igen]) + goto out_unlock; /* success */ + + dput(dentry); + dentry = ERR_PTR(-ESTALE); +out_unlock: + if (br) + au_lcnt_dec(&br->br_nfiles); + si_read_unlock(sb); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +#if 0 /* reserved for future use */ +/* support subtreecheck option */ +static struct dentry *aufs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct dentry *parent; + __u32 *fh = fid->raw; + ino_t dir_ino; + + dir_ino = decode_ino(fh + Fh_dir_ino); + parent = decode_by_ino(sb, dir_ino, 0); + if (IS_ERR(parent)) + goto out; + if (!parent) + parent = decode_by_path(sb, au_br_index(sb, fh[Fh_br_id]), + dir_ino, fh, fh_len); + +out: + AuTraceErrPtr(parent); + return parent; +} +#endif + +/* ---------------------------------------------------------------------- */ + +static int aufs_encode_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *dir) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb, *h_sb; + struct dentry *dentry, *parent, *h_parent; + struct inode *h_dir; + struct au_branch *br; + + err = -ENOSPC; + if (unlikely(*max_len <= Fh_tail)) { + AuWarn1("NFSv2 client (max_len %d)?\n", *max_len); + goto out; + } + + err = FILEID_ROOT; + if (inode->i_ino == AUFS_ROOT_INO) { + AuDebugOn(inode->i_ino != AUFS_ROOT_INO); + goto out; + } + + h_parent = NULL; + sb = inode->i_sb; + err = si_read_lock(sb, AuLock_FLUSH); + if (unlikely(err)) + goto out; + +#ifdef CONFIG_AUFS_DEBUG + if (unlikely(!au_opt_test(au_mntflags(sb), XINO))) + AuWarn1("NFS-exporting requires xino\n"); +#endif + err = -EIO; + parent = NULL; + ii_read_lock_child(inode); + bindex = au_ibtop(inode); + if (!dir) { + dentry = d_find_any_alias(inode); + if (unlikely(!dentry)) + goto out_unlock; + AuDebugOn(au_test_anon(dentry)); + parent = dget_parent(dentry); + dput(dentry); + if (unlikely(!parent)) + goto out_unlock; + if (d_really_is_positive(parent)) + dir = d_inode(parent); + } + + ii_read_lock_parent(dir); + h_dir = au_h_iptr(dir, bindex); + ii_read_unlock(dir); + if (unlikely(!h_dir)) + goto out_parent; + h_parent = d_find_any_alias(h_dir); + if (unlikely(!h_parent)) + goto out_hparent; + + err = -EPERM; + br = au_sbr(sb, bindex); + h_sb = au_br_sb(br); + if (unlikely(!h_sb->s_export_op)) { + AuErr1("%s branch is not exportable\n", au_sbtype(h_sb)); + goto out_hparent; + } + + fh[Fh_br_id] = br->br_id; + fh[Fh_sigen] = au_sigen(sb); + encode_ino(fh + Fh_ino, inode->i_ino); + encode_ino(fh + Fh_dir_ino, dir->i_ino); + fh[Fh_igen] = inode->i_generation; + + *max_len -= Fh_tail; + fh[Fh_h_type] = exportfs_encode_fh(h_parent, (void *)(fh + Fh_tail), + max_len, + /*connectable or subtreecheck*/0); + err = fh[Fh_h_type]; + *max_len += Fh_tail; + /* todo: macros? */ + if (err != FILEID_INVALID) + err = 99; + else + AuWarn1("%s encode_fh failed\n", au_sbtype(h_sb)); + +out_hparent: + dput(h_parent); +out_parent: + dput(parent); +out_unlock: + ii_read_unlock(inode); + si_read_unlock(sb); +out: + if (unlikely(err < 0)) + err = FILEID_INVALID; + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_commit_metadata(struct inode *inode) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; + int (*f)(struct inode *inode); + + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + ii_write_lock_child(inode); + bindex = au_ibtop(inode); + AuDebugOn(bindex < 0); + h_inode = au_h_iptr(inode, bindex); + + f = h_inode->i_sb->s_export_op->commit_metadata; + if (f) + err = f(h_inode); + else { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0 /* metadata only */ + }; + + err = sync_inode(h_inode, &wbc); + } + + au_cpup_attr_timesizes(inode); + ii_write_unlock(inode); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct export_operations aufs_export_op = { + .fh_to_dentry = aufs_fh_to_dentry, + /* .fh_to_parent = aufs_fh_to_parent, */ + .encode_fh = aufs_encode_fh, + .commit_metadata = aufs_commit_metadata +}; + +void au_export_init(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + __u32 u; + + BUILD_BUG_ON_MSG(IS_BUILTIN(CONFIG_AUFS_FS) + && IS_MODULE(CONFIG_EXPORTFS), + AUFS_NAME ": unsupported configuration " + "CONFIG_EXPORTFS=m and CONFIG_AUFS_FS=y"); + + sb->s_export_op = &aufs_export_op; + sbinfo = au_sbi(sb); + sbinfo->si_xigen = NULL; + get_random_bytes(&u, sizeof(u)); + BUILD_BUG_ON(sizeof(u) != sizeof(int)); + atomic_set(&sbinfo->si_xigen_next, u); +} diff --git a/fs/aufs/f_op.c b/fs/aufs/f_op.c new file mode 100644 index 0000000000000..5254b5cd39426 --- /dev/null +++ b/fs/aufs/f_op.c @@ -0,0 +1,819 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file and vm operations + */ + +#include +#include +#include +#include +#include "aufs.h" + +int au_do_open_nondir(struct file *file, int flags, struct file *h_file) +{ + int err; + aufs_bindex_t bindex; + struct dentry *dentry, *h_dentry; + struct au_finfo *finfo; + struct inode *h_inode; + + FiMustWriteLock(file); + + err = 0; + dentry = file->f_path.dentry; + AuDebugOn(IS_ERR_OR_NULL(dentry)); + finfo = au_fi(file); + memset(&finfo->fi_htop, 0, sizeof(finfo->fi_htop)); + atomic_set(&finfo->fi_mmapped, 0); + bindex = au_dbtop(dentry); + if (!h_file) { + h_dentry = au_h_dptr(dentry, bindex); + err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb); + if (unlikely(err)) + goto out; + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + if (IS_ERR(h_file)) { + err = PTR_ERR(h_file); + goto out; + } + } else { + h_dentry = h_file->f_path.dentry; + err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb); + if (unlikely(err)) + goto out; + /* br ref is already inc-ed */ + } + + if ((flags & __O_TMPFILE) + && !(flags & O_EXCL)) { + h_inode = file_inode(h_file); + spin_lock(&h_inode->i_lock); + h_inode->i_state |= I_LINKABLE; + spin_unlock(&h_inode->i_lock); + } + au_set_fbtop(file, bindex); + au_set_h_fptr(file, bindex, h_file); + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + +out: + return err; +} + +static int aufs_open_nondir(struct inode *inode __maybe_unused, + struct file *file) +{ + int err; + struct super_block *sb; + struct au_do_open_args args = { + .open = au_do_open_nondir + }; + + AuDbg("%pD, f_flags 0x%x, f_mode 0x%x\n", + file, vfsub_file_flags(file), file->f_mode); + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + err = au_do_open(file, &args); + si_read_unlock(sb); + return err; +} + +int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file) +{ + struct au_finfo *finfo; + aufs_bindex_t bindex; + + finfo = au_fi(file); + au_hbl_del(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + bindex = finfo->fi_btop; + if (bindex >= 0) + au_set_h_fptr(file, bindex, NULL); + + au_finfo_fin(file); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_flush_nondir(struct file *file, fl_owner_t id) +{ + int err; + struct file *h_file; + + err = 0; + h_file = au_hf_top(file); + if (h_file) + err = vfsub_flush(h_file, id); + return err; +} + +static int aufs_flush_nondir(struct file *file, fl_owner_t id) +{ + return au_do_flush(file, id, au_do_flush_nondir); +} + +/* ---------------------------------------------------------------------- */ +/* + * read and write functions acquire [fdi]_rwsem once, but release before + * mmap_sem. This is because to stop a race condition between mmap(2). + * Releasing these aufs-rwsem should be safe, no branch-management (by keeping + * si_rwsem), no harmful copy-up should happen. Actually copy-up may happen in + * read functions after [fdi]_rwsem are released, but it should be harmless. + */ + +/* Callers should call au_read_post() or fput() in the end */ +struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc) +{ + struct file *h_file; + int err; + + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/0, lsc); + if (!err) { + di_read_unlock(file->f_path.dentry, AuLock_IR); + h_file = au_hf_top(file); + get_file(h_file); + if (!keep_fi) + fi_read_unlock(file); + } else + h_file = ERR_PTR(err); + + return h_file; +} + +static void au_read_post(struct inode *inode, struct file *h_file) +{ + /* update without lock, I don't think it a problem */ + fsstack_copy_attr_atime(inode, file_inode(h_file)); + fput(h_file); +} + +struct au_write_pre { + /* input */ + unsigned int lsc; + + /* output */ + blkcnt_t blks; + aufs_bindex_t btop; +}; + +/* + * return with iinfo is write-locked + * callers should call au_write_post() or iinfo_write_unlock() + fput() in the + * end + */ +static struct file *au_write_pre(struct file *file, int do_ready, + struct au_write_pre *wpre) +{ + struct file *h_file; + struct dentry *dentry; + int err; + unsigned int lsc; + struct au_pin pin; + + lsc = 0; + if (wpre) + lsc = wpre->lsc; + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1, lsc); + h_file = ERR_PTR(err); + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + if (do_ready) { + err = au_ready_to_write(file, -1, &pin); + if (unlikely(err)) { + h_file = ERR_PTR(err); + di_write_unlock(dentry); + goto out_fi; + } + } + + di_downgrade_lock(dentry, /*flags*/0); + if (wpre) + wpre->btop = au_fbtop(file); + h_file = au_hf_top(file); + get_file(h_file); + if (wpre) + wpre->blks = file_inode(h_file)->i_blocks; + if (do_ready) + au_unpin(&pin); + di_read_unlock(dentry, /*flags*/0); + +out_fi: + fi_write_unlock(file); +out: + return h_file; +} + +static void au_write_post(struct inode *inode, struct file *h_file, + struct au_write_pre *wpre, ssize_t written) +{ + struct inode *h_inode; + + au_cpup_attr_timesizes(inode); + AuDebugOn(au_ibtop(inode) != wpre->btop); + h_inode = file_inode(h_file); + inode->i_mode = h_inode->i_mode; + ii_write_unlock(inode); + /* AuDbg("blks %llu, %llu\n", (u64)blks, (u64)h_inode->i_blocks); */ + if (written > 0) + au_fhsm_wrote(inode->i_sb, wpre->btop, + /*force*/h_inode->i_blocks > wpre->blks); + fput(h_file); +} + +static ssize_t aufs_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + ssize_t err; + struct inode *inode; + struct file *h_file; + struct super_block *sb; + + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + /* filedata may be obsoleted by concurrent copyup, but no problem */ + err = vfsub_read_u(h_file, buf, count, ppos); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +/* + * todo: very ugly + * it locks both of i_mutex and si_rwsem for read in safe. + * if the plink maintenance mode continues forever (that is the problem), + * may loop forever. + */ +static void au_mtx_and_read_lock(struct inode *inode) +{ + int err; + struct super_block *sb = inode->i_sb; + + while (1) { + inode_lock(inode); + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (!err) + break; + inode_unlock(inode); + si_read_lock(sb, AuLock_NOPLMW); + si_read_unlock(sb); + } +} + +static ssize_t aufs_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + char __user *buf = (char __user *)ubuf; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_write_u(h_file, buf, count, ppos); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t au_do_iter(struct file *h_file, int rw, struct kiocb *kio, + struct iov_iter *iov_iter) +{ + ssize_t err; + struct file *file; + ssize_t (*iter)(struct kiocb *, struct iov_iter *); + + err = security_file_permission(h_file, rw); + if (unlikely(err)) + goto out; + + err = -ENOSYS; + iter = NULL; + if (rw == MAY_READ) + iter = h_file->f_op->read_iter; + else if (rw == MAY_WRITE) + iter = h_file->f_op->write_iter; + + file = kio->ki_filp; + kio->ki_filp = h_file; + if (iter) { + lockdep_off(); + err = iter(kio, iov_iter); + lockdep_on(); + } else + /* currently there is no such fs */ + WARN_ON_ONCE(1); + kio->ki_filp = file; + +out: + return err; +} + +static ssize_t aufs_read_iter(struct kiocb *kio, struct iov_iter *iov_iter) +{ + ssize_t err; + struct file *file, *h_file; + struct inode *inode; + struct super_block *sb; + + file = kio->ki_filp; + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/1, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + if (au_test_loopback_kthread()) { + au_warn_loopback(h_file->f_path.dentry->d_sb); + if (file->f_mapping != h_file->f_mapping) { + file->f_mapping = h_file->f_mapping; + smp_mb(); /* unnecessary? */ + } + } + fi_read_unlock(file); + + err = au_do_iter(h_file, MAY_READ, kio, iov_iter); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +static ssize_t aufs_write_iter(struct kiocb *kio, struct iov_iter *iov_iter) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *file, *h_file; + + file = kio->ki_filp; + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = au_do_iter(h_file, MAY_WRITE, kio, iov_iter); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t aufs_splice_read(struct file *file, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + ssize_t err; + struct file *h_file; + struct inode *inode; + struct super_block *sb; + + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_splice_to(h_file, ppos, pipe, len, flags); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +static ssize_t +aufs_splice_write(struct pipe_inode_info *pipe, struct file *file, loff_t *ppos, + size_t len, unsigned int flags) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_splice_from(pipe, h_file, ppos, len, flags); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static long aufs_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + long err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + lockdep_off(); + err = vfs_fallocate(h_file, mode, offset, len); + lockdep_on(); + au_write_post(inode, h_file, &wpre, /*written*/1); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t aufs_copy_file_range(struct file *src, loff_t src_pos, + struct file *dst, loff_t dst_pos, + size_t len, unsigned int flags) +{ + ssize_t err; + struct au_write_pre wpre; + enum { SRC, DST }; + struct { + struct inode *inode; + struct file *h_file; + struct super_block *h_sb; + } a[2]; +#define a_src a[SRC] +#define a_dst a[DST] + + err = -EINVAL; + a_src.inode = file_inode(src); + if (unlikely(!S_ISREG(a_src.inode->i_mode))) + goto out; + a_dst.inode = file_inode(dst); + if (unlikely(!S_ISREG(a_dst.inode->i_mode))) + goto out; + + au_mtx_and_read_lock(a_dst.inode); + /* + * in order to match the order in di_write_lock2_{child,parent}(), + * use f_path.dentry for this comparison. + */ + if (src->f_path.dentry < dst->f_path.dentry) { + a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_1); + err = PTR_ERR(a_src.h_file); + if (IS_ERR(a_src.h_file)) + goto out_si; + + wpre.lsc = AuLsc_FI_2; + a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre); + err = PTR_ERR(a_dst.h_file); + if (IS_ERR(a_dst.h_file)) { + au_read_post(a_src.inode, a_src.h_file); + goto out_si; + } + } else { + wpre.lsc = AuLsc_FI_1; + a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre); + err = PTR_ERR(a_dst.h_file); + if (IS_ERR(a_dst.h_file)) + goto out_si; + + a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_2); + err = PTR_ERR(a_src.h_file); + if (IS_ERR(a_src.h_file)) { + au_write_post(a_dst.inode, a_dst.h_file, &wpre, + /*written*/0); + goto out_si; + } + } + + err = -EXDEV; + a_src.h_sb = file_inode(a_src.h_file)->i_sb; + a_dst.h_sb = file_inode(a_dst.h_file)->i_sb; + if (unlikely(a_src.h_sb != a_dst.h_sb)) { + AuDbgFile(src); + AuDbgFile(dst); + goto out_file; + } + + err = vfsub_copy_file_range(a_src.h_file, src_pos, a_dst.h_file, + dst_pos, len, flags); + +out_file: + au_write_post(a_dst.inode, a_dst.h_file, &wpre, err); + fi_read_unlock(src); + au_read_post(a_src.inode, a_src.h_file); +out_si: + si_read_unlock(a_dst.inode->i_sb); + inode_unlock(a_dst.inode); +out: + return err; +#undef a_src +#undef a_dst +} + +/* ---------------------------------------------------------------------- */ + +/* + * The locking order around current->mmap_sem. + * - in most and regular cases + * file I/O syscall -- aufs_read() or something + * -- si_rwsem for read -- mmap_sem + * (Note that [fdi]i_rwsem are released before mmap_sem). + * - in mmap case + * mmap(2) -- mmap_sem -- aufs_mmap() -- si_rwsem for read -- [fdi]i_rwsem + * This AB-BA order is definitely bad, but is not a problem since "si_rwsem for + * read" allows multiple processes to acquire it and [fdi]i_rwsem are not held + * in file I/O. Aufs needs to stop lockdep in aufs_mmap() though. + * It means that when aufs acquires si_rwsem for write, the process should never + * acquire mmap_sem. + * + * Actually aufs_iterate() holds [fdi]i_rwsem before mmap_sem, but this is not a + * problem either since any directory is not able to be mmap-ed. + * The similar scenario is applied to aufs_readlink() too. + */ + +#if 0 /* stop calling security_file_mmap() */ +/* cf. linux/include/linux/mman.h: calc_vm_prot_bits() */ +#define AuConv_VM_PROT(f, b) _calc_vm_trans(f, VM_##b, PROT_##b) + +static unsigned long au_arch_prot_conv(unsigned long flags) +{ + /* currently ppc64 only */ +#ifdef CONFIG_PPC64 + /* cf. linux/arch/powerpc/include/asm/mman.h */ + AuDebugOn(arch_calc_vm_prot_bits(-1) != VM_SAO); + return AuConv_VM_PROT(flags, SAO); +#else + AuDebugOn(arch_calc_vm_prot_bits(-1)); + return 0; +#endif +} + +static unsigned long au_prot_conv(unsigned long flags) +{ + return AuConv_VM_PROT(flags, READ) + | AuConv_VM_PROT(flags, WRITE) + | AuConv_VM_PROT(flags, EXEC) + | au_arch_prot_conv(flags); +} + +/* cf. linux/include/linux/mman.h: calc_vm_flag_bits() */ +#define AuConv_VM_MAP(f, b) _calc_vm_trans(f, VM_##b, MAP_##b) + +static unsigned long au_flag_conv(unsigned long flags) +{ + return AuConv_VM_MAP(flags, GROWSDOWN) + | AuConv_VM_MAP(flags, DENYWRITE) + | AuConv_VM_MAP(flags, LOCKED); +} +#endif + +static int aufs_mmap(struct file *file, struct vm_area_struct *vma) +{ + int err; + const unsigned char wlock + = (file->f_mode & FMODE_WRITE) && (vma->vm_flags & VM_SHARED); + struct super_block *sb; + struct file *h_file; + struct inode *inode; + + AuDbgVmRegion(file, vma); + + inode = file_inode(file); + sb = inode->i_sb; + lockdep_off(); + si_read_lock(sb, AuLock_NOPLMW); + + h_file = au_write_pre(file, wlock, /*wpre*/NULL); + lockdep_on(); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = 0; + au_set_mmapped(file); + au_vm_file_reset(vma, h_file); + /* + * we cannot call security_mmap_file() here since it may acquire + * mmap_sem or i_mutex. + * + * err = security_mmap_file(h_file, au_prot_conv(vma->vm_flags), + * au_flag_conv(vma->vm_flags)); + */ + if (!err) + err = call_mmap(h_file, vma); + if (!err) { + au_vm_prfile_set(vma, file); + fsstack_copy_attr_atime(inode, file_inode(h_file)); + goto out_fput; /* success */ + } + au_unset_mmapped(file); + au_vm_file_reset(vma, file); + +out_fput: + lockdep_off(); + ii_write_unlock(inode); + lockdep_on(); + fput(h_file); +out: + lockdep_off(); + si_read_unlock(sb); + lockdep_on(); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_fsync_nondir(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + err = 0; /* -EBADF; */ /* posix? */ + if (unlikely(!(file->f_mode & FMODE_WRITE))) + goto out; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out_unlock; + + err = vfsub_fsync(h_file, &h_file->f_path, datasync); + au_write_post(inode, h_file, &wpre, /*written*/0); + +out_unlock: + si_read_unlock(inode->i_sb); + inode_unlock(inode); +out: + return err; +} + +static int aufs_fasync(int fd, struct file *file, int flag) +{ + int err; + struct file *h_file; + struct super_block *sb; + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + if (h_file->f_op->fasync) + err = h_file->f_op->fasync(fd, h_file, flag); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + return err; +} + +static int aufs_setfl(struct file *file, unsigned long arg) +{ + int err; + struct file *h_file; + struct super_block *sb; + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + /* stop calling h_file->fasync */ + arg |= vfsub_file_flags(file) & FASYNC; + err = setfl(/*unused fd*/-1, h_file, arg); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* no one supports this operation, currently */ +#if 0 +static ssize_t aufs_sendpage(struct file *file, struct page *page, int offset, + size_t len, loff_t *pos, int more) +{ +} +#endif + +/* ---------------------------------------------------------------------- */ + +const struct file_operations aufs_file_fop = { + .owner = THIS_MODULE, + + .llseek = default_llseek, + + .read = aufs_read, + .write = aufs_write, + .read_iter = aufs_read_iter, + .write_iter = aufs_write_iter, + +#ifdef CONFIG_AUFS_POLL + .poll = aufs_poll, +#endif + .unlocked_ioctl = aufs_ioctl_nondir, +#ifdef CONFIG_COMPAT + .compat_ioctl = aufs_compat_ioctl_nondir, +#endif + .mmap = aufs_mmap, + .open = aufs_open_nondir, + .flush = aufs_flush_nondir, + .release = aufs_release_nondir, + .fsync = aufs_fsync_nondir, + .fasync = aufs_fasync, + /* .sendpage = aufs_sendpage, */ + .setfl = aufs_setfl, + .splice_write = aufs_splice_write, + .splice_read = aufs_splice_read, +#if 0 + .aio_splice_write = aufs_aio_splice_write, + .aio_splice_read = aufs_aio_splice_read, +#endif + .fallocate = aufs_fallocate, + .copy_file_range = aufs_copy_file_range +}; diff --git a/fs/aufs/fhsm.c b/fs/aufs/fhsm.c new file mode 100644 index 0000000000000..d5fca9596b795 --- /dev/null +++ b/fs/aufs/fhsm.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * File-based Hierarchy Storage Management + */ + +#include +#include +#include +#include +#include "aufs.h" + +static aufs_bindex_t au_fhsm_bottom(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + AuDebugOn(!fhsm); + return fhsm->fhsm_bottom; +} + +void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + AuDebugOn(!fhsm); + fhsm->fhsm_bottom = bindex; +} + +/* ---------------------------------------------------------------------- */ + +static int au_fhsm_test_jiffy(struct au_sbinfo *sbinfo, struct au_branch *br) +{ + struct au_br_fhsm *bf; + + bf = br->br_fhsm; + MtxMustLock(&bf->bf_lock); + + return !bf->bf_readable + || time_after(jiffies, + bf->bf_jiffy + sbinfo->si_fhsm.fhsm_expire); +} + +/* ---------------------------------------------------------------------- */ + +static void au_fhsm_notify(struct super_block *sb, int val) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + if (au_fhsm_pid(fhsm) + && atomic_read(&fhsm->fhsm_readable) != -1) { + atomic_set(&fhsm->fhsm_readable, val); + if (val) + wake_up(&fhsm->fhsm_wqh); + } +} + +static int au_fhsm_stfs(struct super_block *sb, aufs_bindex_t bindex, + struct aufs_stfs *rstfs, int do_lock, int do_notify) +{ + int err; + struct au_branch *br; + struct au_br_fhsm *bf; + + br = au_sbr(sb, bindex); + AuDebugOn(au_br_rdonly(br)); + bf = br->br_fhsm; + AuDebugOn(!bf); + + if (do_lock) + mutex_lock(&bf->bf_lock); + else + MtxMustLock(&bf->bf_lock); + + /* sb->s_root for NFS is unreliable */ + err = au_br_stfs(br, &bf->bf_stfs); + if (unlikely(err)) { + AuErr1("FHSM failed (%d), b%d, ignored.\n", bindex, err); + goto out; + } + + bf->bf_jiffy = jiffies; + bf->bf_readable = 1; + if (do_notify) + au_fhsm_notify(sb, /*val*/1); + if (rstfs) + *rstfs = bf->bf_stfs; + +out: + if (do_lock) + mutex_unlock(&bf->bf_lock); + au_fhsm_notify(sb, /*val*/1); + + return err; +} + +void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force) +{ + int err; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + struct au_branch *br; + struct au_br_fhsm *bf; + + AuDbg("b%d, force %d\n", bindex, force); + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + if (!au_ftest_si(sbinfo, FHSM) + || fhsm->fhsm_bottom == bindex) + return; + + br = au_sbr(sb, bindex); + bf = br->br_fhsm; + AuDebugOn(!bf); + mutex_lock(&bf->bf_lock); + if (force + || au_fhsm_pid(fhsm) + || au_fhsm_test_jiffy(sbinfo, br)) + err = au_fhsm_stfs(sb, bindex, /*rstfs*/NULL, /*do_lock*/0, + /*do_notify*/1); + mutex_unlock(&bf->bf_lock); +} + +void au_fhsm_wrote_all(struct super_block *sb, int force) +{ + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + /* exclude the bottom */ + bbot = au_fhsm_bottom(sb); + for (bindex = 0; bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) + au_fhsm_wrote(sb, bindex, force); + } +} + +/* ---------------------------------------------------------------------- */ + +static unsigned int au_fhsm_poll(struct file *file, + struct poll_table_struct *wait) +{ + unsigned int mask; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + mask = 0; + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; + poll_wait(file, &fhsm->fhsm_wqh, wait); + if (atomic_read(&fhsm->fhsm_readable)) + mask = POLLIN /* | POLLRDNORM */; + + if (!mask) + AuDbg("mask 0x%x\n", mask); + return mask; +} + +static int au_fhsm_do_read_one(struct aufs_stbr __user *stbr, + struct aufs_stfs *stfs, __s16 brid) +{ + int err; + + err = copy_to_user(&stbr->stfs, stfs, sizeof(*stfs)); + if (!err) + err = __put_user(brid, &stbr->brid); + if (unlikely(err)) + err = -EFAULT; + + return err; +} + +static ssize_t au_fhsm_do_read(struct super_block *sb, + struct aufs_stbr __user *stbr, size_t count) +{ + ssize_t err; + int nstbr; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + struct au_br_fhsm *bf; + + /* except the bottom branch */ + err = 0; + nstbr = 0; + bbot = au_fhsm_bottom(sb); + for (bindex = 0; !err && bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_fhsm(br->br_perm)) + continue; + + bf = br->br_fhsm; + mutex_lock(&bf->bf_lock); + if (bf->bf_readable) { + err = -EFAULT; + if (count >= sizeof(*stbr)) + err = au_fhsm_do_read_one(stbr++, &bf->bf_stfs, + br->br_id); + if (!err) { + bf->bf_readable = 0; + count -= sizeof(*stbr); + nstbr++; + } + } + mutex_unlock(&bf->bf_lock); + } + if (!err) + err = sizeof(*stbr) * nstbr; + + return err; +} + +static ssize_t au_fhsm_read(struct file *file, char __user *buf, size_t count, + loff_t *pos) +{ + ssize_t err; + int readable; + aufs_bindex_t nfhsm, bindex, bbot; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + struct au_branch *br; + struct super_block *sb; + + err = 0; + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; +need_data: + spin_lock_irq(&fhsm->fhsm_wqh.lock); + if (!atomic_read(&fhsm->fhsm_readable)) { + if (vfsub_file_flags(file) & O_NONBLOCK) + err = -EAGAIN; + else + err = wait_event_interruptible_locked_irq + (fhsm->fhsm_wqh, + atomic_read(&fhsm->fhsm_readable)); + } + spin_unlock_irq(&fhsm->fhsm_wqh.lock); + if (unlikely(err)) + goto out; + + /* sb may already be dead */ + au_rw_read_lock(&sbinfo->si_rwsem); + readable = atomic_read(&fhsm->fhsm_readable); + if (readable > 0) { + sb = sbinfo->si_sb; + AuDebugOn(!sb); + /* exclude the bottom branch */ + nfhsm = 0; + bbot = au_fhsm_bottom(sb); + for (bindex = 0; bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) + nfhsm++; + } + err = -EMSGSIZE; + if (nfhsm * sizeof(struct aufs_stbr) <= count) { + atomic_set(&fhsm->fhsm_readable, 0); + err = au_fhsm_do_read(sbinfo->si_sb, (void __user *)buf, + count); + } + } + au_rw_read_unlock(&sbinfo->si_rwsem); + if (!readable) + goto need_data; + +out: + return err; +} + +static int au_fhsm_release(struct inode *inode, struct file *file) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + /* sb may already be dead */ + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; + spin_lock(&fhsm->fhsm_spin); + fhsm->fhsm_pid = 0; + spin_unlock(&fhsm->fhsm_spin); + kobject_put(&sbinfo->si_kobj); + + return 0; +} + +static const struct file_operations au_fhsm_fops = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .read = au_fhsm_read, + .poll = au_fhsm_poll, + .release = au_fhsm_release +}; + +int au_fhsm_fd(struct super_block *sb, int oflags) +{ + int err, fd; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -EINVAL; + if (unlikely(oflags & ~(O_CLOEXEC | O_NONBLOCK))) + goto out; + + err = 0; + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + spin_lock(&fhsm->fhsm_spin); + if (!fhsm->fhsm_pid) + fhsm->fhsm_pid = current->pid; + else + err = -EBUSY; + spin_unlock(&fhsm->fhsm_spin); + if (unlikely(err)) + goto out; + + oflags |= O_RDONLY; + /* oflags |= FMODE_NONOTIFY; */ + fd = anon_inode_getfd("[aufs_fhsm]", &au_fhsm_fops, sbinfo, oflags); + err = fd; + if (unlikely(fd < 0)) + goto out_pid; + + /* succeed regardless 'fhsm' status */ + kobject_get(&sbinfo->si_kobj); + si_noflush_read_lock(sb); + if (au_ftest_si(sbinfo, FHSM)) + au_fhsm_wrote_all(sb, /*force*/0); + si_read_unlock(sb); + goto out; /* success */ + +out_pid: + spin_lock(&fhsm->fhsm_spin); + fhsm->fhsm_pid = 0; + spin_unlock(&fhsm->fhsm_spin); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_fhsm_br_alloc(struct au_branch *br) +{ + int err; + + err = 0; + br->br_fhsm = kmalloc(sizeof(*br->br_fhsm), GFP_NOFS); + if (br->br_fhsm) + au_br_fhsm_init(br->br_fhsm); + else + err = -ENOMEM; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_fhsm_fin(struct super_block *sb) +{ + au_fhsm_notify(sb, /*val*/-1); +} + +void au_fhsm_init(struct au_sbinfo *sbinfo) +{ + struct au_fhsm *fhsm; + + fhsm = &sbinfo->si_fhsm; + spin_lock_init(&fhsm->fhsm_spin); + init_waitqueue_head(&fhsm->fhsm_wqh); + atomic_set(&fhsm->fhsm_readable, 0); + fhsm->fhsm_expire + = msecs_to_jiffies(AUFS_FHSM_CACHE_DEF_SEC * MSEC_PER_SEC); + fhsm->fhsm_bottom = -1; +} + +void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec) +{ + sbinfo->si_fhsm.fhsm_expire + = msecs_to_jiffies(sec * MSEC_PER_SEC); +} + +void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo) +{ + unsigned int u; + + if (!au_ftest_si(sbinfo, FHSM)) + return; + + u = jiffies_to_msecs(sbinfo->si_fhsm.fhsm_expire) / MSEC_PER_SEC; + if (u != AUFS_FHSM_CACHE_DEF_SEC) + seq_printf(seq, ",fhsm_sec=%u", u); +} diff --git a/fs/aufs/file.c b/fs/aufs/file.c new file mode 100644 index 0000000000000..dbb33855d2260 --- /dev/null +++ b/fs/aufs/file.c @@ -0,0 +1,863 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * handling file/dir, and address_space operation + */ + +#ifdef CONFIG_AUFS_DEBUG +#include +#endif +#include +#include "aufs.h" + +/* drop flags for writing */ +unsigned int au_file_roflags(unsigned int flags) +{ + flags &= ~(O_WRONLY | O_RDWR | O_APPEND | O_CREAT | O_TRUNC); + flags |= O_RDONLY | O_NOATIME; + return flags; +} + +/* common functions to regular file and dir */ +struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags, + struct file *file, int force_wr) +{ + struct file *h_file; + struct dentry *h_dentry; + struct inode *h_inode; + struct super_block *sb; + struct au_branch *br; + struct path h_path; + int err; + + /* a race condition can happen between open and unlink/rmdir */ + h_file = ERR_PTR(-ENOENT); + h_dentry = au_h_dptr(dentry, bindex); + if (au_test_nfsd() && (!h_dentry || d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + spin_lock(&h_dentry->d_lock); + err = (!d_unhashed(dentry) && d_unlinked(h_dentry)) + /* || !d_inode(dentry)->i_nlink */ + ; + spin_unlock(&h_dentry->d_lock); + if (unlikely(err)) + goto out; + + sb = dentry->d_sb; + br = au_sbr(sb, bindex); + err = au_br_test_oflag(flags, br); + h_file = ERR_PTR(err); + if (unlikely(err)) + goto out; + + /* drop flags for writing */ + if (au_test_ro(sb, bindex, d_inode(dentry))) { + if (force_wr && !(flags & O_WRONLY)) + force_wr = 0; + flags = au_file_roflags(flags); + if (force_wr) { + h_file = ERR_PTR(-EROFS); + flags = au_file_roflags(flags); + if (unlikely(vfsub_native_ro(h_inode) + || IS_APPEND(h_inode))) + goto out; + flags &= ~O_ACCMODE; + flags |= O_WRONLY; + } + } + flags &= ~O_CREAT; + au_lcnt_inc(&br->br_nfiles); + h_path.dentry = h_dentry; + h_path.mnt = au_br_mnt(br); + h_file = vfsub_dentry_open(&h_path, flags); + if (IS_ERR(h_file)) + goto out_br; + + if (flags & __FMODE_EXEC) { + err = deny_write_access(h_file); + if (unlikely(err)) { + fput(h_file); + h_file = ERR_PTR(err); + goto out_br; + } + } + fsnotify_open(h_file); + goto out; /* success */ + +out_br: + au_lcnt_dec(&br->br_nfiles); +out: + return h_file; +} + +static int au_cmoo(struct dentry *dentry) +{ + int err, cmoo, matched; + unsigned int udba; + struct path h_path; + struct au_pin pin; + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = -1, + .bsrc = -1, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + struct inode *delegated; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + pid_t pid; + struct au_branch *br; + struct dentry *parent; + struct au_hinode *hdir; + + DiMustWriteLock(dentry); + IiMustWriteLock(d_inode(dentry)); + + err = 0; + if (IS_ROOT(dentry)) + goto out; + cpg.bsrc = au_dbtop(dentry); + if (!cpg.bsrc) + goto out; + + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + pid = au_fhsm_pid(fhsm); + rcu_read_lock(); + matched = (pid + && (current->pid == pid + || rcu_dereference(current->real_parent)->pid == pid)); + rcu_read_unlock(); + if (matched) + goto out; + + br = au_sbr(sb, cpg.bsrc); + cmoo = au_br_cmoo(br->br_perm); + if (!cmoo) + goto out; + if (!d_is_reg(dentry)) + cmoo &= AuBrAttr_COO_ALL; + if (!cmoo) + goto out; + + parent = dget_parent(dentry); + di_write_lock_parent(parent); + err = au_wbr_do_copyup_bu(dentry, cpg.bsrc - 1); + cpg.bdst = err; + if (unlikely(err < 0)) { + err = 0; /* there is no upper writable branch */ + goto out_dgrade; + } + AuDbg("bsrc %d, bdst %d\n", cpg.bsrc, cpg.bdst); + + /* do not respect the coo attrib for the target branch */ + err = au_cpup_dirs(dentry, cpg.bdst); + if (unlikely(err)) + goto out_dgrade; + + di_downgrade_lock(parent, AuLock_IR); + udba = au_opt_udba(sb); + err = au_pin(&pin, dentry, cpg.bdst, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_parent; + + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + if (unlikely(err)) + goto out_parent; + if (!(cmoo & AuBrWAttr_MOO)) + goto out_parent; /* success */ + + err = au_pin(&pin, dentry, cpg.bsrc, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_parent; + + h_path.mnt = au_br_mnt(br); + h_path.dentry = au_h_dptr(dentry, cpg.bsrc); + hdir = au_hi(d_inode(parent), cpg.bsrc); + delegated = NULL; + err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated, /*force*/1); + au_unpin(&pin); + /* todo: keep h_dentry or not? */ + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) { + pr_err("unlink %pd after coo failed (%d), ignored\n", + dentry, err); + err = 0; + } + goto out_parent; /* success */ + +out_dgrade: + di_downgrade_lock(parent, AuLock_IR); +out_parent: + di_read_unlock(parent, AuLock_IR); + dput(parent); +out: + AuTraceErr(err); + return err; +} + +int au_do_open(struct file *file, struct au_do_open_args *args) +{ + int err, aopen = args->aopen; + struct dentry *dentry; + struct au_finfo *finfo; + + if (!aopen) + err = au_finfo_init(file, args->fidir); + else { + lockdep_off(); + err = au_finfo_init(file, args->fidir); + lockdep_on(); + } + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + AuDebugOn(IS_ERR_OR_NULL(dentry)); + di_write_lock_child(dentry); + err = au_cmoo(dentry); + di_downgrade_lock(dentry, AuLock_IR); + if (!err) { + if (!aopen) + err = args->open(file, vfsub_file_flags(file), NULL); + else { + lockdep_off(); + err = args->open(file, vfsub_file_flags(file), + args->h_file); + lockdep_on(); + } + } + di_read_unlock(dentry, AuLock_IR); + + finfo = au_fi(file); + if (!err) { + finfo->fi_file = file; + au_hbl_add(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + } + if (!aopen) + fi_write_unlock(file); + else { + lockdep_off(); + fi_write_unlock(file); + lockdep_on(); + } + if (unlikely(err)) { + finfo->fi_hdir = NULL; + au_finfo_fin(file); + } + +out: + AuTraceErr(err); + return err; +} + +int au_reopen_nondir(struct file *file) +{ + int err; + aufs_bindex_t btop; + struct dentry *dentry; + struct au_branch *br; + struct file *h_file, *h_file_tmp; + + dentry = file->f_path.dentry; + btop = au_dbtop(dentry); + br = au_sbr(dentry->d_sb, btop); + h_file_tmp = NULL; + if (au_fbtop(file) == btop) { + h_file = au_hf_top(file); + if (file->f_mode == h_file->f_mode) + return 0; /* success */ + h_file_tmp = h_file; + get_file(h_file_tmp); + au_lcnt_inc(&br->br_nfiles); + au_set_h_fptr(file, btop, NULL); + } + AuDebugOn(au_fi(file)->fi_hdir); + /* + * it can happen + * file exists on both of rw and ro + * open --> dbtop and fbtop are both 0 + * prepend a branch as rw, "rw" become ro + * remove rw/file + * delete the top branch, "rw" becomes rw again + * --> dbtop is 1, fbtop is still 0 + * write --> fbtop is 0 but dbtop is 1 + */ + /* AuDebugOn(au_fbtop(file) < btop); */ + + h_file = au_h_open(dentry, btop, vfsub_file_flags(file) & ~O_TRUNC, + file, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) { + if (h_file_tmp) { + /* revert */ + au_set_h_fptr(file, btop, h_file_tmp); + h_file_tmp = NULL; + } + goto out; /* todo: close all? */ + } + + err = 0; + au_set_fbtop(file, btop); + au_set_h_fptr(file, btop, h_file); + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + +out: + if (h_file_tmp) { + fput(h_file_tmp); + au_lcnt_dec(&br->br_nfiles); + } + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_reopen_wh(struct file *file, aufs_bindex_t btgt, + struct dentry *hi_wh) +{ + int err; + aufs_bindex_t btop; + struct au_dinfo *dinfo; + struct dentry *h_dentry; + struct au_hdentry *hdp; + + dinfo = au_di(file->f_path.dentry); + AuRwMustWriteLock(&dinfo->di_rwsem); + + btop = dinfo->di_btop; + dinfo->di_btop = btgt; + hdp = au_hdentry(dinfo, btgt); + h_dentry = hdp->hd_dentry; + hdp->hd_dentry = hi_wh; + err = au_reopen_nondir(file); + hdp->hd_dentry = h_dentry; + dinfo->di_btop = btop; + + return err; +} + +static int au_ready_to_write_wh(struct file *file, loff_t len, + aufs_bindex_t bcpup, struct au_pin *pin) +{ + int err; + struct inode *inode, *h_inode; + struct dentry *h_dentry, *hi_wh; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = bcpup, + .bsrc = -1, + .len = len, + .pin = pin + }; + + au_update_dbtop(cpg.dentry); + inode = d_inode(cpg.dentry); + h_inode = NULL; + if (au_dbtop(cpg.dentry) <= bcpup + && au_dbbot(cpg.dentry) >= bcpup) { + h_dentry = au_h_dptr(cpg.dentry, bcpup); + if (h_dentry && d_is_positive(h_dentry)) + h_inode = d_inode(h_dentry); + } + hi_wh = au_hi_wh(inode, bcpup); + if (!hi_wh && !h_inode) + err = au_sio_cpup_wh(&cpg, file); + else + /* already copied-up after unlink */ + err = au_reopen_wh(file, bcpup, hi_wh); + + if (!err + && (inode->i_nlink > 1 + || (inode->i_state & I_LINKABLE)) + && au_opt_test(au_mntflags(cpg.dentry->d_sb), PLINK)) + au_plink_append(inode, bcpup, au_h_dptr(cpg.dentry, bcpup)); + + return err; +} + +/* + * prepare the @file for writing. + */ +int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin) +{ + int err; + aufs_bindex_t dbtop; + struct dentry *parent; + struct inode *inode; + struct super_block *sb; + struct file *h_file; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = -1, + .bsrc = -1, + .len = len, + .pin = pin, + .flags = AuCpup_DTIME + }; + + sb = cpg.dentry->d_sb; + inode = d_inode(cpg.dentry); + cpg.bsrc = au_fbtop(file); + err = au_test_ro(sb, cpg.bsrc, inode); + if (!err && (au_hf_top(file)->f_mode & FMODE_WRITE)) { + err = au_pin(pin, cpg.dentry, cpg.bsrc, AuOpt_UDBA_NONE, + /*flags*/0); + goto out; + } + + /* need to cpup or reopen */ + parent = dget_parent(cpg.dentry); + di_write_lock_parent(parent); + err = AuWbrCopyup(au_sbi(sb), cpg.dentry); + cpg.bdst = err; + if (unlikely(err < 0)) + goto out_dgrade; + err = 0; + + if (!d_unhashed(cpg.dentry) && !au_h_dptr(parent, cpg.bdst)) { + err = au_cpup_dirs(cpg.dentry, cpg.bdst); + if (unlikely(err)) + goto out_dgrade; + } + + err = au_pin(pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_dgrade; + + dbtop = au_dbtop(cpg.dentry); + if (dbtop <= cpg.bdst) + cpg.bsrc = cpg.bdst; + + if (dbtop <= cpg.bdst /* just reopen */ + || !d_unhashed(cpg.dentry) /* copyup and reopen */ + ) { + h_file = au_h_open_pre(cpg.dentry, cpg.bsrc, /*force_wr*/0); + if (IS_ERR(h_file)) + err = PTR_ERR(h_file); + else { + di_downgrade_lock(parent, AuLock_IR); + if (dbtop > cpg.bdst) + err = au_sio_cpup_simple(&cpg); + if (!err) + err = au_reopen_nondir(file); + au_h_open_post(cpg.dentry, cpg.bsrc, h_file); + } + } else { /* copyup as wh and reopen */ + /* + * since writable hfsplus branch is not supported, + * h_open_pre/post() are unnecessary. + */ + err = au_ready_to_write_wh(file, len, cpg.bdst, pin); + di_downgrade_lock(parent, AuLock_IR); + } + + if (!err) { + au_pin_set_parent_lflag(pin, /*lflag*/0); + goto out_dput; /* success */ + } + au_unpin(pin); + goto out_unlock; + +out_dgrade: + di_downgrade_lock(parent, AuLock_IR); +out_unlock: + di_read_unlock(parent, AuLock_IR); +out_dput: + dput(parent); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_do_flush(struct file *file, fl_owner_t id, + int (*flush)(struct file *file, fl_owner_t id)) +{ + int err; + struct super_block *sb; + struct inode *inode; + + inode = file_inode(file); + sb = inode->i_sb; + si_noflush_read_lock(sb); + fi_read_lock(file); + ii_read_lock_child(inode); + + err = flush(file, id); + au_cpup_attr_timesizes(inode); + + ii_read_unlock(inode); + fi_read_unlock(file); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_file_refresh_by_inode(struct file *file, int *need_reopen) +{ + int err; + struct au_pin pin; + struct au_finfo *finfo; + struct dentry *parent, *hi_wh; + struct inode *inode; + struct super_block *sb; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = -1, + .bsrc = -1, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME + }; + + FiMustWriteLock(file); + + err = 0; + finfo = au_fi(file); + sb = cpg.dentry->d_sb; + inode = d_inode(cpg.dentry); + cpg.bdst = au_ibtop(inode); + if (cpg.bdst == finfo->fi_btop || IS_ROOT(cpg.dentry)) + goto out; + + parent = dget_parent(cpg.dentry); + if (au_test_ro(sb, cpg.bdst, inode)) { + di_read_lock_parent(parent, !AuLock_IR); + err = AuWbrCopyup(au_sbi(sb), cpg.dentry); + cpg.bdst = err; + di_read_unlock(parent, !AuLock_IR); + if (unlikely(err < 0)) + goto out_parent; + err = 0; + } + + di_read_lock_parent(parent, AuLock_IR); + hi_wh = au_hi_wh(inode, cpg.bdst); + if (!S_ISDIR(inode->i_mode) + && au_opt_test(au_mntflags(sb), PLINK) + && au_plink_test(inode) + && !d_unhashed(cpg.dentry) + && cpg.bdst < au_dbtop(cpg.dentry)) { + err = au_test_and_cpup_dirs(cpg.dentry, cpg.bdst); + if (unlikely(err)) + goto out_unlock; + + /* always superio. */ + err = au_pin(&pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + } else if (hi_wh) { + /* already copied-up after unlink */ + err = au_reopen_wh(file, cpg.bdst, hi_wh); + *need_reopen = 0; + } + +out_unlock: + di_read_unlock(parent, AuLock_IR); +out_parent: + dput(parent); +out: + return err; +} + +static void au_do_refresh_dir(struct file *file) +{ + aufs_bindex_t bindex, bbot, new_bindex, brid; + struct au_hfile *p, tmp, *q; + struct au_finfo *finfo; + struct super_block *sb; + struct au_fidir *fidir; + + FiMustWriteLock(file); + + sb = file->f_path.dentry->d_sb; + finfo = au_fi(file); + fidir = finfo->fi_hdir; + AuDebugOn(!fidir); + p = fidir->fd_hfile + finfo->fi_btop; + brid = p->hf_br->br_id; + bbot = fidir->fd_bbot; + for (bindex = finfo->fi_btop; bindex <= bbot; bindex++, p++) { + if (!p->hf_file) + continue; + + new_bindex = au_br_index(sb, p->hf_br->br_id); + if (new_bindex == bindex) + continue; + if (new_bindex < 0) { + au_set_h_fptr(file, bindex, NULL); + continue; + } + + /* swap two lower inode, and loop again */ + q = fidir->fd_hfile + new_bindex; + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hf_file) { + bindex--; + p--; + } + } + + p = fidir->fd_hfile; + if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) { + bbot = au_sbbot(sb); + for (finfo->fi_btop = 0; finfo->fi_btop <= bbot; + finfo->fi_btop++, p++) + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; + au_hfput(p, /*execed*/0); + } + } else { + bbot = au_br_index(sb, brid); + for (finfo->fi_btop = 0; finfo->fi_btop < bbot; + finfo->fi_btop++, p++) + if (p->hf_file) + au_hfput(p, /*execed*/0); + bbot = au_sbbot(sb); + } + + p = fidir->fd_hfile + bbot; + for (fidir->fd_bbot = bbot; fidir->fd_bbot >= finfo->fi_btop; + fidir->fd_bbot--, p--) + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; + au_hfput(p, /*execed*/0); + } + AuDebugOn(fidir->fd_bbot < finfo->fi_btop); +} + +/* + * after branch manipulating, refresh the file. + */ +static int refresh_file(struct file *file, int (*reopen)(struct file *file)) +{ + int err, need_reopen, nbr; + aufs_bindex_t bbot, bindex; + struct dentry *dentry; + struct super_block *sb; + struct au_finfo *finfo; + struct au_hfile *hfile; + + dentry = file->f_path.dentry; + sb = dentry->d_sb; + nbr = au_sbbot(sb) + 1; + finfo = au_fi(file); + if (!finfo->fi_hdir) { + hfile = &finfo->fi_htop; + AuDebugOn(!hfile->hf_file); + bindex = au_br_index(sb, hfile->hf_br->br_id); + AuDebugOn(bindex < 0); + if (bindex != finfo->fi_btop) + au_set_fbtop(file, bindex); + } else { + err = au_fidir_realloc(finfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + au_do_refresh_dir(file); + } + + err = 0; + need_reopen = 1; + if (!au_test_mmapped(file)) + err = au_file_refresh_by_inode(file, &need_reopen); + if (finfo->fi_hdir) + /* harmless if err */ + au_fidir_realloc(finfo, nbr, /*may_shrink*/1); + if (!err && need_reopen && !d_unlinked(dentry)) + err = reopen(file); + if (!err) { + au_update_figen(file); + goto out; /* success */ + } + + /* error, close all lower files */ + if (finfo->fi_hdir) { + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); bindex <= bbot; bindex++) + au_set_h_fptr(file, bindex, NULL); + } + +out: + return err; +} + +/* common function to regular file and dir */ +int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file), + int wlock, unsigned int fi_lsc) +{ + int err; + unsigned int sigen, figen; + aufs_bindex_t btop; + unsigned char pseudo_link; + struct dentry *dentry; + struct inode *inode; + + err = 0; + dentry = file->f_path.dentry; + inode = d_inode(dentry); + sigen = au_sigen(dentry->d_sb); + fi_write_lock_nested(file, fi_lsc); + figen = au_figen(file); + if (!fi_lsc) + di_write_lock_child(dentry); + else + di_write_lock_child2(dentry); + btop = au_dbtop(dentry); + pseudo_link = (btop != au_ibtop(inode)); + if (sigen == figen && !pseudo_link && au_fbtop(file) == btop) { + if (!wlock) { + di_downgrade_lock(dentry, AuLock_IR); + fi_downgrade_lock(file); + } + goto out; /* success */ + } + + AuDbg("sigen %d, figen %d\n", sigen, figen); + if (au_digen_test(dentry, sigen)) { + err = au_reval_dpath(dentry, sigen); + AuDebugOn(!err && au_digen_test(dentry, sigen)); + } + + if (!err) + err = refresh_file(file, reopen); + if (!err) { + if (!wlock) { + di_downgrade_lock(dentry, AuLock_IR); + fi_downgrade_lock(file); + } + } else { + di_write_unlock(dentry); + fi_write_unlock(file); + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* cf. aufs_nopage() */ +/* for madvise(2) */ +static int aufs_readpage(struct file *file __maybe_unused, struct page *page) +{ + unlock_page(page); + return 0; +} + +/* it will never be called, but necessary to support O_DIRECT */ +static ssize_t aufs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ BUG(); return 0; } + +/* they will never be called. */ +#ifdef CONFIG_AUFS_DEBUG +static int aufs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ AuUnsupport(); return 0; } +static int aufs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ AuUnsupport(); return 0; } +static int aufs_writepage(struct page *page, struct writeback_control *wbc) +{ AuUnsupport(); return 0; } + +static int aufs_set_page_dirty(struct page *page) +{ AuUnsupport(); return 0; } +static void aufs_invalidatepage(struct page *page, unsigned int offset, + unsigned int length) +{ AuUnsupport(); } +static int aufs_releasepage(struct page *page, gfp_t gfp) +{ AuUnsupport(); return 0; } +#if 0 /* called by memory compaction regardless file */ +static int aufs_migratepage(struct address_space *mapping, struct page *newpage, + struct page *page, enum migrate_mode mode) +{ AuUnsupport(); return 0; } +#endif +static bool aufs_isolate_page(struct page *page, isolate_mode_t mode) +{ AuUnsupport(); return true; } +static void aufs_putback_page(struct page *page) +{ AuUnsupport(); } +static int aufs_launder_page(struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_is_partially_uptodate(struct page *page, + unsigned long from, + unsigned long count) +{ AuUnsupport(); return 0; } +static void aufs_is_dirty_writeback(struct page *page, bool *dirty, + bool *writeback) +{ AuUnsupport(); } +static int aufs_error_remove_page(struct address_space *mapping, + struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ AuUnsupport(); return 0; } +static void aufs_swap_deactivate(struct file *file) +{ AuUnsupport(); } +#endif /* CONFIG_AUFS_DEBUG */ + +const struct address_space_operations aufs_aop = { + .readpage = aufs_readpage, + .direct_IO = aufs_direct_IO, +#ifdef CONFIG_AUFS_DEBUG + .writepage = aufs_writepage, + /* no writepages, because of writepage */ + .set_page_dirty = aufs_set_page_dirty, + /* no readpages, because of readpage */ + .write_begin = aufs_write_begin, + .write_end = aufs_write_end, + /* no bmap, no block device */ + .invalidatepage = aufs_invalidatepage, + .releasepage = aufs_releasepage, + /* is fallback_migrate_page ok? */ + /* .migratepage = aufs_migratepage, */ + .isolate_page = aufs_isolate_page, + .putback_page = aufs_putback_page, + .launder_page = aufs_launder_page, + .is_partially_uptodate = aufs_is_partially_uptodate, + .is_dirty_writeback = aufs_is_dirty_writeback, + .error_remove_page = aufs_error_remove_page, + .swap_activate = aufs_swap_activate, + .swap_deactivate = aufs_swap_deactivate +#endif /* CONFIG_AUFS_DEBUG */ +}; diff --git a/fs/aufs/file.h b/fs/aufs/file.h new file mode 100644 index 0000000000000..4ee95dde03b86 --- /dev/null +++ b/fs/aufs/file.h @@ -0,0 +1,342 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file operations + */ + +#ifndef __AUFS_FILE_H__ +#define __AUFS_FILE_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include "rwsem.h" + +struct au_branch; +struct au_hfile { + struct file *hf_file; + struct au_branch *hf_br; +}; + +struct au_vdir; +struct au_fidir { + aufs_bindex_t fd_bbot; + aufs_bindex_t fd_nent; + struct au_vdir *fd_vdir_cache; + struct au_hfile fd_hfile[]; +}; + +static inline int au_fidir_sz(int nent) +{ + AuDebugOn(nent < 0); + return sizeof(struct au_fidir) + sizeof(struct au_hfile) * nent; +} + +struct au_finfo { + atomic_t fi_generation; + + struct au_rwsem fi_rwsem; + aufs_bindex_t fi_btop; + + /* do not union them */ + struct { /* for non-dir */ + struct au_hfile fi_htop; + atomic_t fi_mmapped; + }; + struct au_fidir *fi_hdir; /* for dir only */ + + struct hlist_bl_node fi_hlist; + struct file *fi_file; /* very ugly */ + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* file.c */ +extern const struct address_space_operations aufs_aop; +unsigned int au_file_roflags(unsigned int flags); +struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags, + struct file *file, int force_wr); +struct au_do_open_args { + int aopen; + int (*open)(struct file *file, int flags, + struct file *h_file); + struct au_fidir *fidir; + struct file *h_file; +}; +int au_do_open(struct file *file, struct au_do_open_args *args); +int au_reopen_nondir(struct file *file); +struct au_pin; +int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin); +int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file), + int wlock, unsigned int fi_lsc); +int au_do_flush(struct file *file, fl_owner_t id, + int (*flush)(struct file *file, fl_owner_t id)); + +/* poll.c */ +#ifdef CONFIG_AUFS_POLL +unsigned int aufs_poll(struct file *file, poll_table *wait); +#endif + +#ifdef CONFIG_AUFS_BR_HFSPLUS +/* hfsplus.c */ +struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex, + int force_wr); +void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file); +#else +AuStub(struct file *, au_h_open_pre, return NULL, struct dentry *dentry, + aufs_bindex_t bindex, int force_wr) +AuStubVoid(au_h_open_post, struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file); +#endif + +/* f_op.c */ +extern const struct file_operations aufs_file_fop; +int au_do_open_nondir(struct file *file, int flags, struct file *h_file); +int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file); +struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc); + +/* finfo.c */ +void au_hfput(struct au_hfile *hf, int execed); +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, + struct file *h_file); + +void au_update_figen(struct file *file); +struct au_fidir *au_fidir_alloc(struct super_block *sb); +int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink); + +void au_fi_init_once(void *_fi); +void au_finfo_fin(struct file *file); +int au_finfo_init(struct file *file, struct au_fidir *fidir); + +/* ioctl.c */ +long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd, + unsigned long arg); +long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd, + unsigned long arg); +#endif + +/* ---------------------------------------------------------------------- */ + +static inline struct au_finfo *au_fi(struct file *file) +{ + return file->private_data; +} + +/* ---------------------------------------------------------------------- */ + +#define fi_read_lock(f) au_rw_read_lock(&au_fi(f)->fi_rwsem) +#define fi_write_lock(f) au_rw_write_lock(&au_fi(f)->fi_rwsem) +#define fi_read_trylock(f) au_rw_read_trylock(&au_fi(f)->fi_rwsem) +#define fi_write_trylock(f) au_rw_write_trylock(&au_fi(f)->fi_rwsem) +/* +#define fi_read_trylock_nested(f) \ + au_rw_read_trylock_nested(&au_fi(f)->fi_rwsem) +#define fi_write_trylock_nested(f) \ + au_rw_write_trylock_nested(&au_fi(f)->fi_rwsem) +*/ + +#define fi_read_unlock(f) au_rw_read_unlock(&au_fi(f)->fi_rwsem) +#define fi_write_unlock(f) au_rw_write_unlock(&au_fi(f)->fi_rwsem) +#define fi_downgrade_lock(f) au_rw_dgrade_lock(&au_fi(f)->fi_rwsem) + +/* lock subclass for finfo */ +enum { + AuLsc_FI_1, + AuLsc_FI_2 +}; + +static inline void fi_read_lock_nested(struct file *f, unsigned int lsc) +{ + au_rw_read_lock_nested(&au_fi(f)->fi_rwsem, lsc); +} + +static inline void fi_write_lock_nested(struct file *f, unsigned int lsc) +{ + au_rw_write_lock_nested(&au_fi(f)->fi_rwsem, lsc); +} + +/* + * fi_read_lock_1, fi_write_lock_1, + * fi_read_lock_2, fi_write_lock_2 + */ +#define AuReadLockFunc(name) \ +static inline void fi_read_lock_##name(struct file *f) \ +{ fi_read_lock_nested(f, AuLsc_FI_##name); } + +#define AuWriteLockFunc(name) \ +static inline void fi_write_lock_##name(struct file *f) \ +{ fi_write_lock_nested(f, AuLsc_FI_##name); } + +#define AuRWLockFuncs(name) \ + AuReadLockFunc(name) \ + AuWriteLockFunc(name) + +AuRWLockFuncs(1); +AuRWLockFuncs(2); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define FiMustNoWaiters(f) AuRwMustNoWaiters(&au_fi(f)->fi_rwsem) +#define FiMustAnyLock(f) AuRwMustAnyLock(&au_fi(f)->fi_rwsem) +#define FiMustWriteLock(f) AuRwMustWriteLock(&au_fi(f)->fi_rwsem) + +/* ---------------------------------------------------------------------- */ + +/* todo: hard/soft set? */ +static inline aufs_bindex_t au_fbtop(struct file *file) +{ + FiMustAnyLock(file); + return au_fi(file)->fi_btop; +} + +static inline aufs_bindex_t au_fbbot_dir(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_bbot; +} + +static inline struct au_vdir *au_fvdir_cache(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_vdir_cache; +} + +static inline void au_set_fbtop(struct file *file, aufs_bindex_t bindex) +{ + FiMustWriteLock(file); + au_fi(file)->fi_btop = bindex; +} + +static inline void au_set_fbbot_dir(struct file *file, aufs_bindex_t bindex) +{ + FiMustWriteLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + au_fi(file)->fi_hdir->fd_bbot = bindex; +} + +static inline void au_set_fvdir_cache(struct file *file, + struct au_vdir *vdir_cache) +{ + FiMustWriteLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + au_fi(file)->fi_hdir->fd_vdir_cache = vdir_cache; +} + +static inline struct file *au_hf_top(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(au_fi(file)->fi_hdir); + return au_fi(file)->fi_htop.hf_file; +} + +static inline struct file *au_hf_dir(struct file *file, aufs_bindex_t bindex) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_hfile[0 + bindex].hf_file; +} + +/* todo: memory barrier? */ +static inline unsigned int au_figen(struct file *f) +{ + return atomic_read(&au_fi(f)->fi_generation); +} + +static inline void au_set_mmapped(struct file *f) +{ + if (atomic_inc_return(&au_fi(f)->fi_mmapped)) + return; + pr_warn("fi_mmapped wrapped around\n"); + while (!atomic_inc_return(&au_fi(f)->fi_mmapped)) + ; +} + +static inline void au_unset_mmapped(struct file *f) +{ + atomic_dec(&au_fi(f)->fi_mmapped); +} + +static inline int au_test_mmapped(struct file *f) +{ + return atomic_read(&au_fi(f)->fi_mmapped); +} + +/* customize vma->vm_file */ + +static inline void au_do_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + struct file *f; + + f = vma->vm_file; + get_file(file); + vma->vm_file = file; + fput(f); +} + +#ifdef CONFIG_MMU +#define AuDbgVmRegion(file, vma) do {} while (0) + +static inline void au_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + au_do_vm_file_reset(vma, file); +} +#else +#define AuDbgVmRegion(file, vma) \ + AuDebugOn((vma)->vm_region && (vma)->vm_region->vm_file != (file)) + +static inline void au_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + struct file *f; + + au_do_vm_file_reset(vma, file); + f = vma->vm_region->vm_file; + get_file(file); + vma->vm_region->vm_file = file; + fput(f); +} +#endif /* CONFIG_MMU */ + +/* handle vma->vm_prfile */ +static inline void au_vm_prfile_set(struct vm_area_struct *vma, + struct file *file) +{ + get_file(file); + vma->vm_prfile = file; +#ifndef CONFIG_MMU + get_file(file); + vma->vm_region->vm_prfile = file; +#endif +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_FILE_H__ */ diff --git a/fs/aufs/finfo.c b/fs/aufs/finfo.c new file mode 100644 index 0000000000000..448c5e9cd85dd --- /dev/null +++ b/fs/aufs/finfo.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file private data + */ + +#include "aufs.h" + +void au_hfput(struct au_hfile *hf, int execed) +{ + if (execed) + allow_write_access(hf->hf_file); + fput(hf->hf_file); + hf->hf_file = NULL; + au_lcnt_dec(&hf->hf_br->br_nfiles); + hf->hf_br = NULL; +} + +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, struct file *val) +{ + struct au_finfo *finfo = au_fi(file); + struct au_hfile *hf; + struct au_fidir *fidir; + + fidir = finfo->fi_hdir; + if (!fidir) { + AuDebugOn(finfo->fi_btop != bindex); + hf = &finfo->fi_htop; + } else + hf = fidir->fd_hfile + bindex; + + if (hf && hf->hf_file) + au_hfput(hf, vfsub_file_execed(file)); + if (val) { + FiMustWriteLock(file); + AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry)); + hf->hf_file = val; + hf->hf_br = au_sbr(file->f_path.dentry->d_sb, bindex); + } +} + +void au_update_figen(struct file *file) +{ + atomic_set(&au_fi(file)->fi_generation, au_digen(file->f_path.dentry)); + /* smp_mb(); */ /* atomic_set */ +} + +/* ---------------------------------------------------------------------- */ + +struct au_fidir *au_fidir_alloc(struct super_block *sb) +{ + struct au_fidir *fidir; + int nbr; + + nbr = au_sbbot(sb) + 1; + if (nbr < 2) + nbr = 2; /* initial allocate for 2 branches */ + fidir = kzalloc(au_fidir_sz(nbr), GFP_NOFS); + if (fidir) { + fidir->fd_bbot = -1; + fidir->fd_nent = nbr; + } + + return fidir; +} + +int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink) +{ + int err; + struct au_fidir *fidir, *p; + + AuRwMustWriteLock(&finfo->fi_rwsem); + fidir = finfo->fi_hdir; + AuDebugOn(!fidir); + + err = -ENOMEM; + p = au_kzrealloc(fidir, au_fidir_sz(fidir->fd_nent), au_fidir_sz(nbr), + GFP_NOFS, may_shrink); + if (p) { + p->fd_nent = nbr; + finfo->fi_hdir = p; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_finfo_fin(struct file *file) +{ + struct au_finfo *finfo; + + au_lcnt_dec(&au_sbi(file->f_path.dentry->d_sb)->si_nfiles); + + finfo = au_fi(file); + AuDebugOn(finfo->fi_hdir); + AuRwDestroy(&finfo->fi_rwsem); + au_cache_free_finfo(finfo); +} + +void au_fi_init_once(void *_finfo) +{ + struct au_finfo *finfo = _finfo; + + au_rw_init(&finfo->fi_rwsem); +} + +int au_finfo_init(struct file *file, struct au_fidir *fidir) +{ + int err; + struct au_finfo *finfo; + struct dentry *dentry; + + err = -ENOMEM; + dentry = file->f_path.dentry; + finfo = au_cache_alloc_finfo(); + if (unlikely(!finfo)) + goto out; + + err = 0; + au_lcnt_inc(&au_sbi(dentry->d_sb)->si_nfiles); + au_rw_write_lock(&finfo->fi_rwsem); + finfo->fi_btop = -1; + finfo->fi_hdir = fidir; + atomic_set(&finfo->fi_generation, au_digen(dentry)); + /* smp_mb(); */ /* atomic_set */ + + file->private_data = finfo; + +out: + return err; +} diff --git a/fs/aufs/fstype.h b/fs/aufs/fstype.h new file mode 100644 index 0000000000000..88cfefac760a0 --- /dev/null +++ b/fs/aufs/fstype.h @@ -0,0 +1,401 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * judging filesystem type + */ + +#ifndef __AUFS_FSTYPE_H__ +#define __AUFS_FSTYPE_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +static inline int au_test_aufs(struct super_block *sb) +{ + return sb->s_magic == AUFS_SUPER_MAGIC; +} + +static inline const char *au_sbtype(struct super_block *sb) +{ + return sb->s_type->name; +} + +static inline int au_test_iso9660(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ISO9660_FS) + return sb->s_magic == ISOFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_romfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ROMFS_FS) + return sb->s_magic == ROMFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_cramfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_CRAMFS) + return sb->s_magic == CRAMFS_MAGIC; +#endif + return 0; +} + +static inline int au_test_nfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NFS_FS) + return sb->s_magic == NFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_fuse(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_FUSE_FS) + return sb->s_magic == FUSE_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_xfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_XFS_FS) + return sb->s_magic == XFS_SB_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_tmpfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_TMPFS + return sb->s_magic == TMPFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ECRYPT_FS) + return !strcmp(au_sbtype(sb), "ecryptfs"); +#else + return 0; +#endif +} + +static inline int au_test_ramfs(struct super_block *sb) +{ + return sb->s_magic == RAMFS_MAGIC; +} + +static inline int au_test_ubifs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_UBIFS_FS) + return sb->s_magic == UBIFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_procfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_PROC_FS + return sb->s_magic == PROC_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_sysfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_SYSFS + return sb->s_magic == SYSFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_configfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_CONFIGFS_FS) + return sb->s_magic == CONFIGFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_minix(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_MINIX_FS) + return sb->s_magic == MINIX3_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC2 + || sb->s_magic == MINIX_SUPER_MAGIC + || sb->s_magic == MINIX_SUPER_MAGIC2; +#else + return 0; +#endif +} + +static inline int au_test_fat(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_FAT_FS) + return sb->s_magic == MSDOS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_msdos(struct super_block *sb) +{ + return au_test_fat(sb); +} + +static inline int au_test_vfat(struct super_block *sb) +{ + return au_test_fat(sb); +} + +static inline int au_test_securityfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_SECURITYFS + return sb->s_magic == SECURITYFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_squashfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_SQUASHFS) + return sb->s_magic == SQUASHFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_btrfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_BTRFS_FS) + return sb->s_magic == BTRFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_xenfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_XENFS) + return sb->s_magic == XENFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_debugfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_DEBUG_FS + return sb->s_magic == DEBUGFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_nilfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NILFS) + return sb->s_magic == NILFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_hfsplus(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_HFSPLUS_FS) + return sb->s_magic == HFSPLUS_SUPER_MAGIC; +#else + return 0; +#endif +} + +/* ---------------------------------------------------------------------- */ +/* + * they can't be an aufs branch. + */ +static inline int au_test_fs_unsuppoted(struct super_block *sb) +{ + return +#ifndef CONFIG_AUFS_BR_RAMFS + au_test_ramfs(sb) || +#endif + au_test_procfs(sb) + || au_test_sysfs(sb) + || au_test_configfs(sb) + || au_test_debugfs(sb) + || au_test_securityfs(sb) + || au_test_xenfs(sb) + || au_test_ecryptfs(sb) + /* || !strcmp(au_sbtype(sb), "unionfs") */ + || au_test_aufs(sb); /* will be supported in next version */ +} + +static inline int au_test_fs_remote(struct super_block *sb) +{ + return !au_test_tmpfs(sb) +#ifdef CONFIG_AUFS_BR_RAMFS + && !au_test_ramfs(sb) +#endif + && !(sb->s_type->fs_flags & FS_REQUIRES_DEV); +} + +/* ---------------------------------------------------------------------- */ + +/* + * Note: these functions (below) are created after reading ->getattr() in all + * filesystems under linux/fs. it means we have to do so in every update... + */ + +/* + * some filesystems require getattr to refresh the inode attributes before + * referencing. + * in most cases, we can rely on the inode attribute in NFS (or every remote fs) + * and leave the work for d_revalidate() + */ +static inline int au_test_fs_refresh_iattr(struct super_block *sb) +{ + return au_test_nfs(sb) + || au_test_fuse(sb) + /* || au_test_btrfs(sb) */ /* untested */ + ; +} + +/* + * filesystems which don't maintain i_size or i_blocks. + */ +static inline int au_test_fs_bad_iattr_size(struct super_block *sb) +{ + return au_test_xfs(sb) + || au_test_btrfs(sb) + || au_test_ubifs(sb) + || au_test_hfsplus(sb) /* maintained, but incorrect */ + /* || au_test_minix(sb) */ /* untested */ + ; +} + +/* + * filesystems which don't store the correct value in some of their inode + * attributes. + */ +static inline int au_test_fs_bad_iattr(struct super_block *sb) +{ + return au_test_fs_bad_iattr_size(sb) + || au_test_fat(sb) + || au_test_msdos(sb) + || au_test_vfat(sb); +} + +/* they don't check i_nlink in link(2) */ +static inline int au_test_fs_no_limit_nlink(struct super_block *sb) +{ + return au_test_tmpfs(sb) +#ifdef CONFIG_AUFS_BR_RAMFS + || au_test_ramfs(sb) +#endif + || au_test_ubifs(sb) + || au_test_hfsplus(sb); +} + +/* + * filesystems which sets S_NOATIME and S_NOCMTIME. + */ +static inline int au_test_fs_notime(struct super_block *sb) +{ + return au_test_nfs(sb) + || au_test_fuse(sb) + || au_test_ubifs(sb) + ; +} + +/* temporary support for i#1 in cramfs */ +static inline int au_test_fs_unique_ino(struct inode *inode) +{ + if (au_test_cramfs(inode->i_sb)) + return inode->i_ino != 1; + return 1; +} + +/* ---------------------------------------------------------------------- */ + +/* + * the filesystem where the xino files placed must support i/o after unlink and + * maintain i_size and i_blocks. + */ +static inline int au_test_fs_bad_xino(struct super_block *sb) +{ + return au_test_fs_remote(sb) + || au_test_fs_bad_iattr_size(sb) + /* don't want unnecessary work for xino */ + || au_test_aufs(sb) + || au_test_ecryptfs(sb) + || au_test_nilfs(sb); +} + +static inline int au_test_fs_trunc_xino(struct super_block *sb) +{ + return au_test_tmpfs(sb) + || au_test_ramfs(sb); +} + +/* + * test if the @sb is real-readonly. + */ +static inline int au_test_fs_rr(struct super_block *sb) +{ + return au_test_squashfs(sb) + || au_test_iso9660(sb) + || au_test_cramfs(sb) + || au_test_romfs(sb); +} + +/* + * test if the @inode is nfs with 'noacl' option + * NFS always sets MS_POSIXACL regardless its mount option 'noacl.' + */ +static inline int au_test_nfs_noacl(struct inode *inode) +{ + return au_test_nfs(inode->i_sb) + /* && IS_POSIXACL(inode) */ + && !nfs_server_capable(inode, NFS_CAP_ACLS); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_FSTYPE_H__ */ diff --git a/fs/aufs/hbl.h b/fs/aufs/hbl.h new file mode 100644 index 0000000000000..6db65b16b08a2 --- /dev/null +++ b/fs/aufs/hbl.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2017-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * helpers for hlist_bl.h + */ + +#ifndef __AUFS_HBL_H__ +#define __AUFS_HBL_H__ + +#ifdef __KERNEL__ + +#include + +static inline void au_hbl_add(struct hlist_bl_node *node, + struct hlist_bl_head *hbl) +{ + hlist_bl_lock(hbl); + hlist_bl_add_head(node, hbl); + hlist_bl_unlock(hbl); +} + +static inline void au_hbl_del(struct hlist_bl_node *node, + struct hlist_bl_head *hbl) +{ + hlist_bl_lock(hbl); + hlist_bl_del(node); + hlist_bl_unlock(hbl); +} + +#define au_hbl_for_each(pos, head) \ + for (pos = hlist_bl_first(head); \ + pos; \ + pos = pos->next) + +static inline unsigned long au_hbl_count(struct hlist_bl_head *hbl) +{ + unsigned long cnt; + struct hlist_bl_node *pos; + + cnt = 0; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) + cnt++; + hlist_bl_unlock(hbl); + return cnt; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_HBL_H__ */ diff --git a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c new file mode 100644 index 0000000000000..e92d73141e7fa --- /dev/null +++ b/fs/aufs/hfsnotify.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * fsnotify for the lower directories + */ + +#include "aufs.h" + +/* FS_IN_IGNORED is unnecessary */ +static const __u32 AuHfsnMask = (FS_MOVED_TO | FS_MOVED_FROM | FS_DELETE + | FS_CREATE | FS_EVENT_ON_CHILD); +static DECLARE_WAIT_QUEUE_HEAD(au_hfsn_wq); +static __cacheline_aligned_in_smp atomic64_t au_hfsn_ifree = ATOMIC64_INIT(0); + +static void au_hfsn_free_mark(struct fsnotify_mark *mark) +{ + struct au_hnotify *hn = container_of(mark, struct au_hnotify, + hn_mark); + /* AuDbg("here\n"); */ + au_cache_free_hnotify(hn); + smp_mb__before_atomic(); /* for atomic64_dec */ + if (atomic64_dec_and_test(&au_hfsn_ifree)) + wake_up(&au_hfsn_wq); +} + +static int au_hfsn_alloc(struct au_hinode *hinode) +{ + int err; + struct au_hnotify *hn; + struct super_block *sb; + struct au_branch *br; + struct fsnotify_mark *mark; + aufs_bindex_t bindex; + + hn = hinode->hi_notify; + sb = hn->hn_aufs_inode->i_sb; + bindex = au_br_index(sb, hinode->hi_id); + br = au_sbr(sb, bindex); + AuDebugOn(!br->br_hfsn); + + mark = &hn->hn_mark; + fsnotify_init_mark(mark, br->br_hfsn->hfsn_group); + mark->mask = AuHfsnMask; + /* + * by udba rename or rmdir, aufs assign a new inode to the known + * h_inode, so specify 1 to allow dups. + */ + lockdep_off(); + err = fsnotify_add_mark(mark, hinode->hi_inode, /*mnt*/NULL, + /*allow_dups*/1); + lockdep_on(); + + return err; +} + +static int au_hfsn_free(struct au_hinode *hinode, struct au_hnotify *hn) +{ + struct fsnotify_mark *mark; + unsigned long long ull; + struct fsnotify_group *group; + + ull = atomic64_inc_return(&au_hfsn_ifree); + BUG_ON(!ull); + + mark = &hn->hn_mark; + spin_lock(&mark->lock); + group = mark->group; + fsnotify_get_group(group); + spin_unlock(&mark->lock); + lockdep_off(); + fsnotify_destroy_mark(mark, group); + fsnotify_put_mark(mark); + fsnotify_put_group(group); + lockdep_on(); + + /* free hn by myself */ + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_ctl(struct au_hinode *hinode, int do_set) +{ + struct fsnotify_mark *mark; + + mark = &hinode->hi_notify->hn_mark; + spin_lock(&mark->lock); + if (do_set) { + AuDebugOn(mark->mask & AuHfsnMask); + mark->mask |= AuHfsnMask; + } else { + AuDebugOn(!(mark->mask & AuHfsnMask)); + mark->mask &= ~AuHfsnMask; + } + spin_unlock(&mark->lock); + /* fsnotify_recalc_inode_mask(hinode->hi_inode); */ +} + +/* ---------------------------------------------------------------------- */ + +/* #define AuDbgHnotify */ +#ifdef AuDbgHnotify +static char *au_hfsn_name(u32 mask) +{ +#ifdef CONFIG_AUFS_DEBUG +#define test_ret(flag) \ + do { \ + if (mask & flag) \ + return #flag; \ + } while (0) + test_ret(FS_ACCESS); + test_ret(FS_MODIFY); + test_ret(FS_ATTRIB); + test_ret(FS_CLOSE_WRITE); + test_ret(FS_CLOSE_NOWRITE); + test_ret(FS_OPEN); + test_ret(FS_MOVED_FROM); + test_ret(FS_MOVED_TO); + test_ret(FS_CREATE); + test_ret(FS_DELETE); + test_ret(FS_DELETE_SELF); + test_ret(FS_MOVE_SELF); + test_ret(FS_UNMOUNT); + test_ret(FS_Q_OVERFLOW); + test_ret(FS_IN_IGNORED); + test_ret(FS_ISDIR); + test_ret(FS_IN_ONESHOT); + test_ret(FS_EVENT_ON_CHILD); + return ""; +#undef test_ret +#else + return "??"; +#endif +} +#endif + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_free_group(struct fsnotify_group *group) +{ + struct au_br_hfsnotify *hfsn = group->private; + + /* AuDbg("here\n"); */ + au_kfree_try_rcu(hfsn); +} + +static int au_hfsn_handle_event(struct fsnotify_group *group, + struct inode *inode, + struct fsnotify_mark *inode_mark, + struct fsnotify_mark *vfsmount_mark, + u32 mask, const void *data, int data_type, + const unsigned char *file_name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + int err; + struct au_hnotify *hnotify; + struct inode *h_dir, *h_inode; + struct qstr h_child_qstr = QSTR_INIT(file_name, strlen(file_name)); + + AuDebugOn(data_type != FSNOTIFY_EVENT_INODE); + + err = 0; + /* if FS_UNMOUNT happens, there must be another bug */ + AuDebugOn(mask & FS_UNMOUNT); + if (mask & (FS_IN_IGNORED | FS_UNMOUNT)) + goto out; + + h_dir = inode; + h_inode = NULL; +#ifdef AuDbgHnotify + au_debug_on(); + if (1 || h_child_qstr.len != sizeof(AUFS_XINO_FNAME) - 1 + || strncmp(h_child_qstr.name, AUFS_XINO_FNAME, h_child_qstr.len)) { + AuDbg("i%lu, mask 0x%x %s, hcname %.*s, hi%lu\n", + h_dir->i_ino, mask, au_hfsn_name(mask), + AuLNPair(&h_child_qstr), h_inode ? h_inode->i_ino : 0); + /* WARN_ON(1); */ + } + au_debug_off(); +#endif + + AuDebugOn(!inode_mark); + hnotify = container_of(inode_mark, struct au_hnotify, hn_mark); + err = au_hnotify(h_dir, hnotify, mask, &h_child_qstr, h_inode); + +out: + return err; +} + +static struct fsnotify_ops au_hfsn_ops = { + .handle_event = au_hfsn_handle_event, + .free_group_priv = au_hfsn_free_group, + .free_mark = au_hfsn_free_mark +}; + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_fin_br(struct au_branch *br) +{ + struct au_br_hfsnotify *hfsn; + + hfsn = br->br_hfsn; + if (hfsn) { + lockdep_off(); + fsnotify_put_group(hfsn->hfsn_group); + lockdep_on(); + } +} + +static int au_hfsn_init_br(struct au_branch *br, int perm) +{ + int err; + struct fsnotify_group *group; + struct au_br_hfsnotify *hfsn; + + err = 0; + br->br_hfsn = NULL; + if (!au_br_hnotifyable(perm)) + goto out; + + err = -ENOMEM; + hfsn = kmalloc(sizeof(*hfsn), GFP_NOFS); + if (unlikely(!hfsn)) + goto out; + + err = 0; + group = fsnotify_alloc_group(&au_hfsn_ops); + if (IS_ERR(group)) { + err = PTR_ERR(group); + pr_err("fsnotify_alloc_group() failed, %d\n", err); + goto out_hfsn; + } + + group->private = hfsn; + hfsn->hfsn_group = group; + br->br_hfsn = hfsn; + goto out; /* success */ + +out_hfsn: + au_kfree_try_rcu(hfsn); +out: + return err; +} + +static int au_hfsn_reset_br(unsigned int udba, struct au_branch *br, int perm) +{ + int err; + + err = 0; + if (!br->br_hfsn) + err = au_hfsn_init_br(br, perm); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_fin(void) +{ + AuDbg("au_hfsn_ifree %lld\n", (long long)atomic64_read(&au_hfsn_ifree)); + wait_event(au_hfsn_wq, !atomic64_read(&au_hfsn_ifree)); +} + +const struct au_hnotify_op au_hnotify_op = { + .ctl = au_hfsn_ctl, + .alloc = au_hfsn_alloc, + .free = au_hfsn_free, + + .fin = au_hfsn_fin, + + .reset_br = au_hfsn_reset_br, + .fin_br = au_hfsn_fin_br, + .init_br = au_hfsn_init_br +}; diff --git a/fs/aufs/hfsplus.c b/fs/aufs/hfsplus.c new file mode 100644 index 0000000000000..3bd869a69c8f5 --- /dev/null +++ b/fs/aufs/hfsplus.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * special support for filesystems which acquires an inode mutex + * at final closing a file, eg, hfsplus. + * + * This trick is very simple and stupid, just to open the file before really + * necessary open to tell hfsplus that this is not the final closing. + * The caller should call au_h_open_pre() after acquiring the inode mutex, + * and au_h_open_post() after releasing it. + */ + +#include "aufs.h" + +struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex, + int force_wr) +{ + struct file *h_file; + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + AuDebugOn(!h_dentry); + AuDebugOn(d_is_negative(h_dentry)); + + h_file = NULL; + if (au_test_hfsplus(h_dentry->d_sb) + && d_is_reg(h_dentry)) + h_file = au_h_open(dentry, bindex, + O_RDONLY | O_NOATIME | O_LARGEFILE, + /*file*/NULL, force_wr); + return h_file; +} + +void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file) +{ + struct au_branch *br; + + if (h_file) { + fput(h_file); + br = au_sbr(dentry->d_sb, bindex); + au_lcnt_dec(&br->br_nfiles); + } +} diff --git a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c new file mode 100644 index 0000000000000..af35561f3ea50 --- /dev/null +++ b/fs/aufs/hnotify.c @@ -0,0 +1,720 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * abstraction to notify the direct changes on lower directories + */ + +#include "aufs.h" + +int au_hn_alloc(struct au_hinode *hinode, struct inode *inode) +{ + int err; + struct au_hnotify *hn; + + err = -ENOMEM; + hn = au_cache_alloc_hnotify(); + if (hn) { + hn->hn_aufs_inode = inode; + hinode->hi_notify = hn; + err = au_hnotify_op.alloc(hinode); + AuTraceErr(err); + if (unlikely(err)) { + hinode->hi_notify = NULL; + au_cache_free_hnotify(hn); + /* + * The upper dir was removed by udba, but the same named + * dir left. In this case, aufs assigns a new inode + * number and set the monitor again. + * For the lower dir, the old monitor is still left. + */ + if (err == -EEXIST) + err = 0; + } + } + + AuTraceErr(err); + return err; +} + +void au_hn_free(struct au_hinode *hinode) +{ + struct au_hnotify *hn; + + hn = hinode->hi_notify; + if (hn) { + hinode->hi_notify = NULL; + if (au_hnotify_op.free(hinode, hn)) + au_cache_free_hnotify(hn); + } +} + +/* ---------------------------------------------------------------------- */ + +void au_hn_ctl(struct au_hinode *hinode, int do_set) +{ + if (hinode->hi_notify) + au_hnotify_op.ctl(hinode, do_set); +} + +void au_hn_reset(struct inode *inode, unsigned int flags) +{ + aufs_bindex_t bindex, bbot; + struct inode *hi; + struct dentry *iwhdentry; + + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) { + hi = au_h_iptr(inode, bindex); + if (!hi) + continue; + + /* inode_lock_nested(hi, AuLsc_I_CHILD); */ + iwhdentry = au_hi_wh(inode, bindex); + if (iwhdentry) + dget(iwhdentry); + au_igrab(hi); + au_set_h_iptr(inode, bindex, NULL, 0); + au_set_h_iptr(inode, bindex, au_igrab(hi), + flags & ~AuHi_XINO); + iput(hi); + dput(iwhdentry); + /* inode_unlock(hi); */ + } +} + +/* ---------------------------------------------------------------------- */ + +static int hn_xino(struct inode *inode, struct inode *h_inode) +{ + int err; + aufs_bindex_t bindex, bbot, bfound, btop; + struct inode *h_i; + + err = 0; + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("branch root dir was changed\n"); + goto out; + } + + bfound = -1; + bbot = au_ibbot(inode); + btop = au_ibtop(inode); +#if 0 /* reserved for future use */ + if (bindex == bbot) { + /* keep this ino in rename case */ + goto out; + } +#endif + for (bindex = btop; bindex <= bbot; bindex++) + if (au_h_iptr(inode, bindex) == h_inode) { + bfound = bindex; + break; + } + if (bfound < 0) + goto out; + + for (bindex = btop; bindex <= bbot; bindex++) { + h_i = au_h_iptr(inode, bindex); + if (!h_i) + continue; + + err = au_xino_write(inode->i_sb, bindex, h_i->i_ino, /*ino*/0); + /* ignore this error */ + /* bad action? */ + } + + /* children inode number will be broken */ + +out: + AuTraceErr(err); + return err; +} + +static int hn_gen_tree(struct dentry *dentry) +{ + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, dentry, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + struct dentry *d; + + d = dentries[j]; + if (IS_ROOT(d)) + continue; + + au_digen_dec(d); + if (d_really_is_positive(d)) + /* todo: reset children xino? + cached children only? */ + au_iigen_dec(d_inode(d)); + } + } + +out_dpages: + au_dpages_free(&dpages); + +#if 0 + /* discard children */ + dentry_unhash(dentry); + dput(dentry); +#endif +out: + return err; +} + +/* + * return 0 if processed. + */ +static int hn_gen_by_inode(char *name, unsigned int nlen, struct inode *inode, + const unsigned int isdir) +{ + int err; + struct dentry *d; + struct qstr *dname; + + err = 1; + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("branch root dir was changed\n"); + err = 0; + goto out; + } + + if (!isdir) { + AuDebugOn(!name); + au_iigen_dec(inode); + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) { + spin_lock(&d->d_lock); + dname = &d->d_name; + if (dname->len != nlen + && memcmp(dname->name, name, nlen)) { + spin_unlock(&d->d_lock); + continue; + } + err = 0; + au_digen_dec(d); + spin_unlock(&d->d_lock); + break; + } + spin_unlock(&inode->i_lock); + } else { + au_fset_si(au_sbi(inode->i_sb), FAILED_REFRESH_DIR); + d = d_find_any_alias(inode); + if (!d) { + au_iigen_dec(inode); + goto out; + } + + spin_lock(&d->d_lock); + dname = &d->d_name; + if (dname->len == nlen && !memcmp(dname->name, name, nlen)) { + spin_unlock(&d->d_lock); + err = hn_gen_tree(d); + spin_lock(&d->d_lock); + } + spin_unlock(&d->d_lock); + dput(d); + } + +out: + AuTraceErr(err); + return err; +} + +static int hn_gen_by_name(struct dentry *dentry, const unsigned int isdir) +{ + int err; + + if (IS_ROOT(dentry)) { + pr_warn("branch root dir was changed\n"); + return 0; + } + + err = 0; + if (!isdir) { + au_digen_dec(dentry); + if (d_really_is_positive(dentry)) + au_iigen_dec(d_inode(dentry)); + } else { + au_fset_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR); + if (d_really_is_positive(dentry)) + err = hn_gen_tree(dentry); + } + + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* hnotify job flags */ +#define AuHnJob_XINO0 1 +#define AuHnJob_GEN (1 << 1) +#define AuHnJob_DIRENT (1 << 2) +#define AuHnJob_ISDIR (1 << 3) +#define AuHnJob_TRYXINO0 (1 << 4) +#define AuHnJob_MNTPNT (1 << 5) +#define au_ftest_hnjob(flags, name) ((flags) & AuHnJob_##name) +#define au_fset_hnjob(flags, name) \ + do { (flags) |= AuHnJob_##name; } while (0) +#define au_fclr_hnjob(flags, name) \ + do { (flags) &= ~AuHnJob_##name; } while (0) + +enum { + AuHn_CHILD, + AuHn_PARENT, + AuHnLast +}; + +struct au_hnotify_args { + struct inode *h_dir, *dir, *h_child_inode; + u32 mask; + unsigned int flags[AuHnLast]; + unsigned int h_child_nlen; + char h_child_name[]; +}; + +struct hn_job_args { + unsigned int flags; + struct inode *inode, *h_inode, *dir, *h_dir; + struct dentry *dentry; + char *h_name; + int h_nlen; +}; + +static int hn_job(struct hn_job_args *a) +{ + const unsigned int isdir = au_ftest_hnjob(a->flags, ISDIR); + int e; + + /* reset xino */ + if (au_ftest_hnjob(a->flags, XINO0) && a->inode) + hn_xino(a->inode, a->h_inode); /* ignore this error */ + + if (au_ftest_hnjob(a->flags, TRYXINO0) + && a->inode + && a->h_inode) { + vfsub_inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD); + if (!a->h_inode->i_nlink + && !(a->h_inode->i_state & I_LINKABLE)) + hn_xino(a->inode, a->h_inode); /* ignore this error */ + inode_unlock_shared(a->h_inode); + } + + /* make the generation obsolete */ + if (au_ftest_hnjob(a->flags, GEN)) { + e = -1; + if (a->inode) + e = hn_gen_by_inode(a->h_name, a->h_nlen, a->inode, + isdir); + if (e && a->dentry) + hn_gen_by_name(a->dentry, isdir); + /* ignore this error */ + } + + /* make dir entries obsolete */ + if (au_ftest_hnjob(a->flags, DIRENT) && a->inode) { + struct au_vdir *vdir; + + vdir = au_ivdir(a->inode); + if (vdir) + vdir->vd_jiffy = 0; + /* IMustLock(a->inode); */ + /* a->inode->i_version++; */ + } + + /* can do nothing but warn */ + if (au_ftest_hnjob(a->flags, MNTPNT) + && a->dentry + && d_mountpoint(a->dentry)) + pr_warn("mount-point %pd is removed or renamed\n", a->dentry); + + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *lookup_wlock_by_name(char *name, unsigned int nlen, + struct inode *dir) +{ + struct dentry *dentry, *d, *parent; + struct qstr *dname; + + parent = d_find_any_alias(dir); + if (!parent) + return NULL; + + dentry = NULL; + spin_lock(&parent->d_lock); + list_for_each_entry(d, &parent->d_subdirs, d_child) { + /* AuDbg("%pd\n", d); */ + spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); + dname = &d->d_name; + if (dname->len != nlen || memcmp(dname->name, name, nlen)) + goto cont_unlock; + if (au_di(d)) + au_digen_dec(d); + else + goto cont_unlock; + if (au_dcount(d) > 0) { + dentry = dget_dlock(d); + spin_unlock(&d->d_lock); + break; + } + +cont_unlock: + spin_unlock(&d->d_lock); + } + spin_unlock(&parent->d_lock); + dput(parent); + + if (dentry) + di_write_lock_child(dentry); + + return dentry; +} + +static struct inode *lookup_wlock_by_ino(struct super_block *sb, + aufs_bindex_t bindex, ino_t h_ino) +{ + struct inode *inode; + ino_t ino; + int err; + + inode = NULL; + err = au_xino_read(sb, bindex, h_ino, &ino); + if (!err && ino) + inode = ilookup(sb, ino); + if (!inode) + goto out; + + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("wrong root branch\n"); + iput(inode); + inode = NULL; + goto out; + } + + ii_write_lock_child(inode); + +out: + return inode; +} + +static void au_hn_bh(void *_args) +{ + struct au_hnotify_args *a = _args; + struct super_block *sb; + aufs_bindex_t bindex, bbot, bfound; + unsigned char xino, try_iput; + int err; + struct inode *inode; + ino_t h_ino; + struct hn_job_args args; + struct dentry *dentry; + struct au_sbinfo *sbinfo; + + AuDebugOn(!_args); + AuDebugOn(!a->h_dir); + AuDebugOn(!a->dir); + AuDebugOn(!a->mask); + AuDbg("mask 0x%x, i%lu, hi%lu, hci%lu\n", + a->mask, a->dir->i_ino, a->h_dir->i_ino, + a->h_child_inode ? a->h_child_inode->i_ino : 0); + + inode = NULL; + dentry = NULL; + /* + * do not lock a->dir->i_mutex here + * because of d_revalidate() may cause a deadlock. + */ + sb = a->dir->i_sb; + AuDebugOn(!sb); + sbinfo = au_sbi(sb); + AuDebugOn(!sbinfo); + si_write_lock(sb, AuLock_NOPLMW); + + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) + switch (a->mask & FS_EVENTS_POSS_ON_CHILD) { + case FS_MOVED_FROM: + case FS_MOVED_TO: + AuWarn1("DIRREN with UDBA may not work correctly " + "for the direct rename(2)\n"); + } + + ii_read_lock_parent(a->dir); + bfound = -1; + bbot = au_ibbot(a->dir); + for (bindex = au_ibtop(a->dir); bindex <= bbot; bindex++) + if (au_h_iptr(a->dir, bindex) == a->h_dir) { + bfound = bindex; + break; + } + ii_read_unlock(a->dir); + if (unlikely(bfound < 0)) + goto out; + + xino = !!au_opt_test(au_mntflags(sb), XINO); + h_ino = 0; + if (a->h_child_inode) + h_ino = a->h_child_inode->i_ino; + + if (a->h_child_nlen + && (au_ftest_hnjob(a->flags[AuHn_CHILD], GEN) + || au_ftest_hnjob(a->flags[AuHn_CHILD], MNTPNT))) + dentry = lookup_wlock_by_name(a->h_child_name, a->h_child_nlen, + a->dir); + try_iput = 0; + if (dentry && d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (xino && !inode && h_ino + && (au_ftest_hnjob(a->flags[AuHn_CHILD], XINO0) + || au_ftest_hnjob(a->flags[AuHn_CHILD], TRYXINO0) + || au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) { + inode = lookup_wlock_by_ino(sb, bfound, h_ino); + try_iput = 1; + } + + args.flags = a->flags[AuHn_CHILD]; + args.dentry = dentry; + args.inode = inode; + args.h_inode = a->h_child_inode; + args.dir = a->dir; + args.h_dir = a->h_dir; + args.h_name = a->h_child_name; + args.h_nlen = a->h_child_nlen; + err = hn_job(&args); + if (dentry) { + if (au_di(dentry)) + di_write_unlock(dentry); + dput(dentry); + } + if (inode && try_iput) { + ii_write_unlock(inode); + iput(inode); + } + + ii_write_lock_parent(a->dir); + args.flags = a->flags[AuHn_PARENT]; + args.dentry = NULL; + args.inode = a->dir; + args.h_inode = a->h_dir; + args.dir = NULL; + args.h_dir = NULL; + args.h_name = NULL; + args.h_nlen = 0; + err = hn_job(&args); + ii_write_unlock(a->dir); + +out: + iput(a->h_child_inode); + iput(a->h_dir); + iput(a->dir); + si_write_unlock(sb); + au_nwt_done(&sbinfo->si_nowait); + au_kfree_rcu(a); +} + +/* ---------------------------------------------------------------------- */ + +int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask, + struct qstr *h_child_qstr, struct inode *h_child_inode) +{ + int err, len; + unsigned int flags[AuHnLast], f; + unsigned char isdir, isroot, wh; + struct inode *dir; + struct au_hnotify_args *args; + char *p, *h_child_name; + + err = 0; + AuDebugOn(!hnotify || !hnotify->hn_aufs_inode); + dir = igrab(hnotify->hn_aufs_inode); + if (!dir) + goto out; + + isroot = (dir->i_ino == AUFS_ROOT_INO); + wh = 0; + h_child_name = (void *)h_child_qstr->name; + len = h_child_qstr->len; + if (h_child_name) { + if (len > AUFS_WH_PFX_LEN + && !memcmp(h_child_name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + h_child_name += AUFS_WH_PFX_LEN; + len -= AUFS_WH_PFX_LEN; + wh = 1; + } + } + + isdir = 0; + if (h_child_inode) + isdir = !!S_ISDIR(h_child_inode->i_mode); + flags[AuHn_PARENT] = AuHnJob_ISDIR; + flags[AuHn_CHILD] = 0; + if (isdir) + flags[AuHn_CHILD] = AuHnJob_ISDIR; + au_fset_hnjob(flags[AuHn_PARENT], DIRENT); + au_fset_hnjob(flags[AuHn_CHILD], GEN); + switch (mask & FS_EVENTS_POSS_ON_CHILD) { + case FS_MOVED_FROM: + case FS_MOVED_TO: + au_fset_hnjob(flags[AuHn_CHILD], XINO0); + au_fset_hnjob(flags[AuHn_CHILD], MNTPNT); + /*FALLTHROUGH*/ + case FS_CREATE: + AuDebugOn(!h_child_name); + break; + + case FS_DELETE: + /* + * aufs never be able to get this child inode. + * revalidation should be in d_revalidate() + * by checking i_nlink, i_generation or d_unhashed(). + */ + AuDebugOn(!h_child_name); + au_fset_hnjob(flags[AuHn_CHILD], TRYXINO0); + au_fset_hnjob(flags[AuHn_CHILD], MNTPNT); + break; + + default: + AuDebugOn(1); + } + + if (wh) + h_child_inode = NULL; + + err = -ENOMEM; + /* iput() and kfree() will be called in au_hnotify() */ + args = kmalloc(sizeof(*args) + len + 1, GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); + iput(dir); + goto out; + } + args->flags[AuHn_PARENT] = flags[AuHn_PARENT]; + args->flags[AuHn_CHILD] = flags[AuHn_CHILD]; + args->mask = mask; + args->dir = dir; + args->h_dir = igrab(h_dir); + if (h_child_inode) + h_child_inode = igrab(h_child_inode); /* can be NULL */ + args->h_child_inode = h_child_inode; + args->h_child_nlen = len; + if (len) { + p = (void *)args; + p += sizeof(*args); + memcpy(p, h_child_name, len); + p[len] = 0; + } + + /* NFS fires the event for silly-renamed one from kworker */ + f = 0; + if (!dir->i_nlink + || (au_test_nfs(h_dir->i_sb) && (mask & FS_DELETE))) + f = AuWkq_NEST; + err = au_wkq_nowait(au_hn_bh, args, dir->i_sb, f); + if (unlikely(err)) { + pr_err("wkq %d\n", err); + iput(args->h_child_inode); + iput(args->h_dir); + iput(args->dir); + au_kfree_rcu(args); + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm) +{ + int err; + + AuDebugOn(!(udba & AuOptMask_UDBA)); + + err = 0; + if (au_hnotify_op.reset_br) + err = au_hnotify_op.reset_br(udba, br, perm); + + return err; +} + +int au_hnotify_init_br(struct au_branch *br, int perm) +{ + int err; + + err = 0; + if (au_hnotify_op.init_br) + err = au_hnotify_op.init_br(br, perm); + + return err; +} + +void au_hnotify_fin_br(struct au_branch *br) +{ + if (au_hnotify_op.fin_br) + au_hnotify_op.fin_br(br); +} + +static void au_hn_destroy_cache(void) +{ + kmem_cache_destroy(au_cache[AuCache_HNOTIFY]); + au_cache[AuCache_HNOTIFY] = NULL; +} + +int __init au_hnotify_init(void) +{ + int err; + + err = -ENOMEM; + au_cache[AuCache_HNOTIFY] = AuCache(au_hnotify); + if (au_cache[AuCache_HNOTIFY]) { + err = 0; + if (au_hnotify_op.init) + err = au_hnotify_op.init(); + if (unlikely(err)) + au_hn_destroy_cache(); + } + AuTraceErr(err); + return err; +} + +void au_hnotify_fin(void) +{ + if (au_hnotify_op.fin) + au_hnotify_op.fin(); + + /* cf. au_cache_fin() */ + if (au_cache[AuCache_HNOTIFY]) + au_hn_destroy_cache(); +} diff --git a/fs/aufs/i_op.c b/fs/aufs/i_op.c new file mode 100644 index 0000000000000..14cbe48ee8ff1 --- /dev/null +++ b/fs/aufs/i_op.c @@ -0,0 +1,1508 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (except add/del/rename) + */ + +#include +#include +#include +#include +#include "aufs.h" + +static int h_permission(struct inode *h_inode, int mask, + struct path *h_path, int brperm) +{ + int err; + const unsigned char write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + + err = -EPERM; + if (write_mask && IS_IMMUTABLE(h_inode)) + goto out; + + err = -EACCES; + if (((mask & MAY_EXEC) + && S_ISREG(h_inode->i_mode) + && (path_noexec(h_path) + || !(h_inode->i_mode & 0111)))) + goto out; + + /* + * - skip the lower fs test in the case of write to ro branch. + * - nfs dir permission write check is optimized, but a policy for + * link/rename requires a real check. + * - nfs always sets MS_POSIXACL regardless its mount option 'noacl.' + * in this case, generic_permission() returns -EOPNOTSUPP. + */ + if ((write_mask && !au_br_writable(brperm)) + || (au_test_nfs(h_inode->i_sb) && S_ISDIR(h_inode->i_mode) + && write_mask && !(mask & MAY_READ)) + || !h_inode->i_op->permission) { + /* AuLabel(generic_permission); */ + /* AuDbg("get_acl %pf\n", h_inode->i_op->get_acl); */ + err = generic_permission(h_inode, mask); + if (err == -EOPNOTSUPP && au_test_nfs_noacl(h_inode)) + err = h_inode->i_op->permission(h_inode, mask); + AuTraceErr(err); + } else { + /* AuLabel(h_inode->permission); */ + err = h_inode->i_op->permission(h_inode, mask); + AuTraceErr(err); + } + + if (!err) + err = devcgroup_inode_permission(h_inode, mask); + if (!err) + err = security_inode_permission(h_inode, mask); + +#if 0 + if (!err) { + /* todo: do we need to call ima_path_check()? */ + struct path h_path = { + .dentry = + .mnt = h_mnt + }; + err = ima_path_check(&h_path, + mask & (MAY_READ | MAY_WRITE | MAY_EXEC), + IMA_COUNT_LEAVE); + } +#endif + +out: + return err; +} + +static int aufs_permission(struct inode *inode, int mask) +{ + int err; + aufs_bindex_t bindex, bbot; + const unsigned char isdir = !!S_ISDIR(inode->i_mode), + write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + struct inode *h_inode; + struct super_block *sb; + struct au_branch *br; + + /* todo: support rcu-walk? */ + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH); + ii_read_lock_child(inode); +#if 0 + err = au_iigen_test(inode, au_sigen(sb)); + if (unlikely(err)) + goto out; +#endif + + if (!isdir + || write_mask + || au_opt_test(au_mntflags(sb), DIRPERM1)) { + err = au_busy_or_stale(); + h_inode = au_h_iptr(inode, au_ibtop(inode)); + if (unlikely(!h_inode + || (h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT))) + goto out; + + err = 0; + bindex = au_ibtop(inode); + br = au_sbr(sb, bindex); + err = h_permission(h_inode, mask, &br->br_path, br->br_perm); + if (write_mask + && !err + && !special_file(h_inode->i_mode)) { + /* test whether the upper writable branch exists */ + err = -EROFS; + for (; bindex >= 0; bindex--) + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = 0; + break; + } + } + goto out; + } + + /* non-write to dir */ + err = 0; + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); !err && bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (h_inode) { + err = au_busy_or_stale(); + if (unlikely(!S_ISDIR(h_inode->i_mode))) + break; + + br = au_sbr(sb, bindex); + err = h_permission(h_inode, mask, &br->br_path, + br->br_perm); + } + } + +out: + ii_read_unlock(inode); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct dentry *ret, *parent; + struct inode *inode; + struct super_block *sb; + int err, npositive; + + IMustLock(dir); + + /* todo: support rcu-walk? */ + ret = ERR_PTR(-ECHILD); + if (flags & LOOKUP_RCU) + goto out; + + ret = ERR_PTR(-ENAMETOOLONG); + if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + goto out; + + sb = dir->i_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + ret = ERR_PTR(err); + if (unlikely(err)) + goto out; + + err = au_di_init(dentry); + ret = ERR_PTR(err); + if (unlikely(err)) + goto out_si; + + inode = NULL; + npositive = 0; /* suppress a warning */ + parent = dentry->d_parent; /* dir inode is locked */ + di_read_lock_parent(parent, AuLock_IR); + err = au_alive_dir(parent); + if (!err) + err = au_digen_test(parent, au_sigen(sb)); + if (!err) { + /* regardless LOOKUP_CREATE, always ALLOW_NEG */ + npositive = au_lkup_dentry(dentry, au_dbtop(parent), + AuLkup_ALLOW_NEG); + err = npositive; + } + di_read_unlock(parent, AuLock_IR); + ret = ERR_PTR(err); + if (unlikely(err < 0)) + goto out_unlock; + + if (npositive) { + inode = au_new_inode(dentry, /*must_new*/0); + if (IS_ERR(inode)) { + ret = (void *)inode; + inode = NULL; + goto out_unlock; + } + } + + if (inode) + atomic_inc(&inode->i_count); + ret = d_splice_alias(inode, dentry); +#if 0 + if (unlikely(d_need_lookup(dentry))) { + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~DCACHE_NEED_LOOKUP; + spin_unlock(&dentry->d_lock); + } else +#endif + if (inode) { + if (!IS_ERR(ret)) { + iput(inode); + if (ret && ret != dentry) + ii_write_unlock(inode); + } else { + ii_write_unlock(inode); + iput(inode); + inode = NULL; + } + } + +out_unlock: + di_write_unlock(dentry); +out_si: + si_read_unlock(sb); +out: + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* + * very dirty and complicated aufs ->atomic_open(). + * aufs_atomic_open() + * + au_aopen_or_create() + * + add_simple() + * + vfsub_atomic_open() + * + branch fs ->atomic_open() + * may call the actual 'open' for h_file + * + inc br_nfiles only if opened + * + au_aopen_no_open() or au_aopen_do_open() + * + * au_aopen_do_open() + * + finish_open() + * + au_do_aopen() + * + au_do_open() the body of all 'open' + * + au_do_open_nondir() + * set the passed h_file + * + * au_aopen_no_open() + * + finish_no_open() + */ + +struct aopen_node { + struct hlist_bl_node hblist; + struct file *file, *h_file; +}; + +static int au_do_aopen(struct inode *inode, struct file *file) +{ + struct hlist_bl_head *aopen; + struct hlist_bl_node *pos; + struct aopen_node *node; + struct au_do_open_args args = { + .aopen = 1, + .open = au_do_open_nondir + }; + + aopen = &au_sbi(inode->i_sb)->si_aopen; + hlist_bl_lock(aopen); + hlist_bl_for_each_entry(node, pos, aopen, hblist) + if (node->file == file) { + args.h_file = node->h_file; + break; + } + hlist_bl_unlock(aopen); + /* AuDebugOn(!args.h_file); */ + + return au_do_open(file, &args); +} + +static int au_aopen_do_open(struct file *file, struct dentry *dentry, + struct aopen_node *aopen_node, int *opened) +{ + int err; + struct hlist_bl_head *aopen; + + AuLabel(here); + aopen = &au_sbi(dentry->d_sb)->si_aopen; + au_hbl_add(&aopen_node->hblist, aopen); + err = finish_open(file, dentry, au_do_aopen, opened); + au_hbl_del(&aopen_node->hblist, aopen); + /* AuDbgFile(file); */ + AuDbg("%pd%s%s\n", dentry, + (*opened & FILE_CREATED) ? " created" : "", + (*opened & FILE_OPENED) ? " opened" : ""); + + AuTraceErr(err); + return err; +} + +static int au_aopen_no_open(struct file *file, struct dentry *dentry) +{ + int err; + + AuLabel(here); + dget(dentry); + err = finish_no_open(file, dentry); + + AuTraceErr(err); + return err; +} + +static int aufs_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned int open_flag, + umode_t create_mode, int *opened) +{ + int err, did_open, h_opened = *opened; + unsigned int lkup_flags; + aufs_bindex_t bindex; + struct super_block *sb; + struct dentry *parent, *d; + struct vfsub_aopen_args args = { + .open_flag = open_flag, + .create_mode = create_mode, + .opened = &h_opened + }; + struct aopen_node aopen_node = { + .file = file + }; + + IMustLock(dir); + AuDbg("open_flag 0%o\n", open_flag); + AuDbgDentry(dentry); + + err = 0; + if (!au_di(dentry)) { + lkup_flags = LOOKUP_OPEN; + if (open_flag & O_CREAT) + lkup_flags |= LOOKUP_CREATE; + d = aufs_lookup(dir, dentry, lkup_flags); + if (IS_ERR(d)) { + err = PTR_ERR(d); + AuTraceErr(err); + goto out; + } else if (d) { + /* + * obsoleted dentry found. + * another error will be returned later. + */ + d_drop(d); + AuDbgDentry(d); + dput(d); + } + AuDbgDentry(dentry); + } + + if (d_is_positive(dentry) + || d_unhashed(dentry) + || d_unlinked(dentry) + || !(open_flag & O_CREAT)) { + err = au_aopen_no_open(file, dentry); + goto out; /* success */ + } + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN); + if (unlikely(err)) + goto out; + + sb = dentry->d_sb; + parent = dentry->d_parent; /* dir is locked */ + di_write_lock_parent(parent); + err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG); + if (unlikely(err < 0)) + goto out_parent; + + AuDbgDentry(dentry); + if (d_is_positive(dentry)) { + err = au_aopen_no_open(file, dentry); + goto out_parent; /* success */ + } + + args.file = get_empty_filp(); + err = PTR_ERR(args.file); + if (IS_ERR(args.file)) + goto out_parent; + args.file->f_flags = file->f_flags; + + bindex = au_dbtop(dentry); + err = au_aopen_or_create(dir, dentry, &args); + AuTraceErr(err); + AuDbg("h_opened 0x%x\n", h_opened); + *opened |= h_opened & ~FILE_OPENED; + did_open = !!(h_opened & FILE_OPENED); + if (!did_open) { + put_filp(args.file); + args.file = NULL; + } + di_write_unlock(parent); + di_write_unlock(dentry); + if (unlikely(err < 0)) { + if (args.file) + fput(args.file); + goto out_sb; + } + + AuDbgFile(args.file); + if (!did_open) + err = au_aopen_no_open(file, dentry); + else { + aopen_node.h_file = args.file; + err = au_aopen_do_open(file, dentry, &aopen_node, opened); + } + if (unlikely(err < 0)) { + if (args.file) + fput(args.file); + if (did_open) + au_lcnt_dec(&args.br->br_nfiles); + } + goto out_sb; /* success */ + +out_parent: + di_write_unlock(parent); + di_write_unlock(dentry); +out_sb: + si_read_unlock(sb); +out: + AuTraceErr(err); + AuDbgFile(file); + return err; +} + + +/* ---------------------------------------------------------------------- */ + +static int au_wr_dir_cpup(struct dentry *dentry, struct dentry *parent, + const unsigned char add_entry, aufs_bindex_t bcpup, + aufs_bindex_t btop) +{ + int err; + struct dentry *h_parent; + struct inode *h_dir; + + if (add_entry) + IMustLock(d_inode(parent)); + else + di_write_lock_parent(parent); + + err = 0; + if (!au_h_dptr(parent, bcpup)) { + if (btop > bcpup) + err = au_cpup_dirs(dentry, bcpup); + else if (btop < bcpup) + err = au_cpdown_dirs(dentry, bcpup); + else + BUG(); + } + if (!err && add_entry && !au_ftest_wrdir(add_entry, TMPFILE)) { + h_parent = au_h_dptr(parent, bcpup); + h_dir = d_inode(h_parent); + vfsub_inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + err = au_lkup_neg(dentry, bcpup, /*wh*/0); + /* todo: no unlock here */ + inode_unlock_shared(h_dir); + + AuDbg("bcpup %d\n", bcpup); + if (!err) { + if (d_really_is_negative(dentry)) + au_set_h_dptr(dentry, btop, NULL); + au_update_dbrange(dentry, /*do_put_zero*/0); + } + } + + if (!add_entry) + di_write_unlock(parent); + if (!err) + err = bcpup; /* success */ + + AuTraceErr(err); + return err; +} + +/* + * decide the branch and the parent dir where we will create a new entry. + * returns new bindex or an error. + * copyup the parent dir if needed. + */ +int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry, + struct au_wr_dir_args *args) +{ + int err; + unsigned int flags; + aufs_bindex_t bcpup, btop, src_btop; + const unsigned char add_entry + = au_ftest_wrdir(args->flags, ADD_ENTRY) + | au_ftest_wrdir(args->flags, TMPFILE); + struct super_block *sb; + struct dentry *parent; + struct au_sbinfo *sbinfo; + + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + parent = dget_parent(dentry); + btop = au_dbtop(dentry); + bcpup = btop; + if (args->force_btgt < 0) { + if (src_dentry) { + src_btop = au_dbtop(src_dentry); + if (src_btop < btop) + bcpup = src_btop; + } else if (add_entry) { + flags = 0; + if (au_ftest_wrdir(args->flags, ISDIR)) + au_fset_wbr(flags, DIR); + err = AuWbrCreate(sbinfo, dentry, flags); + bcpup = err; + } + + if (bcpup < 0 || au_test_ro(sb, bcpup, d_inode(dentry))) { + if (add_entry) + err = AuWbrCopyup(sbinfo, dentry); + else { + if (!IS_ROOT(dentry)) { + di_read_lock_parent(parent, !AuLock_IR); + err = AuWbrCopyup(sbinfo, dentry); + di_read_unlock(parent, !AuLock_IR); + } else + err = AuWbrCopyup(sbinfo, dentry); + } + bcpup = err; + if (unlikely(err < 0)) + goto out; + } + } else { + bcpup = args->force_btgt; + AuDebugOn(au_test_ro(sb, bcpup, d_inode(dentry))); + } + + AuDbg("btop %d, bcpup %d\n", btop, bcpup); + err = bcpup; + if (bcpup == btop) + goto out; /* success */ + + /* copyup the new parent into the branch we process */ + err = au_wr_dir_cpup(dentry, parent, add_entry, bcpup, btop); + if (err >= 0) { + if (d_really_is_negative(dentry)) { + au_set_h_dptr(dentry, btop, NULL); + au_set_dbtop(dentry, bcpup); + au_set_dbbot(dentry, bcpup); + } + AuDebugOn(add_entry + && !au_ftest_wrdir(args->flags, TMPFILE) + && !au_h_dptr(dentry, bcpup)); + } + +out: + dput(parent); + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_pin_hdir_unlock(struct au_pin *p) +{ + if (p->hdir) + au_hn_inode_unlock(p->hdir); +} + +int au_pin_hdir_lock(struct au_pin *p) +{ + int err; + + err = 0; + if (!p->hdir) + goto out; + + /* even if an error happens later, keep this lock */ + au_hn_inode_lock_nested(p->hdir, p->lsc_hi); + + err = -EBUSY; + if (unlikely(p->hdir->hi_inode != d_inode(p->h_parent))) + goto out; + + err = 0; + if (p->h_dentry) + err = au_h_verify(p->h_dentry, p->udba, p->hdir->hi_inode, + p->h_parent, p->br); + +out: + return err; +} + +int au_pin_hdir_relock(struct au_pin *p) +{ + int err, i; + struct inode *h_i; + struct dentry *h_d[] = { + p->h_dentry, + p->h_parent + }; + + err = au_pin_hdir_lock(p); + if (unlikely(err)) + goto out; + + for (i = 0; !err && i < sizeof(h_d)/sizeof(*h_d); i++) { + if (!h_d[i]) + continue; + if (d_is_positive(h_d[i])) { + h_i = d_inode(h_d[i]); + err = !h_i->i_nlink; + } + } + +out: + return err; +} + +static void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task) +{ +#if !defined(CONFIG_RWSEM_GENERIC_SPINLOCK) && defined(CONFIG_RWSEM_SPIN_ON_OWNER) + p->hdir->hi_inode->i_rwsem.owner = task; +#endif +} + +void au_pin_hdir_acquire_nest(struct au_pin *p) +{ + if (p->hdir) { + rwsem_acquire_nest(&p->hdir->hi_inode->i_rwsem.dep_map, + p->lsc_hi, 0, NULL, _RET_IP_); + au_pin_hdir_set_owner(p, current); + } +} + +void au_pin_hdir_release(struct au_pin *p) +{ + if (p->hdir) { + au_pin_hdir_set_owner(p, p->task); + rwsem_release(&p->hdir->hi_inode->i_rwsem.dep_map, 1, _RET_IP_); + } +} + +struct dentry *au_pinned_h_parent(struct au_pin *pin) +{ + if (pin && pin->parent) + return au_h_dptr(pin->parent, pin->bindex); + return NULL; +} + +void au_unpin(struct au_pin *p) +{ + if (p->hdir) + au_pin_hdir_unlock(p); + if (p->h_mnt && au_ftest_pin(p->flags, MNT_WRITE)) + vfsub_mnt_drop_write(p->h_mnt); + if (!p->hdir) + return; + + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + iput(p->hdir->hi_inode); + dput(p->parent); + p->parent = NULL; + p->hdir = NULL; + p->h_mnt = NULL; + /* do not clear p->task */ +} + +int au_do_pin(struct au_pin *p) +{ + int err; + struct super_block *sb; + struct inode *h_dir; + + err = 0; + sb = p->dentry->d_sb; + p->br = au_sbr(sb, p->bindex); + if (IS_ROOT(p->dentry)) { + if (au_ftest_pin(p->flags, MNT_WRITE)) { + p->h_mnt = au_br_mnt(p->br); + err = vfsub_mnt_want_write(p->h_mnt); + if (unlikely(err)) { + au_fclr_pin(p->flags, MNT_WRITE); + goto out_err; + } + } + goto out; + } + + p->h_dentry = NULL; + if (p->bindex <= au_dbbot(p->dentry)) + p->h_dentry = au_h_dptr(p->dentry, p->bindex); + + p->parent = dget_parent(p->dentry); + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_lock(p->parent, AuLock_IR, p->lsc_di); + + h_dir = NULL; + p->h_parent = au_h_dptr(p->parent, p->bindex); + p->hdir = au_hi(d_inode(p->parent), p->bindex); + if (p->hdir) + h_dir = p->hdir->hi_inode; + + /* + * udba case, or + * if DI_LOCKED is not set, then p->parent may be different + * and h_parent can be NULL. + */ + if (unlikely(!p->hdir || !h_dir || !p->h_parent)) { + err = -EBUSY; + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + dput(p->parent); + p->parent = NULL; + goto out_err; + } + + if (au_ftest_pin(p->flags, MNT_WRITE)) { + p->h_mnt = au_br_mnt(p->br); + err = vfsub_mnt_want_write(p->h_mnt); + if (unlikely(err)) { + au_fclr_pin(p->flags, MNT_WRITE); + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + dput(p->parent); + p->parent = NULL; + goto out_err; + } + } + + au_igrab(h_dir); + err = au_pin_hdir_lock(p); + if (!err) + goto out; /* success */ + + au_unpin(p); + +out_err: + pr_err("err %d\n", err); + err = au_busy_or_stale(); +out: + return err; +} + +void au_pin_init(struct au_pin *p, struct dentry *dentry, + aufs_bindex_t bindex, int lsc_di, int lsc_hi, + unsigned int udba, unsigned char flags) +{ + p->dentry = dentry; + p->udba = udba; + p->lsc_di = lsc_di; + p->lsc_hi = lsc_hi; + p->flags = flags; + p->bindex = bindex; + + p->parent = NULL; + p->hdir = NULL; + p->h_mnt = NULL; + + p->h_dentry = NULL; + p->h_parent = NULL; + p->br = NULL; + p->task = current; +} + +int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex, + unsigned int udba, unsigned char flags) +{ + au_pin_init(pin, dentry, bindex, AuLsc_DI_PARENT, AuLsc_I_PARENT2, + udba, flags); + return au_do_pin(pin); +} + +/* ---------------------------------------------------------------------- */ + +/* + * ->setattr() and ->getattr() are called in various cases. + * chmod, stat: dentry is revalidated. + * fchmod, fstat: file and dentry are not revalidated, additionally they may be + * unhashed. + * for ->setattr(), ia->ia_file is passed from ftruncate only. + */ +/* todo: consolidate with do_refresh() and simple_reval_dpath() */ +int au_reval_for_attr(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *parent; + + err = 0; + if (au_digen_test(dentry, sigen)) { + parent = dget_parent(dentry); + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(dentry, parent); + di_read_unlock(parent, AuLock_IR); + dput(parent); + } + + AuTraceErr(err); + return err; +} + +int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia, + struct au_icpup_args *a) +{ + int err; + loff_t sz; + aufs_bindex_t btop, ibtop; + struct dentry *hi_wh, *parent; + struct inode *inode; + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = 0 + }; + + if (d_is_dir(dentry)) + au_fset_wrdir(wr_dir_args.flags, ISDIR); + /* plink or hi_wh() case */ + btop = au_dbtop(dentry); + inode = d_inode(dentry); + ibtop = au_ibtop(inode); + if (btop != ibtop && !au_test_ro(inode->i_sb, ibtop, inode)) + wr_dir_args.force_btgt = ibtop; + err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args); + if (unlikely(err < 0)) + goto out; + a->btgt = err; + if (err != btop) + au_fset_icpup(a->flags, DID_CPUP); + + err = 0; + a->pin_flags = AuPin_MNT_WRITE; + parent = NULL; + if (!IS_ROOT(dentry)) { + au_fset_pin(a->pin_flags, DI_LOCKED); + parent = dget_parent(dentry); + di_write_lock_parent(parent); + } + + err = au_pin(&a->pin, dentry, a->btgt, a->udba, a->pin_flags); + if (unlikely(err)) + goto out_parent; + + sz = -1; + a->h_path.dentry = au_h_dptr(dentry, btop); + a->h_inode = d_inode(a->h_path.dentry); + if (ia && (ia->ia_valid & ATTR_SIZE)) { + vfsub_inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD); + if (ia->ia_size < i_size_read(a->h_inode)) + sz = ia->ia_size; + inode_unlock_shared(a->h_inode); + } + + hi_wh = NULL; + if (au_ftest_icpup(a->flags, DID_CPUP) && d_unlinked(dentry)) { + hi_wh = au_hi_wh(inode, a->btgt); + if (!hi_wh) { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->btgt, + .bsrc = -1, + .len = sz, + .pin = &a->pin + }; + err = au_sio_cpup_wh(&cpg, /*file*/NULL); + if (unlikely(err)) + goto out_unlock; + hi_wh = au_hi_wh(inode, a->btgt); + /* todo: revalidate hi_wh? */ + } + } + + if (parent) { + au_pin_set_parent_lflag(&a->pin, /*lflag*/0); + di_downgrade_lock(parent, AuLock_IR); + dput(parent); + parent = NULL; + } + if (!au_ftest_icpup(a->flags, DID_CPUP)) + goto out; /* success */ + + if (!d_unhashed(dentry)) { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->btgt, + .bsrc = btop, + .len = sz, + .pin = &a->pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + err = au_sio_cpup_simple(&cpg); + if (!err) + a->h_path.dentry = au_h_dptr(dentry, a->btgt); + } else if (!hi_wh) + a->h_path.dentry = au_h_dptr(dentry, a->btgt); + else + a->h_path.dentry = hi_wh; /* do not dget here */ + +out_unlock: + a->h_inode = d_inode(a->h_path.dentry); + if (!err) + goto out; /* success */ + au_unpin(&a->pin); +out_parent: + if (parent) { + di_write_unlock(parent); + dput(parent); + } +out: + if (!err) + inode_lock_nested(a->h_inode, AuLsc_I_CHILD); + return err; +} + +static int aufs_setattr(struct dentry *dentry, struct iattr *ia) +{ + int err; + struct inode *inode, *delegated; + struct super_block *sb; + struct file *file; + struct au_icpup_args *a; + + inode = d_inode(dentry); + IMustLock(inode); + + err = setattr_prepare(dentry, ia); + if (unlikely(err)) + goto out; + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + if (ia->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + ia->ia_valid &= ~ATTR_MODE; + + file = NULL; + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_kfree; + + if (ia->ia_valid & ATTR_FILE) { + /* currently ftruncate(2) only */ + AuDebugOn(!d_is_reg(dentry)); + file = ia->ia_file; + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1, + /*fi_lsc*/0); + if (unlikely(err)) + goto out_si; + ia->ia_file = au_hf_top(file); + a->udba = AuOpt_UDBA_NONE; + } else { + /* fchmod() doesn't pass ia_file */ + a->udba = au_opt_udba(sb); + di_write_lock_child(dentry); + /* no d_unlinked(), to set UDBA_NONE for root */ + if (d_unhashed(dentry)) + a->udba = AuOpt_UDBA_NONE; + if (a->udba != AuOpt_UDBA_NONE) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_for_attr(dentry, au_sigen(sb)); + if (unlikely(err)) + goto out_dentry; + } + } + + err = au_pin_and_icpup(dentry, ia, a); + if (unlikely(err < 0)) + goto out_dentry; + if (au_ftest_icpup(a->flags, DID_CPUP)) { + ia->ia_file = NULL; + ia->ia_valid &= ~ATTR_FILE; + } + + a->h_path.mnt = au_sbr_mnt(sb, a->btgt); + if ((ia->ia_valid & (ATTR_MODE | ATTR_CTIME)) + == (ATTR_MODE | ATTR_CTIME)) { + err = security_path_chmod(&a->h_path, ia->ia_mode); + if (unlikely(err)) + goto out_unlock; + } else if ((ia->ia_valid & (ATTR_UID | ATTR_GID)) + && (ia->ia_valid & ATTR_CTIME)) { + err = security_path_chown(&a->h_path, ia->ia_uid, ia->ia_gid); + if (unlikely(err)) + goto out_unlock; + } + + if (ia->ia_valid & ATTR_SIZE) { + struct file *f; + + if (ia->ia_size < i_size_read(inode)) + /* unmap only */ + truncate_setsize(inode, ia->ia_size); + + f = NULL; + if (ia->ia_valid & ATTR_FILE) + f = ia->ia_file; + inode_unlock(a->h_inode); + err = vfsub_trunc(&a->h_path, ia->ia_size, ia->ia_valid, f); + inode_lock_nested(a->h_inode, AuLsc_I_CHILD); + } else { + delegated = NULL; + while (1) { + err = vfsub_notify_change(&a->h_path, ia, &delegated); + if (delegated) { + err = break_deleg_wait(&delegated); + if (!err) + continue; + } + break; + } + } + /* + * regardless aufs 'acl' option setting. + * why don't all acl-aware fs call this func from their ->setattr()? + */ + if (!err && (ia->ia_valid & ATTR_MODE)) + err = vfsub_acl_chmod(a->h_inode, ia->ia_mode); + if (!err) + au_cpup_attr_changeable(inode); + +out_unlock: + inode_unlock(a->h_inode); + au_unpin(&a->pin); + if (unlikely(err)) + au_update_dbtop(dentry); +out_dentry: + di_write_unlock(dentry); + if (file) { + fi_write_unlock(file); + ia->ia_file = file; + ia->ia_valid |= ATTR_FILE; + } +out_si: + si_read_unlock(sb); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} + +#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL) +static int au_h_path_to_set_attr(struct dentry *dentry, + struct au_icpup_args *a, struct path *h_path) +{ + int err; + struct super_block *sb; + + sb = dentry->d_sb; + a->udba = au_opt_udba(sb); + /* no d_unlinked(), to set UDBA_NONE for root */ + if (d_unhashed(dentry)) + a->udba = AuOpt_UDBA_NONE; + if (a->udba != AuOpt_UDBA_NONE) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_for_attr(dentry, au_sigen(sb)); + if (unlikely(err)) + goto out; + } + err = au_pin_and_icpup(dentry, /*ia*/NULL, a); + if (unlikely(err < 0)) + goto out; + + h_path->dentry = a->h_path.dentry; + h_path->mnt = au_sbr_mnt(sb, a->btgt); + +out: + return err; +} + +ssize_t au_sxattr(struct dentry *dentry, struct inode *inode, + struct au_sxattr *arg) +{ + int err; + struct path h_path; + struct super_block *sb; + struct au_icpup_args *a; + struct inode *h_inode; + + IMustLock(inode); + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_kfree; + + h_path.dentry = NULL; /* silence gcc */ + di_write_lock_child(dentry); + err = au_h_path_to_set_attr(dentry, a, &h_path); + if (unlikely(err)) + goto out_di; + + inode_unlock(a->h_inode); + switch (arg->type) { + case AU_XATTR_SET: + AuDebugOn(d_is_negative(h_path.dentry)); + err = vfsub_setxattr(h_path.dentry, + arg->u.set.name, arg->u.set.value, + arg->u.set.size, arg->u.set.flags); + break; + case AU_ACL_SET: + err = -EOPNOTSUPP; + h_inode = d_inode(h_path.dentry); + if (h_inode->i_op->set_acl) + /* this will call posix_acl_update_mode */ + err = h_inode->i_op->set_acl(h_inode, + arg->u.acl_set.acl, + arg->u.acl_set.type); + break; + } + if (!err) + au_cpup_attr_timesizes(inode); + + au_unpin(&a->pin); + if (unlikely(err)) + au_update_dbtop(dentry); + +out_di: + di_write_unlock(dentry); + si_read_unlock(sb); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} +#endif + +static void au_refresh_iattr(struct inode *inode, struct kstat *st, + unsigned int nlink) +{ + unsigned int n; + + inode->i_mode = st->mode; + /* don't i_[ug]id_write() here */ + inode->i_uid = st->uid; + inode->i_gid = st->gid; + inode->i_atime = st->atime; + inode->i_mtime = st->mtime; + inode->i_ctime = st->ctime; + + au_cpup_attr_nlink(inode, /*force*/0); + if (S_ISDIR(inode->i_mode)) { + n = inode->i_nlink; + n -= nlink; + n += st->nlink; + smp_mb(); /* for i_nlink */ + /* 0 can happen */ + set_nlink(inode, n); + } + + spin_lock(&inode->i_lock); + inode->i_blocks = st->blocks; + i_size_write(inode, st->size); + spin_unlock(&inode->i_lock); +} + +/* + * common routine for aufs_getattr() and au_getxattr(). + * returns zero or negative (an error). + * @dentry will be read-locked in success. + */ +int au_h_path_getattr(struct dentry *dentry, int force, struct path *h_path, + int locked) +{ + int err; + unsigned int mnt_flags, sigen; + unsigned char udba_none; + aufs_bindex_t bindex; + struct super_block *sb, *h_sb; + struct inode *inode; + + h_path->mnt = NULL; + h_path->dentry = NULL; + + err = 0; + sb = dentry->d_sb; + mnt_flags = au_mntflags(sb); + udba_none = !!au_opt_test(mnt_flags, UDBA_NONE); + + if (unlikely(locked)) + goto body; /* skip locking dinfo */ + + /* support fstat(2) */ + if (!d_unlinked(dentry) && !udba_none) { + sigen = au_sigen(sb); + err = au_digen_test(dentry, sigen); + if (!err) { + di_read_lock_child(dentry, AuLock_IR); + err = au_dbrange_test(dentry); + if (unlikely(err)) { + di_read_unlock(dentry, AuLock_IR); + goto out; + } + } else { + AuDebugOn(IS_ROOT(dentry)); + di_write_lock_child(dentry); + err = au_dbrange_test(dentry); + if (!err) + err = au_reval_for_attr(dentry, sigen); + if (!err) + di_downgrade_lock(dentry, AuLock_IR); + else { + di_write_unlock(dentry); + goto out; + } + } + } else + di_read_lock_child(dentry, AuLock_IR); + +body: + inode = d_inode(dentry); + bindex = au_ibtop(inode); + h_path->mnt = au_sbr_mnt(sb, bindex); + h_sb = h_path->mnt->mnt_sb; + if (!force + && !au_test_fs_bad_iattr(h_sb) + && udba_none) + goto out; /* success */ + + if (au_dbtop(dentry) == bindex) + h_path->dentry = au_h_dptr(dentry, bindex); + else if (au_opt_test(mnt_flags, PLINK) && au_plink_test(inode)) { + h_path->dentry = au_plink_lkup(inode, bindex); + if (IS_ERR(h_path->dentry)) + /* pretending success */ + h_path->dentry = NULL; + else + dput(h_path->dentry); + } + +out: + return err; +} + +static int aufs_getattr(const struct path *path, struct kstat *st, + u32 request, unsigned int query) +{ + int err; + unsigned char positive; + struct path h_path; + struct dentry *dentry; + struct inode *inode; + struct super_block *sb; + + dentry = path->dentry; + inode = d_inode(dentry); + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + err = au_h_path_getattr(dentry, /*force*/0, &h_path, /*locked*/0); + if (unlikely(err)) + goto out_si; + if (unlikely(!h_path.dentry)) + /* illegally overlapped or something */ + goto out_fill; /* pretending success */ + + positive = d_is_positive(h_path.dentry); + if (positive) + /* no vfsub version */ + err = vfs_getattr(&h_path, st, request, query); + if (!err) { + if (positive) + au_refresh_iattr(inode, st, + d_inode(h_path.dentry)->i_nlink); + goto out_fill; /* success */ + } + AuTraceErr(err); + goto out_di; + +out_fill: + generic_fillattr(inode, st); +out_di: + di_read_unlock(dentry, AuLock_IR); +out_si: + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static const char *aufs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + const char *ret; + struct dentry *h_dentry; + struct inode *h_inode; + int err; + aufs_bindex_t bindex; + + ret = NULL; /* suppress a warning */ + err = -ECHILD; + if (!dentry) + goto out; + + err = aufs_read_lock(dentry, AuLock_IR | AuLock_GEN); + if (unlikely(err)) + goto out; + + err = au_d_hashed_positive(dentry); + if (unlikely(err)) + goto out_unlock; + + err = -EINVAL; + inode = d_inode(dentry); + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (unlikely(!h_inode->i_op->get_link)) + goto out_unlock; + + err = -EBUSY; + h_dentry = NULL; + if (au_dbtop(dentry) <= bindex) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) + dget(h_dentry); + } + if (!h_dentry) { + h_dentry = d_find_any_alias(h_inode); + if (IS_ERR(h_dentry)) { + err = PTR_ERR(h_dentry); + goto out_unlock; + } + } + if (unlikely(!h_dentry)) + goto out_unlock; + + err = 0; + AuDbg("%pf\n", h_inode->i_op->get_link); + AuDbgDentry(h_dentry); + ret = vfs_get_link(h_dentry, done); + dput(h_dentry); + if (IS_ERR(ret)) + err = PTR_ERR(ret); + +out_unlock: + aufs_read_unlock(dentry, AuLock_IR); +out: + if (unlikely(err)) + ret = ERR_PTR(err); + AuTraceErrPtr(ret); + return ret; +} + +/* ---------------------------------------------------------------------- */ + +static int au_is_special(struct inode *inode) +{ + return (inode->i_mode & (S_IFBLK | S_IFCHR | S_IFIFO | S_IFSOCK)); +} + +static int aufs_update_time(struct inode *inode, struct timespec *ts, int flags) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; + struct vfsmount *h_mnt; + + sb = inode->i_sb; + WARN_ONCE((flags & S_ATIME) && !IS_NOATIME(inode), + "unexpected s_flags 0x%lx", sb->s_flags); + + /* mmap_sem might be acquired already, cf. aufs_mmap() */ + lockdep_off(); + si_read_lock(sb, AuLock_FLUSH); + ii_write_lock_child(inode); + + err = 0; + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (!au_test_ro(sb, bindex, inode)) { + h_mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mnt_want_write(h_mnt); + if (!err) { + err = vfsub_update_time(h_inode, ts, flags); + vfsub_mnt_drop_write(h_mnt); + } + } else if (au_is_special(h_inode)) { + /* + * Never copy-up here. + * These special files may already be opened and used for + * communicating. If we copied it up, then the communication + * would be corrupted. + */ + AuWarn1("timestamps for i%lu are ignored " + "since it is on readonly branch (hi%lu).\n", + inode->i_ino, h_inode->i_ino); + } else if (flags & ~S_ATIME) { + err = -EIO; + AuIOErr1("unexpected flags 0x%x\n", flags); + AuDebugOn(1); + } + + if (!err) + au_cpup_attr_timesizes(inode); + ii_write_unlock(inode); + si_read_unlock(sb); + lockdep_on(); + + if (!err && (flags & S_VERSION)) + inode_inc_iversion(inode); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* no getattr version will be set by module.c:aufs_init() */ +struct inode_operations aufs_iop_nogetattr[AuIop_Last], + aufs_iop[] = { + [AuIop_SYMLINK] = { + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, /* unsupport for symlink? */ +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .get_link = aufs_get_link, + + /* .update_time = aufs_update_time */ + }, + [AuIop_DIR] = { + .create = aufs_create, + .lookup = aufs_lookup, + .link = aufs_link, + .unlink = aufs_unlink, + .symlink = aufs_symlink, + .mkdir = aufs_mkdir, + .rmdir = aufs_rmdir, + .mknod = aufs_mknod, + .rename = aufs_rename, + + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .update_time = aufs_update_time, + .atomic_open = aufs_atomic_open, + .tmpfile = aufs_tmpfile + }, + [AuIop_OTHER] = { + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .update_time = aufs_update_time + } +}; diff --git a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c new file mode 100644 index 0000000000000..f8c0724c8a569 --- /dev/null +++ b/fs/aufs/i_op_add.c @@ -0,0 +1,934 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (add entry) + */ + +#include "aufs.h" + +/* + * final procedure of adding a new entry, except link(2). + * remove whiteout, instantiate, copyup the parent dir's times and size + * and update version. + * if it failed, re-create the removed whiteout. + */ +static int epilog(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct dentry *dentry) +{ + int err, rerr; + aufs_bindex_t bwh; + struct path h_path; + struct super_block *sb; + struct inode *inode, *h_dir; + struct dentry *wh; + + bwh = -1; + sb = dir->i_sb; + if (wh_dentry) { + h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */ + IMustLock(h_dir); + AuDebugOn(au_h_iptr(dir, bindex) != h_dir); + bwh = au_dbwh(dentry); + h_path.dentry = wh_dentry; + h_path.mnt = au_sbr_mnt(sb, bindex); + err = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path, + dentry); + if (unlikely(err)) + goto out; + } + + inode = au_new_inode(dentry, /*must_new*/1); + if (!IS_ERR(inode)) { + d_instantiate(dentry, inode); + dir = d_inode(dentry->d_parent); /* dir inode is locked */ + IMustLock(dir); + au_dir_ts(dir, bindex); + dir->i_version++; + au_fhsm_wrote(sb, bindex, /*force*/0); + return 0; /* success */ + } + + err = PTR_ERR(inode); + if (!wh_dentry) + goto out; + + /* revert */ + /* dir inode is locked */ + wh = au_wh_create(dentry, bwh, wh_dentry->d_parent); + rerr = PTR_ERR(wh); + if (IS_ERR(wh)) { + AuIOErr("%pd reverting whiteout failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } else + dput(wh); + +out: + return err; +} + +static int au_d_may_add(struct dentry *dentry) +{ + int err; + + err = 0; + if (unlikely(d_unhashed(dentry))) + err = -ENOENT; + if (unlikely(d_really_is_positive(dentry))) + err = -EEXIST; + return err; +} + +/* + * simple tests for the adding inode operations. + * following the checks in vfs, plus the parent-child relationship. + */ +int au_may_add(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir) +{ + int err; + umode_t h_mode; + struct dentry *h_dentry; + struct inode *h_inode; + + err = -ENAMETOOLONG; + if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + goto out; + + h_dentry = au_h_dptr(dentry, bindex); + if (d_really_is_negative(dentry)) { + err = -EEXIST; + if (unlikely(d_is_positive(h_dentry))) + goto out; + } else { + /* rename(2) case */ + err = -EIO; + if (unlikely(d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + if (unlikely(!h_inode->i_nlink)) + goto out; + + h_mode = h_inode->i_mode; + if (!isdir) { + err = -EISDIR; + if (unlikely(S_ISDIR(h_mode))) + goto out; + } else if (unlikely(!S_ISDIR(h_mode))) { + err = -ENOTDIR; + goto out; + } + } + + err = 0; + /* expected parent dir is locked */ + if (unlikely(h_parent != h_dentry->d_parent)) + err = -EIO; + +out: + AuTraceErr(err); + return err; +} + +/* + * initial procedure of adding a new entry. + * prepare writable branch and the parent dir, lock it, + * and lookup whiteout for the new entry. + */ +static struct dentry* +lock_hdir_lkup_wh(struct dentry *dentry, struct au_dtime *dt, + struct dentry *src_dentry, struct au_pin *pin, + struct au_wr_dir_args *wr_dir_args) +{ + struct dentry *wh_dentry, *h_parent; + struct super_block *sb; + struct au_branch *br; + int err; + unsigned int udba; + aufs_bindex_t bcpup; + + AuDbg("%pd\n", dentry); + + err = au_wr_dir(dentry, src_dentry, wr_dir_args); + bcpup = err; + wh_dentry = ERR_PTR(err); + if (unlikely(err < 0)) + goto out; + + sb = dentry->d_sb; + udba = au_opt_udba(sb); + err = au_pin(pin, dentry, bcpup, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out; + + h_parent = au_pinned_h_parent(pin); + if (udba != AuOpt_UDBA_NONE + && au_dbtop(dentry) == bcpup) + err = au_may_add(dentry, bcpup, h_parent, + au_ftest_wrdir(wr_dir_args->flags, ISDIR)); + else if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + err = -ENAMETOOLONG; + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_unpin; + + br = au_sbr(sb, bcpup); + if (dt) { + struct path tmp = { + .dentry = h_parent, + .mnt = au_br_mnt(br) + }; + au_dtime_store(dt, au_pinned_parent(pin), &tmp); + } + + wh_dentry = NULL; + if (bcpup != au_dbwh(dentry)) + goto out; /* success */ + + /* + * ENAMETOOLONG here means that if we allowed create such name, then it + * would not be able to removed in the future. So we don't allow such + * name here and we don't handle ENAMETOOLONG differently here. + */ + wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, br); + +out_unpin: + if (IS_ERR(wh_dentry)) + au_unpin(pin); +out: + return wh_dentry; +} + +/* ---------------------------------------------------------------------- */ + +enum { Mknod, Symlink, Creat }; +struct simple_arg { + int type; + union { + struct { + umode_t mode; + bool want_excl; + bool try_aopen; + struct vfsub_aopen_args *aopen; + } c; + struct { + const char *symname; + } s; + struct { + umode_t mode; + dev_t dev; + } m; + } u; +}; + +static int add_simple(struct inode *dir, struct dentry *dentry, + struct simple_arg *arg) +{ + int err, rerr; + aufs_bindex_t btop; + unsigned char created; + const unsigned char try_aopen + = (arg->type == Creat && arg->u.c.try_aopen); + struct vfsub_aopen_args *aopen = arg->u.c.aopen; + struct dentry *wh_dentry, *parent; + struct inode *h_dir; + struct super_block *sb; + struct au_branch *br; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + struct path h_path; + struct au_wr_dir_args wr_dir_args; + } *a; + + AuDbg("%pd\n", dentry); + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + a->wr_dir_args.force_btgt = -1; + a->wr_dir_args.flags = AuWrDir_ADD_ENTRY; + + parent = dentry->d_parent; /* dir inode is locked */ + if (!try_aopen) { + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + } + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + if (!try_aopen) + di_write_lock_parent(parent); + wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL, + &a->pin, &a->wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + btop = au_dbtop(dentry); + sb = dentry->d_sb; + br = au_sbr(sb, btop); + a->h_path.dentry = au_h_dptr(dentry, btop); + a->h_path.mnt = au_br_mnt(br); + h_dir = au_pinned_h_dir(&a->pin); + switch (arg->type) { + case Creat: + if (!try_aopen || !h_dir->i_op->atomic_open) { + err = vfsub_create(h_dir, &a->h_path, arg->u.c.mode, + arg->u.c.want_excl); + created = !err; + if (!err && try_aopen) + *aopen->opened |= FILE_CREATED; + AuDbg("opened 0x%x\n", *aopen->opened); + } else { + aopen->br = br; + err = vfsub_atomic_open(h_dir, a->h_path.dentry, aopen); + AuDbg("opened 0x%x\n", *aopen->opened); + created = err >= 0 && !!(*aopen->opened & FILE_CREATED); + } + break; + case Symlink: + err = vfsub_symlink(h_dir, &a->h_path, arg->u.s.symname); + created = !err; + break; + case Mknod: + err = vfsub_mknod(h_dir, &a->h_path, arg->u.m.mode, + arg->u.m.dev); + created = !err; + break; + default: + BUG(); + } + if (unlikely(err < 0)) + goto out_unpin; + + err = epilog(dir, btop, wh_dentry, dentry); + if (!err) + goto out_unpin; /* success */ + + /* revert */ + if (created /* && d_is_positive(a->h_path.dentry) */) { + /* no delegation since it is just created */ + rerr = vfsub_unlink(h_dir, &a->h_path, /*delegated*/NULL, + /*force*/0); + if (rerr) { + AuIOErr("%pd revert failure(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&a->dt); + } + if (try_aopen && h_dir->i_op->atomic_open + && (*aopen->opened & FILE_OPENED)) + /* aopen->file is still opened */ + au_lcnt_dec(&aopen->br->br_nfiles); + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); +out_parent: + if (!try_aopen) + di_write_unlock(parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + if (!try_aopen) + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} + +int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t dev) +{ + struct simple_arg arg = { + .type = Mknod, + .u.m = { + .mode = mode, + .dev = dev + } + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +{ + struct simple_arg arg = { + .type = Symlink, + .u.s.symname = symname + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + bool want_excl) +{ + struct simple_arg arg = { + .type = Creat, + .u.c = { + .mode = mode, + .want_excl = want_excl + } + }; + return add_simple(dir, dentry, &arg); +} + +int au_aopen_or_create(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *aopen_args) +{ + struct simple_arg arg = { + .type = Creat, + .u.c = { + .mode = aopen_args->create_mode, + .want_excl = aopen_args->open_flag & O_EXCL, + .try_aopen = true, + .aopen = aopen_args + } + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct dentry *parent, *h_parent, *h_dentry; + struct inode *h_dir, *inode; + struct vfsmount *h_mnt; + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = AuWrDir_TMPFILE + }; + + /* copy-up may happen */ + inode_lock(dir); + + sb = dir->i_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + + err = au_di_init(dentry); + if (unlikely(err)) + goto out_si; + + err = -EBUSY; + parent = d_find_any_alias(dir); + AuDebugOn(!parent); + di_write_lock_parent(parent); + if (unlikely(d_inode(parent) != dir)) + goto out_parent; + + err = au_digen_test(parent, au_sigen(sb)); + if (unlikely(err)) + goto out_parent; + + bindex = au_dbtop(parent); + au_set_dbtop(dentry, bindex); + au_set_dbbot(dentry, bindex); + err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args); + bindex = err; + if (unlikely(err < 0)) + goto out_parent; + + err = -EOPNOTSUPP; + h_dir = au_h_iptr(dir, bindex); + if (unlikely(!h_dir->i_op->tmpfile)) + goto out_parent; + + h_mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mnt_want_write(h_mnt); + if (unlikely(err)) + goto out_parent; + + h_parent = au_h_dptr(parent, bindex); + h_dentry = vfs_tmpfile(h_parent, mode, /*open_flag*/0); + if (IS_ERR(h_dentry)) { + err = PTR_ERR(h_dentry); + goto out_mnt; + } + + au_set_dbtop(dentry, bindex); + au_set_dbbot(dentry, bindex); + au_set_h_dptr(dentry, bindex, dget(h_dentry)); + inode = au_new_inode(dentry, /*must_new*/1); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + au_set_h_dptr(dentry, bindex, NULL); + au_set_dbtop(dentry, -1); + au_set_dbbot(dentry, -1); + } else { + if (!inode->i_nlink) + set_nlink(inode, 1); + d_tmpfile(dentry, inode); + au_di(dentry)->di_tmpfile = 1; + + /* update without i_mutex */ + if (au_ibtop(dir) == au_dbtop(dentry)) + au_cpup_attr_timesizes(dir); + } + dput(h_dentry); + +out_mnt: + vfsub_mnt_drop_write(h_mnt); +out_parent: + di_write_unlock(parent); + dput(parent); + di_write_unlock(dentry); + if (unlikely(err)) { + au_di_fin(dentry); + dentry->d_fsdata = NULL; + } +out_si: + si_read_unlock(sb); +out: + inode_unlock(dir); + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_link_args { + aufs_bindex_t bdst, bsrc; + struct au_pin pin; + struct path h_path; + struct dentry *src_parent, *parent; +}; + +static int au_cpup_before_link(struct dentry *src_dentry, + struct au_link_args *a) +{ + int err; + struct dentry *h_src_dentry; + struct au_cp_generic cpg = { + .dentry = src_dentry, + .bdst = a->bdst, + .bsrc = a->bsrc, + .len = -1, + .pin = &a->pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN /* | AuCpup_KEEPLINO */ + }; + + di_read_lock_parent(a->src_parent, AuLock_IR); + err = au_test_and_cpup_dirs(src_dentry, a->bdst); + if (unlikely(err)) + goto out; + + h_src_dentry = au_h_dptr(src_dentry, a->bsrc); + err = au_pin(&a->pin, src_dentry, a->bdst, + au_opt_udba(src_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out; + + err = au_sio_cpup_simple(&cpg); + au_unpin(&a->pin); + +out: + di_read_unlock(a->src_parent, AuLock_IR); + return err; +} + +static int au_cpup_or_link(struct dentry *src_dentry, struct dentry *dentry, + struct au_link_args *a) +{ + int err; + unsigned char plink; + aufs_bindex_t bbot; + struct dentry *h_src_dentry; + struct inode *h_inode, *inode, *delegated; + struct super_block *sb; + struct file *h_file; + + plink = 0; + h_inode = NULL; + sb = src_dentry->d_sb; + inode = d_inode(src_dentry); + if (au_ibtop(inode) <= a->bdst) + h_inode = au_h_iptr(inode, a->bdst); + if (!h_inode || !h_inode->i_nlink) { + /* copyup src_dentry as the name of dentry. */ + bbot = au_dbbot(dentry); + if (bbot < a->bsrc) + au_set_dbbot(dentry, a->bsrc); + au_set_h_dptr(dentry, a->bsrc, + dget(au_h_dptr(src_dentry, a->bsrc))); + dget(a->h_path.dentry); + au_set_h_dptr(dentry, a->bdst, NULL); + AuDbg("temporary d_inode...\n"); + spin_lock(&dentry->d_lock); + dentry->d_inode = d_inode(src_dentry); /* tmp */ + spin_unlock(&dentry->d_lock); + h_file = au_h_open_pre(dentry, a->bsrc, /*force_wr*/0); + if (IS_ERR(h_file)) + err = PTR_ERR(h_file); + else { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->bdst, + .bsrc = -1, + .len = -1, + .pin = &a->pin, + .flags = AuCpup_KEEPLINO + }; + err = au_sio_cpup_simple(&cpg); + au_h_open_post(dentry, a->bsrc, h_file); + if (!err) { + dput(a->h_path.dentry); + a->h_path.dentry = au_h_dptr(dentry, a->bdst); + } else + au_set_h_dptr(dentry, a->bdst, + a->h_path.dentry); + } + spin_lock(&dentry->d_lock); + dentry->d_inode = NULL; /* restore */ + spin_unlock(&dentry->d_lock); + AuDbg("temporary d_inode...done\n"); + au_set_h_dptr(dentry, a->bsrc, NULL); + au_set_dbbot(dentry, bbot); + } else { + /* the inode of src_dentry already exists on a.bdst branch */ + h_src_dentry = d_find_alias(h_inode); + if (!h_src_dentry && au_plink_test(inode)) { + plink = 1; + h_src_dentry = au_plink_lkup(inode, a->bdst); + err = PTR_ERR(h_src_dentry); + if (IS_ERR(h_src_dentry)) + goto out; + + if (unlikely(d_is_negative(h_src_dentry))) { + dput(h_src_dentry); + h_src_dentry = NULL; + } + + } + if (h_src_dentry) { + delegated = NULL; + err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + dput(h_src_dentry); + } else { + AuIOErr("no dentry found for hi%lu on b%d\n", + h_inode->i_ino, a->bdst); + err = -EIO; + } + } + + if (!err && !plink) + au_plink_append(inode, a->bdst, a->h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +int aufs_link(struct dentry *src_dentry, struct inode *dir, + struct dentry *dentry) +{ + int err, rerr; + struct au_dtime dt; + struct au_link_args *a; + struct dentry *wh_dentry, *h_src_dentry; + struct inode *inode, *delegated; + struct super_block *sb; + struct au_wr_dir_args wr_dir_args = { + /* .force_btgt = -1, */ + .flags = AuWrDir_ADD_ENTRY + }; + + IMustLock(dir); + inode = d_inode(src_dentry); + IMustLock(inode); + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + a->parent = dentry->d_parent; /* dir inode is locked */ + err = aufs_read_and_write_lock2(dentry, src_dentry, + AuLock_NOPLM | AuLock_GEN); + if (unlikely(err)) + goto out_kfree; + err = au_d_linkable(src_dentry); + if (unlikely(err)) + goto out_unlock; + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + + a->src_parent = dget_parent(src_dentry); + wr_dir_args.force_btgt = au_ibtop(inode); + + di_write_lock_parent(a->parent); + wr_dir_args.force_btgt = au_wbr(dentry, wr_dir_args.force_btgt); + wh_dentry = lock_hdir_lkup_wh(dentry, &dt, src_dentry, &a->pin, + &wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + err = 0; + sb = dentry->d_sb; + a->bdst = au_dbtop(dentry); + a->h_path.dentry = au_h_dptr(dentry, a->bdst); + a->h_path.mnt = au_sbr_mnt(sb, a->bdst); + a->bsrc = au_ibtop(inode); + h_src_dentry = au_h_d_alias(src_dentry, a->bsrc); + if (!h_src_dentry && au_di(src_dentry)->di_tmpfile) + h_src_dentry = dget(au_hi_wh(inode, a->bsrc)); + if (!h_src_dentry) { + a->bsrc = au_dbtop(src_dentry); + h_src_dentry = au_h_d_alias(src_dentry, a->bsrc); + AuDebugOn(!h_src_dentry); + } else if (IS_ERR(h_src_dentry)) { + err = PTR_ERR(h_src_dentry); + goto out_parent; + } + + /* + * aufs doesn't touch the credential so + * security_dentry_create_files_as() is unnecessary. + */ + if (au_opt_test(au_mntflags(sb), PLINK)) { + if (a->bdst < a->bsrc + /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */) + err = au_cpup_or_link(src_dentry, dentry, a); + else { + delegated = NULL; + err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + dput(h_src_dentry); + } else { + /* + * copyup src_dentry to the branch we process, + * and then link(2) to it. + */ + dput(h_src_dentry); + if (a->bdst < a->bsrc + /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */) { + au_unpin(&a->pin); + di_write_unlock(a->parent); + err = au_cpup_before_link(src_dentry, a); + di_write_lock_parent(a->parent); + if (!err) + err = au_pin(&a->pin, dentry, a->bdst, + au_opt_udba(sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_wh; + } + if (!err) { + h_src_dentry = au_h_dptr(src_dentry, a->bdst); + err = -ENOENT; + if (h_src_dentry && d_is_positive(h_src_dentry)) { + delegated = NULL; + err = vfsub_link(h_src_dentry, + au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry" + " for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + } + } + if (unlikely(err)) + goto out_unpin; + + if (wh_dentry) { + a->h_path.dentry = wh_dentry; + err = au_wh_unlink_dentry(au_pinned_h_dir(&a->pin), &a->h_path, + dentry); + if (unlikely(err)) + goto out_revert; + } + + au_dir_ts(dir, a->bdst); + dir->i_version++; + inc_nlink(inode); + inode->i_ctime = dir->i_ctime; + d_instantiate(dentry, au_igrab(inode)); + if (d_unhashed(a->h_path.dentry)) + /* some filesystem calls d_drop() */ + d_drop(dentry); + /* some filesystems consume an inode even hardlink */ + au_fhsm_wrote(sb, a->bdst, /*force*/0); + goto out_unpin; /* success */ + +out_revert: + /* no delegation since it is just created */ + rerr = vfsub_unlink(au_pinned_h_dir(&a->pin), &a->h_path, + /*delegated*/NULL, /*force*/0); + if (unlikely(rerr)) { + AuIOErr("%pd reverting failed(%d, %d)\n", dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&dt); +out_unpin: + au_unpin(&a->pin); +out_wh: + dput(wh_dentry); +out_parent: + di_write_unlock(a->parent); + dput(a->src_parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + aufs_read_and_write_unlock2(dentry, src_dentry); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} + +int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err, rerr; + aufs_bindex_t bindex; + unsigned char diropq; + struct path h_path; + struct dentry *wh_dentry, *parent, *opq_dentry; + struct inode *h_inode; + struct super_block *sb; + struct { + struct au_pin pin; + struct au_dtime dt; + } *a; /* reduce the stack usage */ + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = AuWrDir_ADD_ENTRY | AuWrDir_ISDIR + }; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL, + &a->pin, &wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + sb = dentry->d_sb; + bindex = au_dbtop(dentry); + h_path.dentry = au_h_dptr(dentry, bindex); + h_path.mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mkdir(au_pinned_h_dir(&a->pin), &h_path, mode); + if (unlikely(err)) + goto out_unpin; + + /* make the dir opaque */ + diropq = 0; + h_inode = d_inode(h_path.dentry); + if (wh_dentry + || au_opt_test(au_mntflags(sb), ALWAYS_DIROPQ)) { + inode_lock_nested(h_inode, AuLsc_I_CHILD); + opq_dentry = au_diropq_create(dentry, bindex); + inode_unlock(h_inode); + err = PTR_ERR(opq_dentry); + if (IS_ERR(opq_dentry)) + goto out_dir; + dput(opq_dentry); + diropq = 1; + } + + err = epilog(dir, bindex, wh_dentry, dentry); + if (!err) { + inc_nlink(dir); + goto out_unpin; /* success */ + } + + /* revert */ + if (diropq) { + AuLabel(revert opq); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + rerr = au_diropq_remove(dentry, bindex); + inode_unlock(h_inode); + if (rerr) { + AuIOErr("%pd reverting diropq failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + } + +out_dir: + AuLabel(revert dir); + rerr = vfsub_rmdir(au_pinned_h_dir(&a->pin), &h_path); + if (rerr) { + AuIOErr("%pd reverting dir failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&a->dt); +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); +out_parent: + di_write_unlock(parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} diff --git a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c new file mode 100644 index 0000000000000..f4e2ef3524e96 --- /dev/null +++ b/fs/aufs/i_op_del.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (del entry) + */ + +#include "aufs.h" + +/* + * decide if a new whiteout for @dentry is necessary or not. + * when it is necessary, prepare the parent dir for the upper branch whose + * branch index is @bcpup for creation. the actual creation of the whiteout will + * be done by caller. + * return value: + * 0: wh is unnecessary + * plus: wh is necessary + * minus: error + */ +int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup) +{ + int need_wh, err; + aufs_bindex_t btop; + struct super_block *sb; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + if (*bcpup < 0) { + *bcpup = btop; + if (au_test_ro(sb, btop, d_inode(dentry))) { + err = AuWbrCopyup(au_sbi(sb), dentry); + *bcpup = err; + if (unlikely(err < 0)) + goto out; + } + } else + AuDebugOn(btop < *bcpup + || au_test_ro(sb, *bcpup, d_inode(dentry))); + AuDbg("bcpup %d, btop %d\n", *bcpup, btop); + + if (*bcpup != btop) { + err = au_cpup_dirs(dentry, *bcpup); + if (unlikely(err)) + goto out; + need_wh = 1; + } else { + struct au_dinfo *dinfo, *tmp; + + need_wh = -ENOMEM; + dinfo = au_di(dentry); + tmp = au_di_alloc(sb, AuLsc_DI_TMP); + if (tmp) { + au_di_cp(tmp, dinfo); + au_di_swap(tmp, dinfo); + /* returns the number of positive dentries */ + need_wh = au_lkup_dentry(dentry, btop + 1, + /* AuLkup_IGNORE_PERM */ 0); + au_di_swap(tmp, dinfo); + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + } + } + AuDbg("need_wh %d\n", need_wh); + err = need_wh; + +out: + return err; +} + +/* + * simple tests for the del-entry operations. + * following the checks in vfs, plus the parent-child relationship. + */ +int au_may_del(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir) +{ + int err; + umode_t h_mode; + struct dentry *h_dentry, *h_latest; + struct inode *h_inode; + + h_dentry = au_h_dptr(dentry, bindex); + if (d_really_is_positive(dentry)) { + err = -ENOENT; + if (unlikely(d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + if (unlikely(!h_inode->i_nlink)) + goto out; + + h_mode = h_inode->i_mode; + if (!isdir) { + err = -EISDIR; + if (unlikely(S_ISDIR(h_mode))) + goto out; + } else if (unlikely(!S_ISDIR(h_mode))) { + err = -ENOTDIR; + goto out; + } + } else { + /* rename(2) case */ + err = -EIO; + if (unlikely(d_is_positive(h_dentry))) + goto out; + } + + err = -ENOENT; + /* expected parent dir is locked */ + if (unlikely(h_parent != h_dentry->d_parent)) + goto out; + err = 0; + + /* + * rmdir a dir may break the consistency on some filesystem. + * let's try heavy test. + */ + err = -EACCES; + if (unlikely(!au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1) + && au_test_h_perm(d_inode(h_parent), + MAY_EXEC | MAY_WRITE))) + goto out; + + h_latest = au_sio_lkup_one(&dentry->d_name, h_parent); + err = -EIO; + if (IS_ERR(h_latest)) + goto out; + if (h_latest == h_dentry) + err = 0; + dput(h_latest); + +out: + return err; +} + +/* + * decide the branch where we operate for @dentry. the branch index will be set + * @rbcpup. after deciding it, 'pin' it and store the timestamps of the parent + * dir for reverting. + * when a new whiteout is necessary, create it. + */ +static struct dentry* +lock_hdir_create_wh(struct dentry *dentry, int isdir, aufs_bindex_t *rbcpup, + struct au_dtime *dt, struct au_pin *pin) +{ + struct dentry *wh_dentry; + struct super_block *sb; + struct path h_path; + int err, need_wh; + unsigned int udba; + aufs_bindex_t bcpup; + + need_wh = au_wr_dir_need_wh(dentry, isdir, rbcpup); + wh_dentry = ERR_PTR(need_wh); + if (unlikely(need_wh < 0)) + goto out; + + sb = dentry->d_sb; + udba = au_opt_udba(sb); + bcpup = *rbcpup; + err = au_pin(pin, dentry, bcpup, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out; + + h_path.dentry = au_pinned_h_parent(pin); + if (udba != AuOpt_UDBA_NONE + && au_dbtop(dentry) == bcpup) { + err = au_may_del(dentry, bcpup, h_path.dentry, isdir); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_unpin; + } + + h_path.mnt = au_sbr_mnt(sb, bcpup); + au_dtime_store(dt, au_pinned_parent(pin), &h_path); + wh_dentry = NULL; + if (!need_wh) + goto out; /* success, no need to create whiteout */ + + wh_dentry = au_wh_create(dentry, bcpup, h_path.dentry); + if (IS_ERR(wh_dentry)) + goto out_unpin; + + /* returns with the parent is locked and wh_dentry is dget-ed */ + goto out; /* success */ + +out_unpin: + au_unpin(pin); +out: + return wh_dentry; +} + +/* + * when removing a dir, rename it to a unique temporary whiteout-ed name first + * in order to be revertible and save time for removing many child whiteouts + * under the dir. + * returns 1 when there are too many child whiteout and caller should remove + * them asynchronously. returns 0 when the number of children is enough small to + * remove now or the branch fs is a remote fs. + * otherwise return an error. + */ +static int renwh_and_rmdir(struct dentry *dentry, aufs_bindex_t bindex, + struct au_nhash *whlist, struct inode *dir) +{ + int rmdir_later, err, dirwh; + struct dentry *h_dentry; + struct super_block *sb; + struct inode *inode; + + sb = dentry->d_sb; + SiMustAnyLock(sb); + h_dentry = au_h_dptr(dentry, bindex); + err = au_whtmp_ren(h_dentry, au_sbr(sb, bindex)); + if (unlikely(err)) + goto out; + + /* stop monitoring */ + inode = d_inode(dentry); + au_hn_free(au_hi(inode, bindex)); + + if (!au_test_fs_remote(h_dentry->d_sb)) { + dirwh = au_sbi(sb)->si_dirwh; + rmdir_later = (dirwh <= 1); + if (!rmdir_later) + rmdir_later = au_nhash_test_longer_wh(whlist, bindex, + dirwh); + if (rmdir_later) + return rmdir_later; + } + + err = au_whtmp_rmdir(dir, bindex, h_dentry, whlist); + if (unlikely(err)) { + AuIOErr("rmdir %pd, b%d failed, %d. ignored\n", + h_dentry, bindex, err); + err = 0; + } + +out: + AuTraceErr(err); + return err; +} + +/* + * final procedure for deleting a entry. + * maintain dentry and iattr. + */ +static void epilog(struct inode *dir, struct dentry *dentry, + aufs_bindex_t bindex) +{ + struct inode *inode; + + inode = d_inode(dentry); + d_drop(dentry); + inode->i_ctime = dir->i_ctime; + + au_dir_ts(dir, bindex); + dir->i_version++; +} + +/* + * when an error happened, remove the created whiteout and revert everything. + */ +static int do_revert(int err, struct inode *dir, aufs_bindex_t bindex, + aufs_bindex_t bwh, struct dentry *wh_dentry, + struct dentry *dentry, struct au_dtime *dt) +{ + int rerr; + struct path h_path = { + .dentry = wh_dentry, + .mnt = au_sbr_mnt(dir->i_sb, bindex) + }; + + rerr = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path, dentry); + if (!rerr) { + au_set_dbwh(dentry, bwh); + au_dtime_revert(dt); + return 0; + } + + AuIOErr("%pd reverting whiteout failed(%d, %d)\n", dentry, err, rerr); + return -EIO; +} + +/* ---------------------------------------------------------------------- */ + +int aufs_unlink(struct inode *dir, struct dentry *dentry) +{ + int err; + aufs_bindex_t bwh, bindex, btop; + struct inode *inode, *h_dir, *delegated; + struct dentry *parent, *wh_dentry; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + struct path h_path; + } *a; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_d_hashed_positive(dentry); + if (unlikely(err)) + goto out_unlock; + inode = d_inode(dentry); + IMustLock(inode); + err = -EISDIR; + if (unlikely(d_is_dir(dentry))) + goto out_unlock; /* possible? */ + + btop = au_dbtop(dentry); + bwh = au_dbwh(dentry); + bindex = -1; + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/0, &bindex, &a->dt, + &a->pin); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + a->h_path.mnt = au_sbr_mnt(dentry->d_sb, btop); + a->h_path.dentry = au_h_dptr(dentry, btop); + dget(a->h_path.dentry); + if (bindex == btop) { + h_dir = au_pinned_h_dir(&a->pin); + delegated = NULL; + err = vfsub_unlink(h_dir, &a->h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } else { + /* dir inode is locked */ + h_dir = d_inode(wh_dentry->d_parent); + IMustLock(h_dir); + err = 0; + } + + if (!err) { + vfsub_drop_nlink(inode); + epilog(dir, dentry, bindex); + + /* update target timestamps */ + if (bindex == btop) { + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); + /*ignore*/ + inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + } else + /* todo: this timestamp may be reverted later */ + inode->i_ctime = h_dir->i_ctime; + goto out_unpin; /* success */ + } + + /* revert */ + if (wh_dentry) { + int rerr; + + rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry, + &a->dt); + if (rerr) + err = rerr; + } + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); + dput(a->h_path.dentry); +out_parent: + di_write_unlock(parent); +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} + +int aufs_rmdir(struct inode *dir, struct dentry *dentry) +{ + int err, rmdir_later; + aufs_bindex_t bwh, bindex, btop; + struct inode *inode; + struct dentry *parent, *wh_dentry, *h_dentry; + struct au_whtmp_rmdir *args; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + } *a; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_alive_dir(dentry); + if (unlikely(err)) + goto out_unlock; + inode = d_inode(dentry); + IMustLock(inode); + err = -ENOTDIR; + if (unlikely(!d_is_dir(dentry))) + goto out_unlock; /* possible? */ + + err = -ENOMEM; + args = au_whtmp_rmdir_alloc(dir->i_sb, GFP_NOFS); + if (unlikely(!args)) + goto out_unlock; + + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + err = au_test_empty(dentry, &args->whlist); + if (unlikely(err)) + goto out_parent; + + btop = au_dbtop(dentry); + bwh = au_dbwh(dentry); + bindex = -1; + wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/1, &bindex, &a->dt, + &a->pin); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + h_dentry = au_h_dptr(dentry, btop); + dget(h_dentry); + rmdir_later = 0; + if (bindex == btop) { + err = renwh_and_rmdir(dentry, btop, &args->whlist, dir); + if (err > 0) { + rmdir_later = err; + err = 0; + } + } else { + /* stop monitoring */ + au_hn_free(au_hi(inode, btop)); + + /* dir inode is locked */ + IMustLock(d_inode(wh_dentry->d_parent)); + err = 0; + } + + if (!err) { + vfsub_dead_dir(inode); + au_set_dbdiropq(dentry, -1); + epilog(dir, dentry, bindex); + + if (rmdir_later) { + au_whtmp_kick_rmdir(dir, btop, h_dentry, args); + args = NULL; + } + + goto out_unpin; /* success */ + } + + /* revert */ + AuLabel(revert); + if (wh_dentry) { + int rerr; + + rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry, + &a->dt); + if (rerr) + err = rerr; + } + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); + dput(h_dentry); +out_parent: + di_write_unlock(parent); + if (args) + au_whtmp_rmdir_free(args); +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c new file mode 100644 index 0000000000000..8524b08523b89 --- /dev/null +++ b/fs/aufs/i_op_ren.c @@ -0,0 +1,1249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operation (rename entry) + * todo: this is crazy monster + */ + +#include "aufs.h" + +enum { AuSRC, AuDST, AuSrcDst }; +enum { AuPARENT, AuCHILD, AuParentChild }; + +#define AuRen_ISDIR_SRC 1 +#define AuRen_ISDIR_DST (1 << 1) +#define AuRen_ISSAMEDIR (1 << 2) +#define AuRen_WHSRC (1 << 3) +#define AuRen_WHDST (1 << 4) +#define AuRen_MNT_WRITE (1 << 5) +#define AuRen_DT_DSTDIR (1 << 6) +#define AuRen_DIROPQ_SRC (1 << 7) +#define AuRen_DIROPQ_DST (1 << 8) +#define AuRen_DIRREN (1 << 9) +#define AuRen_DROPPED_SRC (1 << 10) +#define AuRen_DROPPED_DST (1 << 11) +#define au_ftest_ren(flags, name) ((flags) & AuRen_##name) +#define au_fset_ren(flags, name) \ + do { (flags) |= AuRen_##name; } while (0) +#define au_fclr_ren(flags, name) \ + do { (flags) &= ~AuRen_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuRen_DIRREN +#define AuRen_DIRREN 0 +#endif + +struct au_ren_args { + struct { + struct dentry *dentry, *h_dentry, *parent, *h_parent, + *wh_dentry; + struct inode *dir, *inode; + struct au_hinode *hdir, *hinode; + struct au_dtime dt[AuParentChild]; + aufs_bindex_t btop, bdiropq; + } sd[AuSrcDst]; + +#define src_dentry sd[AuSRC].dentry +#define src_dir sd[AuSRC].dir +#define src_inode sd[AuSRC].inode +#define src_h_dentry sd[AuSRC].h_dentry +#define src_parent sd[AuSRC].parent +#define src_h_parent sd[AuSRC].h_parent +#define src_wh_dentry sd[AuSRC].wh_dentry +#define src_hdir sd[AuSRC].hdir +#define src_hinode sd[AuSRC].hinode +#define src_h_dir sd[AuSRC].hdir->hi_inode +#define src_dt sd[AuSRC].dt +#define src_btop sd[AuSRC].btop +#define src_bdiropq sd[AuSRC].bdiropq + +#define dst_dentry sd[AuDST].dentry +#define dst_dir sd[AuDST].dir +#define dst_inode sd[AuDST].inode +#define dst_h_dentry sd[AuDST].h_dentry +#define dst_parent sd[AuDST].parent +#define dst_h_parent sd[AuDST].h_parent +#define dst_wh_dentry sd[AuDST].wh_dentry +#define dst_hdir sd[AuDST].hdir +#define dst_hinode sd[AuDST].hinode +#define dst_h_dir sd[AuDST].hdir->hi_inode +#define dst_dt sd[AuDST].dt +#define dst_btop sd[AuDST].btop +#define dst_bdiropq sd[AuDST].bdiropq + + struct dentry *h_trap; + struct au_branch *br; + struct path h_path; + struct au_nhash whlist; + aufs_bindex_t btgt, src_bwh; + + struct { + unsigned short auren_flags; + unsigned char flags; /* syscall parameter */ + unsigned char exchange; + } __packed; + + struct au_whtmp_rmdir *thargs; + struct dentry *h_dst; + struct au_hinode *h_root; +}; + +/* ---------------------------------------------------------------------- */ + +/* + * functions for reverting. + * when an error happened in a single rename systemcall, we should revert + * everything as if nothing happened. + * we don't need to revert the copied-up/down the parent dir since they are + * harmless. + */ + +#define RevertFailure(fmt, ...) do { \ + AuIOErr("revert failure: " fmt " (%d, %d)\n", \ + ##__VA_ARGS__, err, rerr); \ + err = -EIO; \ +} while (0) + +static void au_ren_do_rev_diropq(int err, struct au_ren_args *a, int idx) +{ + int rerr; + struct dentry *d; +#define src_or_dst(member) a->sd[idx].member + + d = src_or_dst(dentry); /* {src,dst}_dentry */ + au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD); + rerr = au_diropq_remove(d, a->btgt); + au_hn_inode_unlock(src_or_dst(hinode)); + au_set_dbdiropq(d, src_or_dst(bdiropq)); + if (rerr) + RevertFailure("remove diropq %pd", d); + +#undef src_or_dst_ +} + +static void au_ren_rev_diropq(int err, struct au_ren_args *a) +{ + if (au_ftest_ren(a->auren_flags, DIROPQ_SRC)) + au_ren_do_rev_diropq(err, a, AuSRC); + if (au_ftest_ren(a->auren_flags, DIROPQ_DST)) + au_ren_do_rev_diropq(err, a, AuDST); +} + +static void au_ren_rev_rename(int err, struct au_ren_args *a) +{ + int rerr; + struct inode *delegated; + + a->h_path.dentry = vfsub_lkup_one(&a->src_dentry->d_name, + a->src_h_parent); + rerr = PTR_ERR(a->h_path.dentry); + if (IS_ERR(a->h_path.dentry)) { + RevertFailure("lkup one %pd", a->src_dentry); + return; + } + + delegated = NULL; + rerr = vfsub_rename(a->dst_h_dir, + au_h_dptr(a->src_dentry, a->btgt), + a->src_h_dir, &a->h_path, &delegated, a->flags); + if (unlikely(rerr == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + /* au_set_h_dptr(a->src_dentry, a->btgt, NULL); */ + if (rerr) + RevertFailure("rename %pd", a->src_dentry); +} + +static void au_ren_rev_whtmp(int err, struct au_ren_args *a) +{ + int rerr; + struct inode *delegated; + + a->h_path.dentry = vfsub_lkup_one(&a->dst_dentry->d_name, + a->dst_h_parent); + rerr = PTR_ERR(a->h_path.dentry); + if (IS_ERR(a->h_path.dentry)) { + RevertFailure("lkup one %pd", a->dst_dentry); + return; + } + if (d_is_positive(a->h_path.dentry)) { + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + return; + } + + delegated = NULL; + rerr = vfsub_rename(a->dst_h_dir, a->h_dst, a->dst_h_dir, &a->h_path, + &delegated, a->flags); + if (unlikely(rerr == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + if (!rerr) + au_set_h_dptr(a->dst_dentry, a->btgt, dget(a->h_dst)); + else + RevertFailure("rename %pd", a->h_dst); +} + +static void au_ren_rev_whsrc(int err, struct au_ren_args *a) +{ + int rerr; + + a->h_path.dentry = a->src_wh_dentry; + rerr = au_wh_unlink_dentry(a->src_h_dir, &a->h_path, a->src_dentry); + au_set_dbwh(a->src_dentry, a->src_bwh); + if (rerr) + RevertFailure("unlink %pd", a->src_wh_dentry); +} +#undef RevertFailure + +/* ---------------------------------------------------------------------- */ + +/* + * when we have to copyup the renaming entry, do it with the rename-target name + * in order to minimize the cost (the later actual rename is unnecessary). + * otherwise rename it on the target branch. + */ +static int au_ren_or_cpup(struct au_ren_args *a) +{ + int err; + struct dentry *d; + struct inode *delegated; + + d = a->src_dentry; + if (au_dbtop(d) == a->btgt) { + a->h_path.dentry = a->dst_h_dentry; + AuDebugOn(au_dbtop(d) != a->btgt); + delegated = NULL; + err = vfsub_rename(a->src_h_dir, au_h_dptr(d, a->btgt), + a->dst_h_dir, &a->h_path, &delegated, + a->flags); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + } else + BUG(); + + if (!err && a->h_dst) + /* it will be set to dinfo later */ + dget(a->h_dst); + + return err; +} + +/* cf. aufs_rmdir() */ +static int au_ren_del_whtmp(struct au_ren_args *a) +{ + int err; + struct inode *dir; + + dir = a->dst_dir; + SiMustAnyLock(dir->i_sb); + if (!au_nhash_test_longer_wh(&a->whlist, a->btgt, + au_sbi(dir->i_sb)->si_dirwh) + || au_test_fs_remote(a->h_dst->d_sb)) { + err = au_whtmp_rmdir(dir, a->btgt, a->h_dst, &a->whlist); + if (unlikely(err)) + pr_warn("failed removing whtmp dir %pd (%d), " + "ignored.\n", a->h_dst, err); + } else { + au_nhash_wh_free(&a->thargs->whlist); + a->thargs->whlist = a->whlist; + a->whlist.nh_num = 0; + au_whtmp_kick_rmdir(dir, a->btgt, a->h_dst, a->thargs); + dput(a->h_dst); + a->thargs = NULL; + } + + return 0; +} + +/* make it 'opaque' dir. */ +static int au_ren_do_diropq(struct au_ren_args *a, int idx) +{ + int err; + struct dentry *d, *diropq; +#define src_or_dst(member) a->sd[idx].member + + err = 0; + d = src_or_dst(dentry); /* {src,dst}_dentry */ + src_or_dst(bdiropq) = au_dbdiropq(d); + src_or_dst(hinode) = au_hi(src_or_dst(inode), a->btgt); + au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD); + diropq = au_diropq_create(d, a->btgt); + au_hn_inode_unlock(src_or_dst(hinode)); + if (IS_ERR(diropq)) + err = PTR_ERR(diropq); + else + dput(diropq); + +#undef src_or_dst_ + return err; +} + +static int au_ren_diropq(struct au_ren_args *a) +{ + int err; + unsigned char always; + struct dentry *d; + + err = 0; + d = a->dst_dentry; /* already renamed on the branch */ + always = !!au_opt_test(au_mntflags(d->d_sb), ALWAYS_DIROPQ); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) + && !au_ftest_ren(a->auren_flags, DIRREN) + && a->btgt != au_dbdiropq(a->src_dentry) + && (a->dst_wh_dentry + || a->btgt <= au_dbdiropq(d) + /* hide the lower to keep xino */ + /* the lowers may not be a dir, but we hide them anyway */ + || a->btgt < au_dbbot(d) + || always)) { + AuDbg("here\n"); + err = au_ren_do_diropq(a, AuSRC); + if (unlikely(err)) + goto out; + au_fset_ren(a->auren_flags, DIROPQ_SRC); + } + if (!a->exchange) + goto out; /* success */ + + d = a->src_dentry; /* already renamed on the branch */ + if (au_ftest_ren(a->auren_flags, ISDIR_DST) + && a->btgt != au_dbdiropq(a->dst_dentry) + && (a->btgt < au_dbdiropq(d) + || a->btgt < au_dbbot(d) + || always)) { + AuDbgDentry(a->src_dentry); + AuDbgDentry(a->dst_dentry); + err = au_ren_do_diropq(a, AuDST); + if (unlikely(err)) + goto out_rev_src; + au_fset_ren(a->auren_flags, DIROPQ_DST); + } + goto out; /* success */ + +out_rev_src: + AuDbg("err %d, reverting src\n", err); + au_ren_rev_diropq(err, a); +out: + return err; +} + +static int do_rename(struct au_ren_args *a) +{ + int err; + struct dentry *d, *h_d; + + if (!a->exchange) { + /* prepare workqueue args for asynchronous rmdir */ + h_d = a->dst_h_dentry; + if (au_ftest_ren(a->auren_flags, ISDIR_DST) + /* && !au_ftest_ren(a->auren_flags, DIRREN) */ + && d_is_positive(h_d)) { + err = -ENOMEM; + a->thargs = au_whtmp_rmdir_alloc(a->src_dentry->d_sb, + GFP_NOFS); + if (unlikely(!a->thargs)) + goto out; + a->h_dst = dget(h_d); + } + + /* create whiteout for src_dentry */ + if (au_ftest_ren(a->auren_flags, WHSRC)) { + a->src_bwh = au_dbwh(a->src_dentry); + AuDebugOn(a->src_bwh >= 0); + a->src_wh_dentry = au_wh_create(a->src_dentry, a->btgt, + a->src_h_parent); + err = PTR_ERR(a->src_wh_dentry); + if (IS_ERR(a->src_wh_dentry)) + goto out_thargs; + } + + /* lookup whiteout for dentry */ + if (au_ftest_ren(a->auren_flags, WHDST)) { + h_d = au_wh_lkup(a->dst_h_parent, + &a->dst_dentry->d_name, a->br); + err = PTR_ERR(h_d); + if (IS_ERR(h_d)) + goto out_whsrc; + if (d_is_negative(h_d)) + dput(h_d); + else + a->dst_wh_dentry = h_d; + } + + /* rename dentry to tmpwh */ + if (a->thargs) { + err = au_whtmp_ren(a->dst_h_dentry, a->br); + if (unlikely(err)) + goto out_whdst; + + d = a->dst_dentry; + au_set_h_dptr(d, a->btgt, NULL); + err = au_lkup_neg(d, a->btgt, /*wh*/0); + if (unlikely(err)) + goto out_whtmp; + a->dst_h_dentry = au_h_dptr(d, a->btgt); + } + } + + BUG_ON(d_is_positive(a->dst_h_dentry) && a->src_btop != a->btgt); +#if 0 + BUG_ON(!au_ftest_ren(a->auren_flags, DIRREN) + && d_is_positive(a->dst_h_dentry) + && a->src_btop != a->btgt); +#endif + + /* rename by vfs_rename or cpup */ + err = au_ren_or_cpup(a); + if (unlikely(err)) + /* leave the copied-up one */ + goto out_whtmp; + + /* make dir opaque */ + err = au_ren_diropq(a); + if (unlikely(err)) + goto out_rename; + + /* update target timestamps */ + if (a->exchange) { + AuDebugOn(au_dbtop(a->dst_dentry) != a->btgt); + a->h_path.dentry = au_h_dptr(a->dst_dentry, a->btgt); + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/ + a->dst_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + } + AuDebugOn(au_dbtop(a->src_dentry) != a->btgt); + a->h_path.dentry = au_h_dptr(a->src_dentry, a->btgt); + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/ + a->src_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + + if (!a->exchange) { + /* remove whiteout for dentry */ + if (a->dst_wh_dentry) { + a->h_path.dentry = a->dst_wh_dentry; + err = au_wh_unlink_dentry(a->dst_h_dir, &a->h_path, + a->dst_dentry); + if (unlikely(err)) + goto out_diropq; + } + + /* remove whtmp */ + if (a->thargs) + au_ren_del_whtmp(a); /* ignore this error */ + + au_fhsm_wrote(a->src_dentry->d_sb, a->btgt, /*force*/0); + } + err = 0; + goto out_success; + +out_diropq: + au_ren_rev_diropq(err, a); +out_rename: + au_ren_rev_rename(err, a); + dput(a->h_dst); +out_whtmp: + if (a->thargs) + au_ren_rev_whtmp(err, a); +out_whdst: + dput(a->dst_wh_dentry); + a->dst_wh_dentry = NULL; +out_whsrc: + if (a->src_wh_dentry) + au_ren_rev_whsrc(err, a); +out_success: + dput(a->src_wh_dentry); + dput(a->dst_wh_dentry); +out_thargs: + if (a->thargs) { + dput(a->h_dst); + au_whtmp_rmdir_free(a->thargs); + a->thargs = NULL; + } +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * test if @dentry dir can be rename destination or not. + * success means, it is a logically empty dir. + */ +static int may_rename_dstdir(struct dentry *dentry, struct au_nhash *whlist) +{ + return au_test_empty(dentry, whlist); +} + +/* + * test if @a->src_dentry dir can be rename source or not. + * if it can, return 0. + * success means, + * - it is a logically empty dir. + * - or, it exists on writable branch and has no children including whiteouts + * on the lower branch unless DIRREN is on. + */ +static int may_rename_srcdir(struct au_ren_args *a) +{ + int err; + unsigned int rdhash; + aufs_bindex_t btop, btgt; + struct dentry *dentry; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + dentry = a->src_dentry; + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) + au_fset_ren(a->auren_flags, DIRREN); + + btgt = a->btgt; + btop = au_dbtop(dentry); + if (btop != btgt) { + struct au_nhash whlist; + + SiMustAnyLock(sb); + rdhash = sbinfo->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, + dentry)); + err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_test_empty(dentry, &whlist); + au_nhash_wh_free(&whlist); + goto out; + } + + if (btop == au_dbtaildir(dentry)) + return 0; /* success */ + + err = au_test_empty_lower(dentry); + +out: + if (err == -ENOTEMPTY) { + if (au_ftest_ren(a->auren_flags, DIRREN)) { + err = 0; + } else { + AuWarn1("renaming dir who has child(ren) on multiple " + "branches, is not supported\n"); + err = -EXDEV; + } + } + return err; +} + +/* side effect: sets whlist and h_dentry */ +static int au_ren_may_dir(struct au_ren_args *a) +{ + int err; + unsigned int rdhash; + struct dentry *d; + + d = a->dst_dentry; + SiMustAnyLock(d->d_sb); + + err = 0; + if (au_ftest_ren(a->auren_flags, ISDIR_DST) && a->dst_inode) { + rdhash = au_sbi(d->d_sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, d)); + err = au_nhash_alloc(&a->whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + + if (!a->exchange) { + au_set_dbtop(d, a->dst_btop); + err = may_rename_dstdir(d, &a->whlist); + au_set_dbtop(d, a->btgt); + } else + err = may_rename_srcdir(a); + } + a->dst_h_dentry = au_h_dptr(d, au_dbtop(d)); + if (unlikely(err)) + goto out; + + d = a->src_dentry; + a->src_h_dentry = au_h_dptr(d, au_dbtop(d)); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) { + err = may_rename_srcdir(a); + if (unlikely(err)) { + au_nhash_wh_free(&a->whlist); + a->whlist.nh_num = 0; + } + } +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * simple tests for rename. + * following the checks in vfs, plus the parent-child relationship. + */ +static int au_may_ren(struct au_ren_args *a) +{ + int err, isdir; + struct inode *h_inode; + + if (a->src_btop == a->btgt) { + err = au_may_del(a->src_dentry, a->btgt, a->src_h_parent, + au_ftest_ren(a->auren_flags, ISDIR_SRC)); + if (unlikely(err)) + goto out; + err = -EINVAL; + if (unlikely(a->src_h_dentry == a->h_trap)) + goto out; + } + + err = 0; + if (a->dst_btop != a->btgt) + goto out; + + err = -ENOTEMPTY; + if (unlikely(a->dst_h_dentry == a->h_trap)) + goto out; + + err = -EIO; + isdir = !!au_ftest_ren(a->auren_flags, ISDIR_DST); + if (d_really_is_negative(a->dst_dentry)) { + if (d_is_negative(a->dst_h_dentry)) + err = au_may_add(a->dst_dentry, a->btgt, + a->dst_h_parent, isdir); + } else { + if (unlikely(d_is_negative(a->dst_h_dentry))) + goto out; + h_inode = d_inode(a->dst_h_dentry); + if (h_inode->i_nlink) + err = au_may_del(a->dst_dentry, a->btgt, + a->dst_h_parent, isdir); + } + +out: + if (unlikely(err == -ENOENT || err == -EEXIST)) + err = -EIO; + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * locking order + * (VFS) + * - src_dir and dir by lock_rename() + * - inode if exists + * (aufs) + * - lock all + * + src_dentry and dentry by aufs_read_and_write_lock2() which calls, + * + si_read_lock + * + di_write_lock2_child() + * + di_write_lock_child() + * + ii_write_lock_child() + * + di_write_lock_child2() + * + ii_write_lock_child2() + * + src_parent and parent + * + di_write_lock_parent() + * + ii_write_lock_parent() + * + di_write_lock_parent2() + * + ii_write_lock_parent2() + * + lower src_dir and dir by vfsub_lock_rename() + * + verify the every relationships between child and parent. if any + * of them failed, unlock all and return -EBUSY. + */ +static void au_ren_unlock(struct au_ren_args *a) +{ + vfsub_unlock_rename(a->src_h_parent, a->src_hdir, + a->dst_h_parent, a->dst_hdir); + if (au_ftest_ren(a->auren_flags, DIRREN) + && a->h_root) + au_hn_inode_unlock(a->h_root); + if (au_ftest_ren(a->auren_flags, MNT_WRITE)) + vfsub_mnt_drop_write(au_br_mnt(a->br)); +} + +static int au_ren_lock(struct au_ren_args *a) +{ + int err; + unsigned int udba; + + err = 0; + a->src_h_parent = au_h_dptr(a->src_parent, a->btgt); + a->src_hdir = au_hi(a->src_dir, a->btgt); + a->dst_h_parent = au_h_dptr(a->dst_parent, a->btgt); + a->dst_hdir = au_hi(a->dst_dir, a->btgt); + + err = vfsub_mnt_want_write(au_br_mnt(a->br)); + if (unlikely(err)) + goto out; + au_fset_ren(a->auren_flags, MNT_WRITE); + if (au_ftest_ren(a->auren_flags, DIRREN)) { + struct dentry *root; + struct inode *dir; + + /* + * sbinfo is already locked, so this ii_read_lock is + * unnecessary. but our debugging feature checks it. + */ + root = a->src_inode->i_sb->s_root; + if (root != a->src_parent && root != a->dst_parent) { + dir = d_inode(root); + ii_read_lock_parent3(dir); + a->h_root = au_hi(dir, a->btgt); + ii_read_unlock(dir); + au_hn_inode_lock_nested(a->h_root, AuLsc_I_PARENT3); + } + } + a->h_trap = vfsub_lock_rename(a->src_h_parent, a->src_hdir, + a->dst_h_parent, a->dst_hdir); + udba = au_opt_udba(a->src_dentry->d_sb); + if (unlikely(a->src_hdir->hi_inode != d_inode(a->src_h_parent) + || a->dst_hdir->hi_inode != d_inode(a->dst_h_parent))) + err = au_busy_or_stale(); + if (!err && au_dbtop(a->src_dentry) == a->btgt) + err = au_h_verify(a->src_h_dentry, udba, + d_inode(a->src_h_parent), a->src_h_parent, + a->br); + if (!err && au_dbtop(a->dst_dentry) == a->btgt) + err = au_h_verify(a->dst_h_dentry, udba, + d_inode(a->dst_h_parent), a->dst_h_parent, + a->br); + if (!err) + goto out; /* success */ + + err = au_busy_or_stale(); + au_ren_unlock(a); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_ren_refresh_dir(struct au_ren_args *a) +{ + struct inode *dir; + + dir = a->dst_dir; + dir->i_version++; + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) { + /* is this updating defined in POSIX? */ + au_cpup_attr_timesizes(a->src_inode); + au_cpup_attr_nlink(dir, /*force*/1); + } + au_dir_ts(dir, a->btgt); + + if (a->exchange) { + dir = a->src_dir; + dir->i_version++; + if (au_ftest_ren(a->auren_flags, ISDIR_DST)) { + /* is this updating defined in POSIX? */ + au_cpup_attr_timesizes(a->dst_inode); + au_cpup_attr_nlink(dir, /*force*/1); + } + au_dir_ts(dir, a->btgt); + } + + if (au_ftest_ren(a->auren_flags, ISSAMEDIR)) + return; + + dir = a->src_dir; + dir->i_version++; + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) + au_cpup_attr_nlink(dir, /*force*/1); + au_dir_ts(dir, a->btgt); +} + +static void au_ren_refresh(struct au_ren_args *a) +{ + aufs_bindex_t bbot, bindex; + struct dentry *d, *h_d; + struct inode *i, *h_i; + struct super_block *sb; + + d = a->dst_dentry; + d_drop(d); + if (a->h_dst) + /* already dget-ed by au_ren_or_cpup() */ + au_set_h_dptr(d, a->btgt, a->h_dst); + + i = a->dst_inode; + if (i) { + if (!a->exchange) { + if (!au_ftest_ren(a->auren_flags, ISDIR_DST)) + vfsub_drop_nlink(i); + else { + vfsub_dead_dir(i); + au_cpup_attr_timesizes(i); + } + au_update_dbrange(d, /*do_put_zero*/1); + } else + au_cpup_attr_nlink(i, /*force*/1); + } else { + bbot = a->btgt; + for (bindex = au_dbtop(d); bindex < bbot; bindex++) + au_set_h_dptr(d, bindex, NULL); + bbot = au_dbbot(d); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) + au_set_h_dptr(d, bindex, NULL); + au_update_dbrange(d, /*do_put_zero*/0); + } + + if (a->exchange + || au_ftest_ren(a->auren_flags, DIRREN)) { + d_drop(a->src_dentry); + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_set_dbwh(a->src_dentry, -1); + return; + } + + d = a->src_dentry; + au_set_dbwh(d, -1); + bbot = au_dbbot(d); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) { + h_d = au_h_dptr(d, bindex); + if (h_d) + au_set_h_dptr(d, bindex, NULL); + } + au_set_dbbot(d, a->btgt); + + sb = d->d_sb; + i = a->src_inode; + if (au_opt_test(au_mntflags(sb), PLINK) && au_plink_test(i)) + return; /* success */ + + bbot = au_ibbot(i); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) { + h_i = au_h_iptr(i, bindex); + if (h_i) { + au_xino_write(sb, bindex, h_i->i_ino, /*ino*/0); + /* ignore this error */ + au_set_h_iptr(i, bindex, NULL, 0); + } + } + au_set_ibbot(i, a->btgt); +} + +/* ---------------------------------------------------------------------- */ + +/* mainly for link(2) and rename(2) */ +int au_wbr(struct dentry *dentry, aufs_bindex_t btgt) +{ + aufs_bindex_t bdiropq, bwh; + struct dentry *parent; + struct au_branch *br; + + parent = dentry->d_parent; + IMustLock(d_inode(parent)); /* dir is locked */ + + bdiropq = au_dbdiropq(parent); + bwh = au_dbwh(dentry); + br = au_sbr(dentry->d_sb, btgt); + if (au_br_rdonly(br) + || (0 <= bdiropq && bdiropq < btgt) + || (0 <= bwh && bwh < btgt)) + btgt = -1; + + AuDbg("btgt %d\n", btgt); + return btgt; +} + +/* sets src_btop, dst_btop and btgt */ +static int au_ren_wbr(struct au_ren_args *a) +{ + int err; + struct au_wr_dir_args wr_dir_args = { + /* .force_btgt = -1, */ + .flags = AuWrDir_ADD_ENTRY + }; + + a->src_btop = au_dbtop(a->src_dentry); + a->dst_btop = au_dbtop(a->dst_dentry); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) + || au_ftest_ren(a->auren_flags, ISDIR_DST)) + au_fset_wrdir(wr_dir_args.flags, ISDIR); + wr_dir_args.force_btgt = a->src_btop; + if (a->dst_inode && a->dst_btop < a->src_btop) + wr_dir_args.force_btgt = a->dst_btop; + wr_dir_args.force_btgt = au_wbr(a->dst_dentry, wr_dir_args.force_btgt); + err = au_wr_dir(a->dst_dentry, a->src_dentry, &wr_dir_args); + a->btgt = err; + if (a->exchange) + au_update_dbtop(a->dst_dentry); + + return err; +} + +static void au_ren_dt(struct au_ren_args *a) +{ + a->h_path.dentry = a->src_h_parent; + au_dtime_store(a->src_dt + AuPARENT, a->src_parent, &a->h_path); + if (!au_ftest_ren(a->auren_flags, ISSAMEDIR)) { + a->h_path.dentry = a->dst_h_parent; + au_dtime_store(a->dst_dt + AuPARENT, a->dst_parent, &a->h_path); + } + + au_fclr_ren(a->auren_flags, DT_DSTDIR); + if (!au_ftest_ren(a->auren_flags, ISDIR_SRC) + && !a->exchange) + return; + + a->h_path.dentry = a->src_h_dentry; + au_dtime_store(a->src_dt + AuCHILD, a->src_dentry, &a->h_path); + if (d_is_positive(a->dst_h_dentry)) { + au_fset_ren(a->auren_flags, DT_DSTDIR); + a->h_path.dentry = a->dst_h_dentry; + au_dtime_store(a->dst_dt + AuCHILD, a->dst_dentry, &a->h_path); + } +} + +static void au_ren_rev_dt(int err, struct au_ren_args *a) +{ + struct dentry *h_d; + struct inode *h_inode; + + au_dtime_revert(a->src_dt + AuPARENT); + if (!au_ftest_ren(a->auren_flags, ISSAMEDIR)) + au_dtime_revert(a->dst_dt + AuPARENT); + + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) && err != -EIO) { + h_d = a->src_dt[AuCHILD].dt_h_path.dentry; + h_inode = d_inode(h_d); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + au_dtime_revert(a->src_dt + AuCHILD); + inode_unlock(h_inode); + + if (au_ftest_ren(a->auren_flags, DT_DSTDIR)) { + h_d = a->dst_dt[AuCHILD].dt_h_path.dentry; + h_inode = d_inode(h_d); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + au_dtime_revert(a->dst_dt + AuCHILD); + inode_unlock(h_inode); + } + } +} + +/* ---------------------------------------------------------------------- */ + +int aufs_rename(struct inode *_src_dir, struct dentry *_src_dentry, + struct inode *_dst_dir, struct dentry *_dst_dentry, + unsigned int _flags) +{ + int err, lock_flags; + void *rev; + /* reduce stack space */ + struct au_ren_args *a; + struct au_pin pin; + + AuDbg("%pd, %pd, 0x%x\n", _src_dentry, _dst_dentry, _flags); + IMustLock(_src_dir); + IMustLock(_dst_dir); + + err = -EINVAL; + if (unlikely(_flags & RENAME_WHITEOUT)) + goto out; + + err = -ENOMEM; + BUILD_BUG_ON(sizeof(*a) > PAGE_SIZE); + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + a->flags = _flags; + BUILD_BUG_ON(sizeof(a->exchange) == sizeof(u8) + && RENAME_EXCHANGE > U8_MAX); + a->exchange = _flags & RENAME_EXCHANGE; + a->src_dir = _src_dir; + a->src_dentry = _src_dentry; + a->src_inode = NULL; + if (d_really_is_positive(a->src_dentry)) + a->src_inode = d_inode(a->src_dentry); + a->src_parent = a->src_dentry->d_parent; /* dir inode is locked */ + a->dst_dir = _dst_dir; + a->dst_dentry = _dst_dentry; + a->dst_inode = NULL; + if (d_really_is_positive(a->dst_dentry)) + a->dst_inode = d_inode(a->dst_dentry); + a->dst_parent = a->dst_dentry->d_parent; /* dir inode is locked */ + if (a->dst_inode) { + /* + * if EXCHANGE && src is non-dir && dst is dir, + * dst is not locked. + */ + /* IMustLock(a->dst_inode); */ + au_igrab(a->dst_inode); + } + + err = -ENOTDIR; + lock_flags = AuLock_FLUSH | AuLock_NOPLM | AuLock_GEN; + if (d_is_dir(a->src_dentry)) { + au_fset_ren(a->auren_flags, ISDIR_SRC); + if (unlikely(!a->exchange + && d_really_is_positive(a->dst_dentry) + && !d_is_dir(a->dst_dentry))) + goto out_free; + lock_flags |= AuLock_DIRS; + } + if (a->dst_inode && d_is_dir(a->dst_dentry)) { + au_fset_ren(a->auren_flags, ISDIR_DST); + if (unlikely(!a->exchange + && d_really_is_positive(a->src_dentry) + && !d_is_dir(a->src_dentry))) + goto out_free; + lock_flags |= AuLock_DIRS; + } + err = aufs_read_and_write_lock2(a->dst_dentry, a->src_dentry, + lock_flags); + if (unlikely(err)) + goto out_free; + + err = au_d_hashed_positive(a->src_dentry); + if (unlikely(err)) + goto out_unlock; + err = -ENOENT; + if (a->dst_inode) { + /* + * If it is a dir, VFS unhash it before this + * function. It means we cannot rely upon d_unhashed(). + */ + if (unlikely(!a->dst_inode->i_nlink)) + goto out_unlock; + if (!au_ftest_ren(a->auren_flags, ISDIR_DST)) { + err = au_d_hashed_positive(a->dst_dentry); + if (unlikely(err && !a->exchange)) + goto out_unlock; + } else if (unlikely(IS_DEADDIR(a->dst_inode))) + goto out_unlock; + } else if (unlikely(d_unhashed(a->dst_dentry))) + goto out_unlock; + + /* + * is it possible? + * yes, it happened (in linux-3.3-rcN) but I don't know why. + * there may exist a problem somewhere else. + */ + err = -EINVAL; + if (unlikely(d_inode(a->dst_parent) == d_inode(a->src_dentry))) + goto out_unlock; + + au_fset_ren(a->auren_flags, ISSAMEDIR); /* temporary */ + di_write_lock_parent(a->dst_parent); + + /* which branch we process */ + err = au_ren_wbr(a); + if (unlikely(err < 0)) + goto out_parent; + a->br = au_sbr(a->dst_dentry->d_sb, a->btgt); + a->h_path.mnt = au_br_mnt(a->br); + + /* are they available to be renamed */ + err = au_ren_may_dir(a); + if (unlikely(err)) + goto out_children; + + /* prepare the writable parent dir on the same branch */ + if (a->dst_btop == a->btgt) { + au_fset_ren(a->auren_flags, WHDST); + } else { + err = au_cpup_dirs(a->dst_dentry, a->btgt); + if (unlikely(err)) + goto out_children; + } + + err = 0; + if (!a->exchange) { + if (a->src_dir != a->dst_dir) { + /* + * this temporary unlock is safe, + * because both dir->i_mutex are locked. + */ + di_write_unlock(a->dst_parent); + di_write_lock_parent(a->src_parent); + err = au_wr_dir_need_wh(a->src_dentry, + au_ftest_ren(a->auren_flags, + ISDIR_SRC), + &a->btgt); + di_write_unlock(a->src_parent); + di_write_lock2_parent(a->src_parent, a->dst_parent, + /*isdir*/1); + au_fclr_ren(a->auren_flags, ISSAMEDIR); + } else + err = au_wr_dir_need_wh(a->src_dentry, + au_ftest_ren(a->auren_flags, + ISDIR_SRC), + &a->btgt); + } + if (unlikely(err < 0)) + goto out_children; + if (err) + au_fset_ren(a->auren_flags, WHSRC); + + /* cpup src */ + if (a->src_btop != a->btgt) { + err = au_pin(&pin, a->src_dentry, a->btgt, + au_opt_udba(a->src_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + struct au_cp_generic cpg = { + .dentry = a->src_dentry, + .bdst = a->btgt, + .bsrc = a->src_btop, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + AuDebugOn(au_dbtop(a->src_dentry) != a->src_btop); + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + if (unlikely(err)) + goto out_children; + a->src_btop = a->btgt; + a->src_h_dentry = au_h_dptr(a->src_dentry, a->btgt); + if (!a->exchange) + au_fset_ren(a->auren_flags, WHSRC); + } + + /* cpup dst */ + if (a->exchange && a->dst_inode + && a->dst_btop != a->btgt) { + err = au_pin(&pin, a->dst_dentry, a->btgt, + au_opt_udba(a->dst_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + struct au_cp_generic cpg = { + .dentry = a->dst_dentry, + .bdst = a->btgt, + .bsrc = a->dst_btop, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + if (unlikely(err)) + goto out_children; + a->dst_btop = a->btgt; + a->dst_h_dentry = au_h_dptr(a->dst_dentry, a->btgt); + } + + /* lock them all */ + err = au_ren_lock(a); + if (unlikely(err)) + /* leave the copied-up one */ + goto out_children; + + if (!a->exchange) { + if (!au_opt_test(au_mntflags(a->dst_dir->i_sb), UDBA_NONE)) + err = au_may_ren(a); + else if (unlikely(a->dst_dentry->d_name.len > AUFS_MAX_NAMELEN)) + err = -ENAMETOOLONG; + if (unlikely(err)) + goto out_hdir; + } + + /* store timestamps to be revertible */ + au_ren_dt(a); + + /* store dirren info */ + if (au_ftest_ren(a->auren_flags, DIRREN)) { + err = au_dr_rename(a->src_dentry, a->btgt, + &a->dst_dentry->d_name, &rev); + AuTraceErr(err); + if (unlikely(err)) + goto out_dt; + } + + /* here we go */ + err = do_rename(a); + if (unlikely(err)) + goto out_dirren; + + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_dr_rename_fin(a->src_dentry, a->btgt, rev); + + /* update dir attributes */ + au_ren_refresh_dir(a); + + /* dput/iput all lower dentries */ + au_ren_refresh(a); + + goto out_hdir; /* success */ + +out_dirren: + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_dr_rename_rev(a->src_dentry, a->btgt, rev); +out_dt: + au_ren_rev_dt(err, a); +out_hdir: + au_ren_unlock(a); +out_children: + au_nhash_wh_free(&a->whlist); + if (err && a->dst_inode && a->dst_btop != a->btgt) { + AuDbg("btop %d, btgt %d\n", a->dst_btop, a->btgt); + au_set_h_dptr(a->dst_dentry, a->btgt, NULL); + au_set_dbtop(a->dst_dentry, a->dst_btop); + } +out_parent: + if (!err) { + if (d_unhashed(a->src_dentry)) + au_fset_ren(a->auren_flags, DROPPED_SRC); + if (d_unhashed(a->dst_dentry)) + au_fset_ren(a->auren_flags, DROPPED_DST); + if (!a->exchange) + d_move(a->src_dentry, a->dst_dentry); + else { + d_exchange(a->src_dentry, a->dst_dentry); + if (au_ftest_ren(a->auren_flags, DROPPED_DST)) + d_drop(a->dst_dentry); + } + if (au_ftest_ren(a->auren_flags, DROPPED_SRC)) + d_drop(a->src_dentry); + } else { + au_update_dbtop(a->dst_dentry); + if (!a->dst_inode) + d_drop(a->dst_dentry); + } + if (au_ftest_ren(a->auren_flags, ISSAMEDIR)) + di_write_unlock(a->dst_parent); + else + di_write_unlock2(a->src_parent, a->dst_parent); +out_unlock: + aufs_read_and_write_unlock2(a->dst_dentry, a->src_dentry); +out_free: + iput(a->dst_inode); + if (a->thargs) + au_whtmp_rmdir_free(a->thargs); + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c new file mode 100644 index 0000000000000..098fd115052a3 --- /dev/null +++ b/fs/aufs/iinfo.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode private data + */ + +#include "aufs.h" + +struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex) +{ + struct inode *h_inode; + struct au_hinode *hinode; + + IiMustAnyLock(inode); + + hinode = au_hinode(au_ii(inode), bindex); + h_inode = hinode->hi_inode; + AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0); + return h_inode; +} + +/* todo: hard/soft set? */ +void au_hiput(struct au_hinode *hinode) +{ + au_hn_free(hinode); + dput(hinode->hi_whdentry); + iput(hinode->hi_inode); +} + +unsigned int au_hi_flags(struct inode *inode, int isdir) +{ + unsigned int flags; + const unsigned int mnt_flags = au_mntflags(inode->i_sb); + + flags = 0; + if (au_opt_test(mnt_flags, XINO)) + au_fset_hi(flags, XINO); + if (isdir && au_opt_test(mnt_flags, UDBA_HNOTIFY)) + au_fset_hi(flags, HNOTIFY); + return flags; +} + +void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode, unsigned int flags) +{ + struct au_hinode *hinode; + struct inode *hi; + struct au_iinfo *iinfo = au_ii(inode); + + IiMustWriteLock(inode); + + hinode = au_hinode(iinfo, bindex); + hi = hinode->hi_inode; + AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0); + + if (hi) + au_hiput(hinode); + hinode->hi_inode = h_inode; + if (h_inode) { + int err; + struct super_block *sb = inode->i_sb; + struct au_branch *br; + + AuDebugOn(inode->i_mode + && (h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT)); + if (bindex == iinfo->ii_btop) + au_cpup_igen(inode, h_inode); + br = au_sbr(sb, bindex); + hinode->hi_id = br->br_id; + if (au_ftest_hi(flags, XINO)) { + err = au_xino_write(sb, bindex, h_inode->i_ino, + inode->i_ino); + if (unlikely(err)) + AuIOErr1("failed au_xino_write() %d\n", err); + } + + if (au_ftest_hi(flags, HNOTIFY) + && au_br_hnotifyable(br->br_perm)) { + err = au_hn_alloc(hinode, inode); + if (unlikely(err)) + AuIOErr1("au_hn_alloc() %d\n", err); + } + } +} + +void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_wh) +{ + struct au_hinode *hinode; + + IiMustWriteLock(inode); + + hinode = au_hinode(au_ii(inode), bindex); + AuDebugOn(hinode->hi_whdentry); + hinode->hi_whdentry = h_wh; +} + +void au_update_iigen(struct inode *inode, int half) +{ + struct au_iinfo *iinfo; + struct au_iigen *iigen; + unsigned int sigen; + + sigen = au_sigen(inode->i_sb); + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + iigen->ig_generation = sigen; + if (half) + au_ig_fset(iigen->ig_flags, HALF_REFRESHED); + else + au_ig_fclr(iigen->ig_flags, HALF_REFRESHED); + spin_unlock(&iigen->ig_spin); +} + +/* it may be called at remount time, too */ +void au_update_ibrange(struct inode *inode, int do_put_zero) +{ + struct au_iinfo *iinfo; + aufs_bindex_t bindex, bbot; + + AuDebugOn(au_is_bad_inode(inode)); + IiMustWriteLock(inode); + + iinfo = au_ii(inode); + if (do_put_zero && iinfo->ii_btop >= 0) { + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; + bindex++) { + struct inode *h_i; + + h_i = au_hinode(iinfo, bindex)->hi_inode; + if (h_i + && !h_i->i_nlink + && !(h_i->i_state & I_LINKABLE)) + au_set_h_iptr(inode, bindex, NULL, 0); + } + } + + iinfo->ii_btop = -1; + iinfo->ii_bbot = -1; + bbot = au_sbbot(inode->i_sb); + for (bindex = 0; bindex <= bbot; bindex++) + if (au_hinode(iinfo, bindex)->hi_inode) { + iinfo->ii_btop = bindex; + break; + } + if (iinfo->ii_btop >= 0) + for (bindex = bbot; bindex >= iinfo->ii_btop; bindex--) + if (au_hinode(iinfo, bindex)->hi_inode) { + iinfo->ii_bbot = bindex; + break; + } + AuDebugOn(iinfo->ii_btop > iinfo->ii_bbot); +} + +/* ---------------------------------------------------------------------- */ + +void au_icntnr_init_once(void *_c) +{ + struct au_icntnr *c = _c; + struct au_iinfo *iinfo = &c->iinfo; + + spin_lock_init(&iinfo->ii_generation.ig_spin); + au_rw_init(&iinfo->ii_rwsem); + inode_init_once(&c->vfs_inode); +} + +void au_hinode_init(struct au_hinode *hinode) +{ + hinode->hi_inode = NULL; + hinode->hi_id = -1; + au_hn_init(hinode); + hinode->hi_whdentry = NULL; +} + +int au_iinfo_init(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct super_block *sb; + struct au_hinode *hi; + int nbr, i; + + sb = inode->i_sb; + iinfo = &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo); + nbr = au_sbbot(sb) + 1; + if (unlikely(nbr <= 0)) + nbr = 1; + hi = kmalloc_array(nbr, sizeof(*iinfo->ii_hinode), GFP_NOFS); + if (hi) { + au_lcnt_inc(&au_sbi(sb)->si_ninodes); + + iinfo->ii_hinode = hi; + for (i = 0; i < nbr; i++, hi++) + au_hinode_init(hi); + + iinfo->ii_generation.ig_generation = au_sigen(sb); + iinfo->ii_btop = -1; + iinfo->ii_bbot = -1; + iinfo->ii_vdir = NULL; + return 0; + } + return -ENOMEM; +} + +int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink) +{ + int err, i; + struct au_hinode *hip; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + err = -ENOMEM; + hip = au_krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS, + may_shrink); + if (hip) { + iinfo->ii_hinode = hip; + i = iinfo->ii_bbot + 1; + hip += i; + for (; i < nbr; i++, hip++) + au_hinode_init(hip); + err = 0; + } + + return err; +} + +void au_iinfo_fin(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_hinode *hi; + struct super_block *sb; + aufs_bindex_t bindex, bbot; + const unsigned char unlinked = !inode->i_nlink; + + AuDebugOn(au_is_bad_inode(inode)); + + sb = inode->i_sb; + au_lcnt_dec(&au_sbi(sb)->si_ninodes); + if (si_pid_test(sb)) + au_xino_delete_inode(inode, unlinked); + else { + /* + * it is safe to hide the dependency between sbinfo and + * sb->s_umount. + */ + lockdep_off(); + si_noflush_read_lock(sb); + au_xino_delete_inode(inode, unlinked); + si_read_unlock(sb); + lockdep_on(); + } + + iinfo = au_ii(inode); + if (iinfo->ii_vdir) + au_vdir_free(iinfo->ii_vdir); + + bindex = iinfo->ii_btop; + if (bindex >= 0) { + hi = au_hinode(iinfo, bindex); + bbot = iinfo->ii_bbot; + while (bindex++ <= bbot) { + if (hi->hi_inode) + au_hiput(hi); + hi++; + } + } + au_kfree_rcu(iinfo->ii_hinode); + AuRwDestroy(&iinfo->ii_rwsem); +} diff --git a/fs/aufs/inode.c b/fs/aufs/inode.c new file mode 100644 index 0000000000000..2db13c0aafe51 --- /dev/null +++ b/fs/aufs/inode.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode functions + */ + +#include "aufs.h" + +struct inode *au_igrab(struct inode *inode) +{ + if (inode) { + AuDebugOn(!atomic_read(&inode->i_count)); + ihold(inode); + } + return inode; +} + +static void au_refresh_hinode_attr(struct inode *inode, int do_version) +{ + au_cpup_attr_all(inode, /*force*/0); + au_update_iigen(inode, /*half*/1); + if (do_version) + inode->i_version++; +} + +static int au_ii_refresh(struct inode *inode, int *update) +{ + int err, e, nbr; + umode_t type; + aufs_bindex_t bindex, new_bindex; + struct super_block *sb; + struct au_iinfo *iinfo; + struct au_hinode *p, *q, tmp; + + AuDebugOn(au_is_bad_inode(inode)); + IiMustWriteLock(inode); + + *update = 0; + sb = inode->i_sb; + nbr = au_sbbot(sb) + 1; + type = inode->i_mode & S_IFMT; + iinfo = au_ii(inode); + err = au_hinode_realloc(iinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + + AuDebugOn(iinfo->ii_btop < 0); + p = au_hinode(iinfo, iinfo->ii_btop); + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; + bindex++, p++) { + if (!p->hi_inode) + continue; + + AuDebugOn(type != (p->hi_inode->i_mode & S_IFMT)); + new_bindex = au_br_index(sb, p->hi_id); + if (new_bindex == bindex) + continue; + + if (new_bindex < 0) { + *update = 1; + au_hiput(p); + p->hi_inode = NULL; + continue; + } + + if (new_bindex < iinfo->ii_btop) + iinfo->ii_btop = new_bindex; + if (iinfo->ii_bbot < new_bindex) + iinfo->ii_bbot = new_bindex; + /* swap two lower inode, and loop again */ + q = au_hinode(iinfo, new_bindex); + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hi_inode) { + bindex--; + p--; + } + } + au_update_ibrange(inode, /*do_put_zero*/0); + au_hinode_realloc(iinfo, nbr, /*may_shrink*/1); /* harmless if err */ + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; + +out: + AuTraceErr(err); + return err; +} + +void au_refresh_iop(struct inode *inode, int force_getattr) +{ + int type; + struct au_sbinfo *sbi = au_sbi(inode->i_sb); + const struct inode_operations *iop + = force_getattr ? aufs_iop : sbi->si_iop_array; + + if (inode->i_op == iop) + return; + + switch (inode->i_mode & S_IFMT) { + case S_IFDIR: + type = AuIop_DIR; + break; + case S_IFLNK: + type = AuIop_SYMLINK; + break; + default: + type = AuIop_OTHER; + break; + } + + inode->i_op = iop + type; + /* unnecessary smp_wmb() */ +} + +int au_refresh_hinode_self(struct inode *inode) +{ + int err, update; + + err = au_ii_refresh(inode, &update); + if (!err) + au_refresh_hinode_attr(inode, update && S_ISDIR(inode->i_mode)); + + AuTraceErr(err); + return err; +} + +int au_refresh_hinode(struct inode *inode, struct dentry *dentry) +{ + int err, e, update; + unsigned int flags; + umode_t mode; + aufs_bindex_t bindex, bbot; + unsigned char isdir; + struct au_hinode *p; + struct au_iinfo *iinfo; + + err = au_ii_refresh(inode, &update); + if (unlikely(err)) + goto out; + + update = 0; + iinfo = au_ii(inode); + p = au_hinode(iinfo, iinfo->ii_btop); + mode = (inode->i_mode & S_IFMT); + isdir = S_ISDIR(mode); + flags = au_hi_flags(inode, isdir); + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) { + struct inode *h_i, *h_inode; + struct dentry *h_d; + + h_d = au_h_dptr(dentry, bindex); + if (!h_d || d_is_negative(h_d)) + continue; + + h_inode = d_inode(h_d); + AuDebugOn(mode != (h_inode->i_mode & S_IFMT)); + if (iinfo->ii_btop <= bindex && bindex <= iinfo->ii_bbot) { + h_i = au_h_iptr(inode, bindex); + if (h_i) { + if (h_i == h_inode) + continue; + err = -EIO; + break; + } + } + if (bindex < iinfo->ii_btop) + iinfo->ii_btop = bindex; + if (iinfo->ii_bbot < bindex) + iinfo->ii_bbot = bindex; + au_set_h_iptr(inode, bindex, au_igrab(h_inode), flags); + update = 1; + } + au_update_ibrange(inode, /*do_put_zero*/0); + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; + if (!err) + au_refresh_hinode_attr(inode, update && isdir); + +out: + AuTraceErr(err); + return err; +} + +static int set_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + unsigned int flags; + umode_t mode; + aufs_bindex_t bindex, btop, btail; + unsigned char isdir; + struct dentry *h_dentry; + struct inode *h_inode; + struct au_iinfo *iinfo; + struct inode_operations *iop; + + IiMustWriteLock(inode); + + err = 0; + isdir = 0; + iop = au_sbi(inode->i_sb)->si_iop_array; + btop = au_dbtop(dentry); + h_dentry = au_h_dptr(dentry, btop); + h_inode = d_inode(h_dentry); + mode = h_inode->i_mode; + switch (mode & S_IFMT) { + case S_IFREG: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_OTHER; + inode->i_fop = &aufs_file_fop; + err = au_dy_iaop(inode, btop, h_inode); + if (unlikely(err)) + goto out; + break; + case S_IFDIR: + isdir = 1; + btail = au_dbtaildir(dentry); + inode->i_op = iop + AuIop_DIR; + inode->i_fop = &aufs_dir_fop; + break; + case S_IFLNK: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_SYMLINK; + break; + case S_IFBLK: + case S_IFCHR: + case S_IFIFO: + case S_IFSOCK: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_OTHER; + init_special_inode(inode, mode, h_inode->i_rdev); + break; + default: + AuIOErr("Unknown file type 0%o\n", mode); + err = -EIO; + goto out; + } + + /* do not set hnotify for whiteouted dirs (SHWH mode) */ + flags = au_hi_flags(inode, isdir); + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH) + && au_ftest_hi(flags, HNOTIFY) + && dentry->d_name.len > AUFS_WH_PFX_LEN + && !memcmp(dentry->d_name.name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) + au_fclr_hi(flags, HNOTIFY); + iinfo = au_ii(inode); + iinfo->ii_btop = btop; + iinfo->ii_bbot = btail; + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) + au_set_h_iptr(inode, bindex, + au_igrab(d_inode(h_dentry)), flags); + } + au_cpup_attr_all(inode, /*force*/1); + /* + * to force calling aufs_get_acl() every time, + * do not call cache_no_acl() for aufs inode. + */ + +out: + return err; +} + +/* + * successful returns with iinfo write_locked + * minus: errno + * zero: success, matched + * plus: no error, but unmatched + */ +static int reval_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + unsigned int gen, igflags; + aufs_bindex_t bindex, bbot; + struct inode *h_inode, *h_dinode; + struct dentry *h_dentry; + + /* + * before this function, if aufs got any iinfo lock, it must be only + * one, the parent dir. + * it can happen by UDBA and the obsoleted inode number. + */ + err = -EIO; + if (unlikely(inode->i_ino == parent_ino(dentry))) + goto out; + + err = 1; + ii_write_lock_new_child(inode); + h_dentry = au_h_dptr(dentry, au_dbtop(dentry)); + h_dinode = d_inode(h_dentry); + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (!h_inode || h_inode != h_dinode) + continue; + + err = 0; + gen = au_iigen(inode, &igflags); + if (gen == au_digen(dentry) + && !au_ig_ftest(igflags, HALF_REFRESHED)) + break; + + /* fully refresh inode using dentry */ + err = au_refresh_hinode(inode, dentry); + if (!err) + au_update_iigen(inode, /*half*/0); + break; + } + + if (unlikely(err)) + ii_write_unlock(inode); +out: + return err; +} + +int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + unsigned int d_type, ino_t *ino) +{ + int err, idx; + const int isnondir = d_type != DT_DIR; + + /* prevent hardlinked inode number from race condition */ + if (isnondir) { + err = au_xinondir_enter(sb, bindex, h_ino, &idx); + if (unlikely(err)) + goto out; + } + + err = au_xino_read(sb, bindex, h_ino, ino); + if (unlikely(err)) + goto out_xinondir; + + if (!*ino) { + err = -EIO; + *ino = au_xino_new_ino(sb); + if (unlikely(!*ino)) + goto out_xinondir; + err = au_xino_write(sb, bindex, h_ino, *ino); + if (unlikely(err)) + goto out_xinondir; + } + +out_xinondir: + if (isnondir && idx >= 0) + au_xinondir_leave(sb, bindex, h_ino, idx); +out: + return err; +} + +/* successful returns with iinfo write_locked */ +/* todo: return with unlocked? */ +struct inode *au_new_inode(struct dentry *dentry, int must_new) +{ + struct inode *inode, *h_inode; + struct dentry *h_dentry; + struct super_block *sb; + ino_t h_ino, ino; + int err, idx, hlinked; + aufs_bindex_t btop; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + h_dentry = au_h_dptr(dentry, btop); + h_inode = d_inode(h_dentry); + h_ino = h_inode->i_ino; + hlinked = !d_is_dir(h_dentry) && h_inode->i_nlink > 1; + +new_ino: + /* + * stop 'race'-ing between hardlinks under different + * parents. + */ + if (hlinked) { + err = au_xinondir_enter(sb, btop, h_ino, &idx); + inode = ERR_PTR(err); + if (unlikely(err)) + goto out; + } + + err = au_xino_read(sb, btop, h_ino, &ino); + inode = ERR_PTR(err); + if (unlikely(err)) + goto out_xinondir; + + if (!ino) { + ino = au_xino_new_ino(sb); + if (unlikely(!ino)) { + inode = ERR_PTR(-EIO); + goto out_xinondir; + } + } + + AuDbg("i%lu\n", (unsigned long)ino); + inode = au_iget_locked(sb, ino); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_xinondir; + + AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW)); + if (inode->i_state & I_NEW) { + ii_write_lock_new_child(inode); + err = set_inode(inode, dentry); + if (!err) { + unlock_new_inode(inode); + goto out_xinondir; /* success */ + } + + /* + * iget_failed() calls iput(), but we need to call + * ii_write_unlock() after iget_failed(). so dirty hack for + * i_count. + */ + atomic_inc(&inode->i_count); + iget_failed(inode); + ii_write_unlock(inode); + au_xino_write(sb, btop, h_ino, /*ino*/0); + /* ignore this error */ + goto out_iput; + } else if (!must_new && !IS_DEADDIR(inode) && inode->i_nlink) { + /* + * horrible race condition between lookup, readdir and copyup + * (or something). + */ + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); + err = reval_inode(inode, dentry); + if (unlikely(err < 0)) { + hlinked = 0; + goto out_iput; + } + if (!err) + goto out; /* success */ + else if (hlinked && idx >= 0) { + err = au_xinondir_enter(sb, btop, h_ino, &idx); + if (unlikely(err)) { + iput(inode); + inode = ERR_PTR(err); + goto out; + } + } + } + + if (unlikely(au_test_fs_unique_ino(h_inode))) + AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir," + " b%d, %s, %pd, hi%lu, i%lu.\n", + btop, au_sbtype(h_dentry->d_sb), dentry, + (unsigned long)h_ino, (unsigned long)ino); + ino = 0; + err = au_xino_write(sb, btop, h_ino, /*ino*/0); + if (!err) { + iput(inode); + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); + goto new_ino; + } + +out_iput: + iput(inode); + inode = ERR_PTR(err); +out_xinondir: + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); +out: + return inode; +} + +/* ---------------------------------------------------------------------- */ + +int au_test_ro(struct super_block *sb, aufs_bindex_t bindex, + struct inode *inode) +{ + int err; + struct inode *hi; + + err = au_br_rdonly(au_sbr(sb, bindex)); + + /* pseudo-link after flushed may happen out of bounds */ + if (!err + && inode + && au_ibtop(inode) <= bindex + && bindex <= au_ibbot(inode)) { + /* + * permission check is unnecessary since vfsub routine + * will be called later + */ + hi = au_h_iptr(inode, bindex); + if (hi) + err = IS_IMMUTABLE(hi) ? -EROFS : 0; + } + + return err; +} + +int au_test_h_perm(struct inode *h_inode, int mask) +{ + if (uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) + return 0; + return inode_permission(h_inode, mask); +} + +int au_test_h_perm_sio(struct inode *h_inode, int mask) +{ + if (au_test_nfs(h_inode->i_sb) + && (mask & MAY_WRITE) + && S_ISDIR(h_inode->i_mode)) + mask |= MAY_READ; /* force permission check */ + return au_test_h_perm(h_inode, mask); +} diff --git a/fs/aufs/inode.h b/fs/aufs/inode.h new file mode 100644 index 0000000000000..741b5b835ac69 --- /dev/null +++ b/fs/aufs/inode.h @@ -0,0 +1,698 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations + */ + +#ifndef __AUFS_INODE_H__ +#define __AUFS_INODE_H__ + +#ifdef __KERNEL__ + +#include +#include "rwsem.h" + +struct vfsmount; + +struct au_hnotify { +#ifdef CONFIG_AUFS_HNOTIFY +#ifdef CONFIG_AUFS_HFSNOTIFY + /* never use fsnotify_add_vfsmount_mark() */ + struct fsnotify_mark hn_mark; +#endif + struct inode *hn_aufs_inode; /* no get/put */ + struct rcu_head rcu; +#endif +} ____cacheline_aligned_in_smp; + +struct au_hinode { + struct inode *hi_inode; + aufs_bindex_t hi_id; +#ifdef CONFIG_AUFS_HNOTIFY + struct au_hnotify *hi_notify; +#endif + + /* reference to the copied-up whiteout with get/put */ + struct dentry *hi_whdentry; +}; + +/* ig_flags */ +#define AuIG_HALF_REFRESHED 1 +#define au_ig_ftest(flags, name) ((flags) & AuIG_##name) +#define au_ig_fset(flags, name) \ + do { (flags) |= AuIG_##name; } while (0) +#define au_ig_fclr(flags, name) \ + do { (flags) &= ~AuIG_##name; } while (0) + +struct au_iigen { + spinlock_t ig_spin; + __u32 ig_generation, ig_flags; +}; + +struct au_vdir; +struct au_iinfo { + struct au_iigen ii_generation; + struct super_block *ii_hsb1; /* no get/put */ + + struct au_rwsem ii_rwsem; + aufs_bindex_t ii_btop, ii_bbot; + __u32 ii_higen; + struct au_hinode *ii_hinode; + struct au_vdir *ii_vdir; +}; + +struct au_icntnr { + struct au_iinfo iinfo; + struct inode vfs_inode; + struct hlist_bl_node plink; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* au_pin flags */ +#define AuPin_DI_LOCKED 1 +#define AuPin_MNT_WRITE (1 << 1) +#define au_ftest_pin(flags, name) ((flags) & AuPin_##name) +#define au_fset_pin(flags, name) \ + do { (flags) |= AuPin_##name; } while (0) +#define au_fclr_pin(flags, name) \ + do { (flags) &= ~AuPin_##name; } while (0) + +struct au_pin { + /* input */ + struct dentry *dentry; + unsigned int udba; + unsigned char lsc_di, lsc_hi, flags; + aufs_bindex_t bindex; + + /* output */ + struct dentry *parent; + struct au_hinode *hdir; + struct vfsmount *h_mnt; + + /* temporary unlock/relock for copyup */ + struct dentry *h_dentry, *h_parent; + struct au_branch *br; + struct task_struct *task; +}; + +void au_pin_hdir_unlock(struct au_pin *p); +int au_pin_hdir_lock(struct au_pin *p); +int au_pin_hdir_relock(struct au_pin *p); +void au_pin_hdir_acquire_nest(struct au_pin *p); +void au_pin_hdir_release(struct au_pin *p); + +/* ---------------------------------------------------------------------- */ + +static inline struct au_iinfo *au_ii(struct inode *inode) +{ + BUG_ON(is_bad_inode(inode)); + return &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo); +} + +/* ---------------------------------------------------------------------- */ + +/* inode.c */ +struct inode *au_igrab(struct inode *inode); +void au_refresh_iop(struct inode *inode, int force_getattr); +int au_refresh_hinode_self(struct inode *inode); +int au_refresh_hinode(struct inode *inode, struct dentry *dentry); +int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + unsigned int d_type, ino_t *ino); +struct inode *au_new_inode(struct dentry *dentry, int must_new); +int au_test_ro(struct super_block *sb, aufs_bindex_t bindex, + struct inode *inode); +int au_test_h_perm(struct inode *h_inode, int mask); +int au_test_h_perm_sio(struct inode *h_inode, int mask); + +static inline int au_wh_ino(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, unsigned int d_type, ino_t *ino) +{ +#ifdef CONFIG_AUFS_SHWH + return au_ino(sb, bindex, h_ino, d_type, ino); +#else + return 0; +#endif +} + +/* i_op.c */ +enum { + AuIop_SYMLINK, + AuIop_DIR, + AuIop_OTHER, + AuIop_Last +}; +extern struct inode_operations aufs_iop[AuIop_Last], + aufs_iop_nogetattr[AuIop_Last]; + +/* au_wr_dir flags */ +#define AuWrDir_ADD_ENTRY 1 +#define AuWrDir_ISDIR (1 << 1) +#define AuWrDir_TMPFILE (1 << 2) +#define au_ftest_wrdir(flags, name) ((flags) & AuWrDir_##name) +#define au_fset_wrdir(flags, name) \ + do { (flags) |= AuWrDir_##name; } while (0) +#define au_fclr_wrdir(flags, name) \ + do { (flags) &= ~AuWrDir_##name; } while (0) + +struct au_wr_dir_args { + aufs_bindex_t force_btgt; + unsigned char flags; +}; +int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry, + struct au_wr_dir_args *args); + +struct dentry *au_pinned_h_parent(struct au_pin *pin); +void au_pin_init(struct au_pin *pin, struct dentry *dentry, + aufs_bindex_t bindex, int lsc_di, int lsc_hi, + unsigned int udba, unsigned char flags); +int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex, + unsigned int udba, unsigned char flags) __must_check; +int au_do_pin(struct au_pin *pin) __must_check; +void au_unpin(struct au_pin *pin); +int au_reval_for_attr(struct dentry *dentry, unsigned int sigen); + +#define AuIcpup_DID_CPUP 1 +#define au_ftest_icpup(flags, name) ((flags) & AuIcpup_##name) +#define au_fset_icpup(flags, name) \ + do { (flags) |= AuIcpup_##name; } while (0) +#define au_fclr_icpup(flags, name) \ + do { (flags) &= ~AuIcpup_##name; } while (0) + +struct au_icpup_args { + unsigned char flags; + unsigned char pin_flags; + aufs_bindex_t btgt; + unsigned int udba; + struct au_pin pin; + struct path h_path; + struct inode *h_inode; +}; + +int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia, + struct au_icpup_args *a); + +int au_h_path_getattr(struct dentry *dentry, int force, struct path *h_path, + int locked); + +/* i_op_add.c */ +int au_may_add(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir); +int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t dev); +int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname); +int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + bool want_excl); +struct vfsub_aopen_args; +int au_aopen_or_create(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args); +int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode); +int aufs_link(struct dentry *src_dentry, struct inode *dir, + struct dentry *dentry); +int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode); + +/* i_op_del.c */ +int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup); +int au_may_del(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir); +int aufs_unlink(struct inode *dir, struct dentry *dentry); +int aufs_rmdir(struct inode *dir, struct dentry *dentry); + +/* i_op_ren.c */ +int au_wbr(struct dentry *dentry, aufs_bindex_t btgt); +int aufs_rename(struct inode *src_dir, struct dentry *src_dentry, + struct inode *dir, struct dentry *dentry, + unsigned int flags); + +/* iinfo.c */ +struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex); +void au_hiput(struct au_hinode *hinode); +void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_wh); +unsigned int au_hi_flags(struct inode *inode, int isdir); + +/* hinode flags */ +#define AuHi_XINO 1 +#define AuHi_HNOTIFY (1 << 1) +#define au_ftest_hi(flags, name) ((flags) & AuHi_##name) +#define au_fset_hi(flags, name) \ + do { (flags) |= AuHi_##name; } while (0) +#define au_fclr_hi(flags, name) \ + do { (flags) &= ~AuHi_##name; } while (0) + +#ifndef CONFIG_AUFS_HNOTIFY +#undef AuHi_HNOTIFY +#define AuHi_HNOTIFY 0 +#endif + +void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode, unsigned int flags); + +void au_update_iigen(struct inode *inode, int half); +void au_update_ibrange(struct inode *inode, int do_put_zero); + +void au_icntnr_init_once(void *_c); +void au_hinode_init(struct au_hinode *hinode); +int au_iinfo_init(struct inode *inode); +void au_iinfo_fin(struct inode *inode); +int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink); + +#ifdef CONFIG_PROC_FS +/* plink.c */ +int au_plink_maint(struct super_block *sb, int flags); +struct au_sbinfo; +void au_plink_maint_leave(struct au_sbinfo *sbinfo); +int au_plink_maint_enter(struct super_block *sb); +#ifdef CONFIG_AUFS_DEBUG +void au_plink_list(struct super_block *sb); +#else +AuStubVoid(au_plink_list, struct super_block *sb) +#endif +int au_plink_test(struct inode *inode); +struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex); +void au_plink_append(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry); +void au_plink_put(struct super_block *sb, int verbose); +void au_plink_clean(struct super_block *sb, int verbose); +void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id); +#else +AuStubInt0(au_plink_maint, struct super_block *sb, int flags); +AuStubVoid(au_plink_maint_leave, struct au_sbinfo *sbinfo); +AuStubInt0(au_plink_maint_enter, struct super_block *sb); +AuStubVoid(au_plink_list, struct super_block *sb); +AuStubInt0(au_plink_test, struct inode *inode); +AuStub(struct dentry *, au_plink_lkup, return NULL, + struct inode *inode, aufs_bindex_t bindex); +AuStubVoid(au_plink_append, struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry); +AuStubVoid(au_plink_put, struct super_block *sb, int verbose); +AuStubVoid(au_plink_clean, struct super_block *sb, int verbose); +AuStubVoid(au_plink_half_refresh, struct super_block *sb, aufs_bindex_t br_id); +#endif /* CONFIG_PROC_FS */ + +#ifdef CONFIG_AUFS_XATTR +/* xattr.c */ +int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, + unsigned int verbose); +ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size); +void au_xattr_init(struct super_block *sb); +#else +AuStubInt0(au_cpup_xattr, struct dentry *h_dst, struct dentry *h_src, + int ignore_flags, unsigned int verbose); +AuStubVoid(au_xattr_init, struct super_block *sb); +#endif + +#ifdef CONFIG_FS_POSIX_ACL +struct posix_acl *aufs_get_acl(struct inode *inode, int type); +int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +#endif + +#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL) +enum { + AU_XATTR_SET, + AU_ACL_SET +}; + +struct au_sxattr { + int type; + union { + struct { + const char *name; + const void *value; + size_t size; + int flags; + } set; + struct { + struct posix_acl *acl; + int type; + } acl_set; + } u; +}; +ssize_t au_sxattr(struct dentry *dentry, struct inode *inode, + struct au_sxattr *arg); +#endif + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for iinfo */ +enum { + AuLsc_II_CHILD, /* child first */ + AuLsc_II_CHILD2, /* rename(2), link(2), and cpup at hnotify */ + AuLsc_II_CHILD3, /* copyup dirs */ + AuLsc_II_PARENT, /* see AuLsc_I_PARENT in vfsub.h */ + AuLsc_II_PARENT2, + AuLsc_II_PARENT3, /* copyup dirs */ + AuLsc_II_NEW_CHILD +}; + +/* + * ii_read_lock_child, ii_write_lock_child, + * ii_read_lock_child2, ii_write_lock_child2, + * ii_read_lock_child3, ii_write_lock_child3, + * ii_read_lock_parent, ii_write_lock_parent, + * ii_read_lock_parent2, ii_write_lock_parent2, + * ii_read_lock_parent3, ii_write_lock_parent3, + * ii_read_lock_new_child, ii_write_lock_new_child, + */ +#define AuReadLockFunc(name, lsc) \ +static inline void ii_read_lock_##name(struct inode *i) \ +{ \ + au_rw_read_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \ +} + +#define AuWriteLockFunc(name, lsc) \ +static inline void ii_write_lock_##name(struct inode *i) \ +{ \ + au_rw_write_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \ +} + +#define AuRWLockFuncs(name, lsc) \ + AuReadLockFunc(name, lsc) \ + AuWriteLockFunc(name, lsc) + +AuRWLockFuncs(child, CHILD); +AuRWLockFuncs(child2, CHILD2); +AuRWLockFuncs(child3, CHILD3); +AuRWLockFuncs(parent, PARENT); +AuRWLockFuncs(parent2, PARENT2); +AuRWLockFuncs(parent3, PARENT3); +AuRWLockFuncs(new_child, NEW_CHILD); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define ii_read_unlock(i) au_rw_read_unlock(&au_ii(i)->ii_rwsem) +#define ii_write_unlock(i) au_rw_write_unlock(&au_ii(i)->ii_rwsem) +#define ii_downgrade_lock(i) au_rw_dgrade_lock(&au_ii(i)->ii_rwsem) + +#define IiMustNoWaiters(i) AuRwMustNoWaiters(&au_ii(i)->ii_rwsem) +#define IiMustAnyLock(i) AuRwMustAnyLock(&au_ii(i)->ii_rwsem) +#define IiMustWriteLock(i) AuRwMustWriteLock(&au_ii(i)->ii_rwsem) + +/* ---------------------------------------------------------------------- */ + +static inline void au_icntnr_init(struct au_icntnr *c) +{ +#ifdef CONFIG_AUFS_DEBUG + c->vfs_inode.i_mode = 0; +#endif +} + +static inline unsigned int au_iigen(struct inode *inode, unsigned int *igflags) +{ + unsigned int gen; + struct au_iinfo *iinfo; + struct au_iigen *iigen; + + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + if (igflags) + *igflags = iigen->ig_flags; + gen = iigen->ig_generation; + spin_unlock(&iigen->ig_spin); + + return gen; +} + +/* tiny test for inode number */ +/* tmpfs generation is too rough */ +static inline int au_test_higen(struct inode *inode, struct inode *h_inode) +{ + struct au_iinfo *iinfo; + + iinfo = au_ii(inode); + AuRwMustAnyLock(&iinfo->ii_rwsem); + return !(iinfo->ii_hsb1 == h_inode->i_sb + && iinfo->ii_higen == h_inode->i_generation); +} + +static inline void au_iigen_dec(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_iigen *iigen; + + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + iigen->ig_generation--; + spin_unlock(&iigen->ig_spin); +} + +static inline int au_iigen_test(struct inode *inode, unsigned int sigen) +{ + int err; + + err = 0; + if (unlikely(inode && au_iigen(inode, NULL) != sigen)) + err = -EIO; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static inline struct au_hinode *au_hinode(struct au_iinfo *iinfo, + aufs_bindex_t bindex) +{ + return iinfo->ii_hinode + bindex; +} + +static inline int au_is_bad_inode(struct inode *inode) +{ + return !!(is_bad_inode(inode) || !au_hinode(au_ii(inode), 0)); +} + +static inline aufs_bindex_t au_ii_br_id(struct inode *inode, + aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex)->hi_id; +} + +static inline aufs_bindex_t au_ibtop(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_btop; +} + +static inline aufs_bindex_t au_ibbot(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_bbot; +} + +static inline struct au_vdir *au_ivdir(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_vdir; +} + +static inline struct dentry *au_hi_wh(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex)->hi_whdentry; +} + +static inline void au_set_ibtop(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_btop = bindex; +} + +static inline void au_set_ibbot(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_bbot = bindex; +} + +static inline void au_set_ivdir(struct inode *inode, struct au_vdir *vdir) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_vdir = vdir; +} + +static inline struct au_hinode *au_hi(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex); +} + +/* ---------------------------------------------------------------------- */ + +static inline struct dentry *au_pinned_parent(struct au_pin *pin) +{ + if (pin) + return pin->parent; + return NULL; +} + +static inline struct inode *au_pinned_h_dir(struct au_pin *pin) +{ + if (pin && pin->hdir) + return pin->hdir->hi_inode; + return NULL; +} + +static inline struct au_hinode *au_pinned_hdir(struct au_pin *pin) +{ + if (pin) + return pin->hdir; + return NULL; +} + +static inline void au_pin_set_dentry(struct au_pin *pin, struct dentry *dentry) +{ + if (pin) + pin->dentry = dentry; +} + +static inline void au_pin_set_parent_lflag(struct au_pin *pin, + unsigned char lflag) +{ + if (pin) { + if (lflag) + au_fset_pin(pin->flags, DI_LOCKED); + else + au_fclr_pin(pin->flags, DI_LOCKED); + } +} + +#if 0 /* reserved */ +static inline void au_pin_set_parent(struct au_pin *pin, struct dentry *parent) +{ + if (pin) { + dput(pin->parent); + pin->parent = dget(parent); + } +} +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_branch; +#ifdef CONFIG_AUFS_HNOTIFY +struct au_hnotify_op { + void (*ctl)(struct au_hinode *hinode, int do_set); + int (*alloc)(struct au_hinode *hinode); + + /* + * if it returns true, the the caller should free hinode->hi_notify, + * otherwise ->free() frees it. + */ + int (*free)(struct au_hinode *hinode, + struct au_hnotify *hn) __must_check; + + void (*fin)(void); + int (*init)(void); + + int (*reset_br)(unsigned int udba, struct au_branch *br, int perm); + void (*fin_br)(struct au_branch *br); + int (*init_br)(struct au_branch *br, int perm); +}; + +/* hnotify.c */ +int au_hn_alloc(struct au_hinode *hinode, struct inode *inode); +void au_hn_free(struct au_hinode *hinode); +void au_hn_ctl(struct au_hinode *hinode, int do_set); +void au_hn_reset(struct inode *inode, unsigned int flags); +int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask, + struct qstr *h_child_qstr, struct inode *h_child_inode); +int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm); +int au_hnotify_init_br(struct au_branch *br, int perm); +void au_hnotify_fin_br(struct au_branch *br); +int __init au_hnotify_init(void); +void au_hnotify_fin(void); + +/* hfsnotify.c */ +extern const struct au_hnotify_op au_hnotify_op; + +static inline +void au_hn_init(struct au_hinode *hinode) +{ + hinode->hi_notify = NULL; +} + +static inline struct au_hnotify *au_hn(struct au_hinode *hinode) +{ + return hinode->hi_notify; +} + +#else +AuStub(int, au_hn_alloc, return -EOPNOTSUPP, + struct au_hinode *hinode __maybe_unused, + struct inode *inode __maybe_unused) +AuStub(struct au_hnotify *, au_hn, return NULL, struct au_hinode *hinode) +AuStubVoid(au_hn_free, struct au_hinode *hinode __maybe_unused) +AuStubVoid(au_hn_ctl, struct au_hinode *hinode __maybe_unused, + int do_set __maybe_unused) +AuStubVoid(au_hn_reset, struct inode *inode __maybe_unused, + unsigned int flags __maybe_unused) +AuStubInt0(au_hnotify_reset_br, unsigned int udba __maybe_unused, + struct au_branch *br __maybe_unused, + int perm __maybe_unused) +AuStubInt0(au_hnotify_init_br, struct au_branch *br __maybe_unused, + int perm __maybe_unused) +AuStubVoid(au_hnotify_fin_br, struct au_branch *br __maybe_unused) +AuStubInt0(__init au_hnotify_init, void) +AuStubVoid(au_hnotify_fin, void) +AuStubVoid(au_hn_init, struct au_hinode *hinode __maybe_unused) +#endif /* CONFIG_AUFS_HNOTIFY */ + +static inline void au_hn_suspend(struct au_hinode *hdir) +{ + au_hn_ctl(hdir, /*do_set*/0); +} + +static inline void au_hn_resume(struct au_hinode *hdir) +{ + au_hn_ctl(hdir, /*do_set*/1); +} + +static inline void au_hn_inode_lock(struct au_hinode *hdir) +{ + inode_lock(hdir->hi_inode); + au_hn_suspend(hdir); +} + +static inline void au_hn_inode_lock_nested(struct au_hinode *hdir, + unsigned int sc __maybe_unused) +{ + inode_lock_nested(hdir->hi_inode, sc); + au_hn_suspend(hdir); +} + +#if 0 /* unused */ +#include "vfsub.h" +static inline void au_hn_inode_lock_shared_nested(struct au_hinode *hdir, + unsigned int sc) +{ + vfsub_inode_lock_shared_nested(hdir->hi_inode, sc); + au_hn_suspend(hdir); +} +#endif + +static inline void au_hn_inode_unlock(struct au_hinode *hdir) +{ + au_hn_resume(hdir); + inode_unlock(hdir->hi_inode); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_INODE_H__ */ diff --git a/fs/aufs/ioctl.c b/fs/aufs/ioctl.c new file mode 100644 index 0000000000000..fb2fb758283e6 --- /dev/null +++ b/fs/aufs/ioctl.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * ioctl + * plink-management and readdir in userspace. + * assist the pathconf(3) wrapper library. + * move-down + * File-based Hierarchical Storage Management. + */ + +#include +#include +#include "aufs.h" + +static int au_wbr_fd(struct path *path, struct aufs_wbr_fd __user *arg) +{ + int err, fd; + aufs_bindex_t wbi, bindex, bbot; + struct file *h_file; + struct super_block *sb; + struct dentry *root; + struct au_branch *br; + struct aufs_wbr_fd wbrfd = { + .oflags = au_dir_roflags, + .brid = -1 + }; + const int valid = O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_DIRECTORY + | O_NOATIME | O_CLOEXEC; + + AuDebugOn(wbrfd.oflags & ~valid); + + if (arg) { + err = copy_from_user(&wbrfd, arg, sizeof(wbrfd)); + if (unlikely(err)) { + err = -EFAULT; + goto out; + } + + err = -EINVAL; + AuDbg("wbrfd{0%o, %d}\n", wbrfd.oflags, wbrfd.brid); + wbrfd.oflags |= au_dir_roflags; + AuDbg("0%o\n", wbrfd.oflags); + if (unlikely(wbrfd.oflags & ~valid)) + goto out; + } + + fd = get_unused_fd_flags(0); + err = fd; + if (unlikely(fd < 0)) + goto out; + + h_file = ERR_PTR(-EINVAL); + wbi = 0; + br = NULL; + sb = path->dentry->d_sb; + root = sb->s_root; + aufs_read_lock(root, AuLock_IR); + bbot = au_sbbot(sb); + if (wbrfd.brid >= 0) { + wbi = au_br_index(sb, wbrfd.brid); + if (unlikely(wbi < 0 || wbi > bbot)) + goto out_unlock; + } + + h_file = ERR_PTR(-ENOENT); + br = au_sbr(sb, wbi); + if (!au_br_writable(br->br_perm)) { + if (arg) + goto out_unlock; + + bindex = wbi + 1; + wbi = -1; + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm)) { + wbi = bindex; + br = au_sbr(sb, wbi); + break; + } + } + } + AuDbg("wbi %d\n", wbi); + if (wbi >= 0) + h_file = au_h_open(root, wbi, wbrfd.oflags, NULL, + /*force_wr*/0); + +out_unlock: + aufs_read_unlock(root, AuLock_IR); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out_fd; + + au_lcnt_dec(&br->br_nfiles); /* cf. au_h_open() */ + fd_install(fd, h_file); + err = fd; + goto out; /* success */ + +out_fd: + put_unused_fd(fd); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err; + struct dentry *dentry; + + switch (cmd) { + case AUFS_CTL_RDU: + case AUFS_CTL_RDU_INO: + err = au_rdu_ioctl(file, cmd, arg); + break; + + case AUFS_CTL_WBR_FD: + err = au_wbr_fd(&file->f_path, (void __user *)arg); + break; + + case AUFS_CTL_IBUSY: + err = au_ibusy_ioctl(file, arg); + break; + + case AUFS_CTL_BRINFO: + err = au_brinfo_ioctl(file, arg); + break; + + case AUFS_CTL_FHSM_FD: + dentry = file->f_path.dentry; + if (IS_ROOT(dentry)) + err = au_fhsm_fd(dentry->d_sb, arg); + else + err = -ENOTTY; + break; + + default: + /* do not call the lower */ + AuDbg("0x%x\n", cmd); + err = -ENOTTY; + } + + AuTraceErr(err); + return err; +} + +long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err; + + switch (cmd) { + case AUFS_CTL_MVDOWN: + err = au_mvdown(file->f_path.dentry, (void __user *)arg); + break; + + case AUFS_CTL_WBR_FD: + err = au_wbr_fd(&file->f_path, (void __user *)arg); + break; + + default: + /* do not call the lower */ + AuDbg("0x%x\n", cmd); + err = -ENOTTY; + } + + AuTraceErr(err); + return err; +} + +#ifdef CONFIG_COMPAT +long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd, + unsigned long arg) +{ + long err; + + switch (cmd) { + case AUFS_CTL_RDU: + case AUFS_CTL_RDU_INO: + err = au_rdu_compat_ioctl(file, cmd, arg); + break; + + case AUFS_CTL_IBUSY: + err = au_ibusy_compat_ioctl(file, arg); + break; + + case AUFS_CTL_BRINFO: + err = au_brinfo_compat_ioctl(file, arg); + break; + + default: + err = aufs_ioctl_dir(file, cmd, arg); + } + + AuTraceErr(err); + return err; +} + +long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return aufs_ioctl_nondir(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif diff --git a/fs/aufs/lcnt.h b/fs/aufs/lcnt.h new file mode 100644 index 0000000000000..d8508f875f283 --- /dev/null +++ b/fs/aufs/lcnt.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * simple long counter wrapper + */ + +#ifndef __AUFS_LCNT_H__ +#define __AUFS_LCNT_H__ + +#ifdef __KERNEL__ + +#include "debug.h" + +#define AuLCntATOMIC 1 +#define AuLCntPCPUCNT 2 +/* + * why does percpu_refcount require extra synchronize_rcu()s in + * au_br_do_free() + */ +#define AuLCntPCPUREF 3 + +/* #define AuLCntChosen AuLCntATOMIC */ +#define AuLCntChosen AuLCntPCPUCNT +/* #define AuLCntChosen AuLCntPCPUREF */ + +#if AuLCntChosen == AuLCntATOMIC +#include + +typedef atomic_long_t au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused) +{ + atomic_long_set(cnt, 0); + return 0; +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt __maybe_unused, + int do_sync __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + atomic_long_inc(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + atomic_long_dec(cnt); +} + +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused) +{ + return atomic_long_read(cnt); +} +#endif + +#if AuLCntChosen == AuLCntPCPUCNT +#include + +typedef struct percpu_counter au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused) +{ + return percpu_counter_init(cnt, 0, GFP_NOFS); +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync __maybe_unused) +{ + percpu_counter_destroy(cnt); +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + percpu_counter_inc(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + percpu_counter_dec(cnt); +} + +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused) +{ + s64 n; + + n = percpu_counter_sum(cnt); + BUG_ON(n < 0); + if (LONG_MAX != LLONG_MAX + && n > LONG_MAX) + AuWarn1("%s\n", "wrap-around"); + + return n; +} +#endif + +#if AuLCntChosen == AuLCntPCPUREF +#include + +typedef struct percpu_ref au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, percpu_ref_func_t *release) +{ + if (!release) + release = percpu_ref_exit; + return percpu_ref_init(cnt, release, /*percpu mode*/0, GFP_NOFS); +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + synchronize_rcu(); +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync) +{ + percpu_ref_kill(cnt); + if (do_sync) + au_lcnt_wait_for_fin(cnt); +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + percpu_ref_get(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + percpu_ref_put(cnt); +} + +/* + * avoid calling this func as possible. + */ +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev) +{ + long l; + + percpu_ref_switch_to_atomic_sync(cnt); + l = atomic_long_read(&cnt->count); + if (do_rev) + percpu_ref_switch_to_percpu(cnt); + + /* percpu_ref is initialized by 1 instead of 0 */ + return l - 1; +} +#endif + +#ifdef CONFIG_AUFS_DEBUG +#define AuLCntZero(val) do { \ + long l = val; \ + if (l) \ + AuDbg("%s = %ld\n", #val, l); \ +} while (0) +#else +#define AuLCntZero(val) do {} while (0) +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_LCNT_H__ */ diff --git a/fs/aufs/loop.c b/fs/aufs/loop.c new file mode 100644 index 0000000000000..cef7d96362c7e --- /dev/null +++ b/fs/aufs/loop.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * support for loopback block device as a branch + */ + +#include "aufs.h" + +/* added into drivers/block/loop.c */ +static struct file *(*backing_file_func)(struct super_block *sb); + +/* + * test if two lower dentries have overlapping branches. + */ +int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding) +{ + struct super_block *h_sb; + struct file *backing_file; + + if (unlikely(!backing_file_func)) { + /* don't load "loop" module here */ + backing_file_func = symbol_get(loop_backing_file); + if (unlikely(!backing_file_func)) + /* "loop" module is not loaded */ + return 0; + } + + h_sb = h_adding->d_sb; + backing_file = backing_file_func(h_sb); + if (!backing_file) + return 0; + + h_adding = backing_file->f_path.dentry; + /* + * h_adding can be local NFS. + * in this case aufs cannot detect the loop. + */ + if (unlikely(h_adding->d_sb == sb)) + return 1; + return !!au_test_subdir(h_adding, sb->s_root); +} + +/* true if a kernel thread named 'loop[0-9].*' accesses a file */ +int au_test_loopback_kthread(void) +{ + int ret; + struct task_struct *tsk = current; + char c, comm[sizeof(tsk->comm)]; + + ret = 0; + if (tsk->flags & PF_KTHREAD) { + get_task_comm(comm, tsk); + c = comm[4]; + ret = ('0' <= c && c <= '9' + && !strncmp(comm, "loop", 4)); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ + +#define au_warn_loopback_step 16 +static int au_warn_loopback_nelem = au_warn_loopback_step; +static unsigned long *au_warn_loopback_array; + +void au_warn_loopback(struct super_block *h_sb) +{ + int i, new_nelem; + unsigned long *a, magic; + static DEFINE_SPINLOCK(spin); + + magic = h_sb->s_magic; + spin_lock(&spin); + a = au_warn_loopback_array; + for (i = 0; i < au_warn_loopback_nelem && *a; i++) + if (a[i] == magic) { + spin_unlock(&spin); + return; + } + + /* h_sb is new to us, print it */ + if (i < au_warn_loopback_nelem) { + a[i] = magic; + goto pr; + } + + /* expand the array */ + new_nelem = au_warn_loopback_nelem + au_warn_loopback_step; + a = au_kzrealloc(au_warn_loopback_array, + au_warn_loopback_nelem * sizeof(unsigned long), + new_nelem * sizeof(unsigned long), GFP_ATOMIC, + /*may_shrink*/0); + if (a) { + au_warn_loopback_nelem = new_nelem; + au_warn_loopback_array = a; + a[i] = magic; + goto pr; + } + + spin_unlock(&spin); + AuWarn1("realloc failed, ignored\n"); + return; + +pr: + spin_unlock(&spin); + pr_warn("you may want to try another patch for loopback file " + "on %s(0x%lx) branch\n", au_sbtype(h_sb), magic); +} + +int au_loopback_init(void) +{ + int err; + struct super_block *sb __maybe_unused; + + BUILD_BUG_ON(sizeof(sb->s_magic) != sizeof(unsigned long)); + + err = 0; + au_warn_loopback_array = kcalloc(au_warn_loopback_step, + sizeof(unsigned long), GFP_NOFS); + if (unlikely(!au_warn_loopback_array)) + err = -ENOMEM; + + return err; +} + +void au_loopback_fin(void) +{ + if (backing_file_func) + symbol_put(loop_backing_file); + au_kfree_try_rcu(au_warn_loopback_array); +} diff --git a/fs/aufs/loop.h b/fs/aufs/loop.h new file mode 100644 index 0000000000000..7fc394e72d758 --- /dev/null +++ b/fs/aufs/loop.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * support for loopback mount as a branch + */ + +#ifndef __AUFS_LOOP_H__ +#define __AUFS_LOOP_H__ + +#ifdef __KERNEL__ + +struct dentry; +struct super_block; + +#ifdef CONFIG_AUFS_BDEV_LOOP +/* drivers/block/loop.c */ +struct file *loop_backing_file(struct super_block *sb); + +/* loop.c */ +int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding); +int au_test_loopback_kthread(void); +void au_warn_loopback(struct super_block *h_sb); + +int au_loopback_init(void); +void au_loopback_fin(void); +#else +AuStub(struct file *, loop_backing_file, return NULL, struct super_block *sb) + +AuStubInt0(au_test_loopback_overlap, struct super_block *sb, + struct dentry *h_adding) +AuStubInt0(au_test_loopback_kthread, void) +AuStubVoid(au_warn_loopback, struct super_block *h_sb) + +AuStubInt0(au_loopback_init, void) +AuStubVoid(au_loopback_fin, void) +#endif /* BLK_DEV_LOOP */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_LOOP_H__ */ diff --git a/fs/aufs/magic.mk b/fs/aufs/magic.mk new file mode 100644 index 0000000000000..7bc9eef3ffec5 --- /dev/null +++ b/fs/aufs/magic.mk @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0 + +# defined in ${srctree}/fs/fuse/inode.c +# tristate +ifdef CONFIG_FUSE_FS +ccflags-y += -DFUSE_SUPER_MAGIC=0x65735546 +endif + +# defined in ${srctree}/fs/xfs/xfs_sb.h +# tristate +ifdef CONFIG_XFS_FS +ccflags-y += -DXFS_SB_MAGIC=0x58465342 +endif + +# defined in ${srctree}/fs/configfs/mount.c +# tristate +ifdef CONFIG_CONFIGFS_FS +ccflags-y += -DCONFIGFS_MAGIC=0x62656570 +endif + +# defined in ${srctree}/fs/ubifs/ubifs.h +# tristate +ifdef CONFIG_UBIFS_FS +ccflags-y += -DUBIFS_SUPER_MAGIC=0x24051905 +endif + +# defined in ${srctree}/fs/hfsplus/hfsplus_raw.h +# tristate +ifdef CONFIG_HFSPLUS_FS +ccflags-y += -DHFSPLUS_SUPER_MAGIC=0x482b +endif diff --git a/fs/aufs/module.c b/fs/aufs/module.c new file mode 100644 index 0000000000000..295736adfc3c3 --- /dev/null +++ b/fs/aufs/module.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * module global variables and operations + */ + +#include +#include +#include "aufs.h" + +/* shrinkable realloc */ +void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink) +{ + size_t sz; + int diff; + + sz = 0; + diff = -1; + if (p) { +#if 0 /* unused */ + if (!new_sz) { + au_kfree_rcu(p); + p = NULL; + goto out; + } +#else + AuDebugOn(!new_sz); +#endif + sz = ksize(p); + diff = au_kmidx_sub(sz, new_sz); + } + if (sz && !diff) + goto out; + + if (sz < new_sz) + /* expand or SLOB */ + p = krealloc(p, new_sz, gfp); + else if (new_sz < sz && may_shrink) { + /* shrink */ + void *q; + + q = kmalloc(new_sz, gfp); + if (q) { + if (p) { + memcpy(q, p, new_sz); + au_kfree_try_rcu(p); + } + p = q; + } else + p = NULL; + } + +out: + return p; +} + +void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, + int may_shrink) +{ + p = au_krealloc(p, new_sz, gfp, may_shrink); + if (p && new_sz > nused) + memset(p + nused, 0, new_sz - nused); + return p; +} + +/* ---------------------------------------------------------------------- */ +/* + * aufs caches + */ +struct kmem_cache *au_cache[AuCache_Last]; + +static void au_cache_fin(void) +{ + int i; + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + + /* excluding AuCache_HNOTIFY */ + BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last); + for (i = 0; i < AuCache_HNOTIFY; i++) { + kmem_cache_destroy(au_cache[i]); + au_cache[i] = NULL; + } +} + +static int __init au_cache_init(void) +{ + au_cache[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once); + if (au_cache[AuCache_DINFO]) + /* SLAB_DESTROY_BY_RCU */ + au_cache[AuCache_ICNTNR] = AuCacheCtor(au_icntnr, + au_icntnr_init_once); + if (au_cache[AuCache_ICNTNR]) + au_cache[AuCache_FINFO] = AuCacheCtor(au_finfo, + au_fi_init_once); + if (au_cache[AuCache_FINFO]) + au_cache[AuCache_VDIR] = AuCache(au_vdir); + if (au_cache[AuCache_VDIR]) + au_cache[AuCache_DEHSTR] = AuCache(au_vdir_dehstr); + if (au_cache[AuCache_DEHSTR]) + return 0; + + au_cache_fin(); + return -ENOMEM; +} + +/* ---------------------------------------------------------------------- */ + +int au_dir_roflags; + +#ifdef CONFIG_AUFS_SBILIST +/* + * iterate_supers_type() doesn't protect us from + * remounting (branch management) + */ +struct hlist_bl_head au_sbilist; +#endif + +/* + * functions for module interface. + */ +MODULE_LICENSE("GPL"); +/* MODULE_LICENSE("GPL v2"); */ +MODULE_AUTHOR("Junjiro R. Okajima "); +MODULE_DESCRIPTION(AUFS_NAME + " -- Advanced multi layered unification filesystem"); +MODULE_VERSION(AUFS_VERSION); +MODULE_ALIAS_FS(AUFS_NAME); + +/* this module parameter has no meaning when SYSFS is disabled */ +int sysaufs_brs = 1; +MODULE_PARM_DESC(brs, "use /fs/aufs/si_*/brN"); +module_param_named(brs, sysaufs_brs, int, 0444); + +/* this module parameter has no meaning when USER_NS is disabled */ +bool au_userns; +MODULE_PARM_DESC(allow_userns, "allow unprivileged to mount under userns"); +module_param_named(allow_userns, au_userns, bool, 0444); + +/* ---------------------------------------------------------------------- */ + +static char au_esc_chars[0x20 + 3]; /* 0x01-0x20, backslash, del, and NULL */ + +int au_seq_path(struct seq_file *seq, struct path *path) +{ + int err; + + err = seq_path(seq, path, au_esc_chars); + if (err >= 0) + err = 0; + else + err = -ENOMEM; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int __init aufs_init(void) +{ + int err, i; + char *p; + + p = au_esc_chars; + for (i = 1; i <= ' '; i++) + *p++ = i; + *p++ = '\\'; + *p++ = '\x7f'; + *p = 0; + + au_dir_roflags = au_file_roflags(O_DIRECTORY | O_LARGEFILE); + + memcpy(aufs_iop_nogetattr, aufs_iop, sizeof(aufs_iop)); + for (i = 0; i < AuIop_Last; i++) + aufs_iop_nogetattr[i].getattr = NULL; + + memset(au_cache, 0, sizeof(au_cache)); /* including hnotify */ + + au_sbilist_init(); + sysaufs_brs_init(); + au_debug_init(); + au_dy_init(); + err = sysaufs_init(); + if (unlikely(err)) + goto out; + err = dbgaufs_init(); + if (unlikely(err)) + goto out_sysaufs; + err = au_procfs_init(); + if (unlikely(err)) + goto out_dbgaufs; + err = au_wkq_init(); + if (unlikely(err)) + goto out_procfs; + err = au_loopback_init(); + if (unlikely(err)) + goto out_wkq; + err = au_hnotify_init(); + if (unlikely(err)) + goto out_loopback; + err = au_sysrq_init(); + if (unlikely(err)) + goto out_hin; + err = au_cache_init(); + if (unlikely(err)) + goto out_sysrq; + + aufs_fs_type.fs_flags |= au_userns ? FS_USERNS_MOUNT : 0; + err = register_filesystem(&aufs_fs_type); + if (unlikely(err)) + goto out_cache; + + /* since we define pr_fmt, call printk directly */ + printk(KERN_INFO AUFS_NAME " " AUFS_VERSION "\n"); + goto out; /* success */ + +out_cache: + au_cache_fin(); +out_sysrq: + au_sysrq_fin(); +out_hin: + au_hnotify_fin(); +out_loopback: + au_loopback_fin(); +out_wkq: + au_wkq_fin(); +out_procfs: + au_procfs_fin(); +out_dbgaufs: + dbgaufs_fin(); +out_sysaufs: + sysaufs_fin(); + au_dy_fin(); +out: + return err; +} + +static void __exit aufs_exit(void) +{ + unregister_filesystem(&aufs_fs_type); + au_cache_fin(); + au_sysrq_fin(); + au_hnotify_fin(); + au_loopback_fin(); + au_wkq_fin(); + au_procfs_fin(); + dbgaufs_fin(); + sysaufs_fin(); + au_dy_fin(); +} + +module_init(aufs_init); +module_exit(aufs_exit); diff --git a/fs/aufs/module.h b/fs/aufs/module.h new file mode 100644 index 0000000000000..3036754b90d00 --- /dev/null +++ b/fs/aufs/module.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * module initialization and module-global + */ + +#ifndef __AUFS_MODULE_H__ +#define __AUFS_MODULE_H__ + +#ifdef __KERNEL__ + +#include +#include "debug.h" +#include "dentry.h" +#include "dir.h" +#include "file.h" +#include "inode.h" + +struct path; +struct seq_file; + +/* module parameters */ +extern int sysaufs_brs; +extern bool au_userns; + +/* ---------------------------------------------------------------------- */ + +extern int au_dir_roflags; + +void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink); +void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, + int may_shrink); + +/* + * Comparing the size of the object with sizeof(struct rcu_head) + * case 1: object is always larger + * --> au_kfree_rcu() or au_kfree_do_rcu() + * case 2: object is always smaller + * --> au_kfree_small() + * case 3: object can be any size + * --> au_kfree_try_rcu() + */ + +static inline void au_kfree_do_rcu(const void *p) +{ + struct { + struct rcu_head rcu; + } *a = (void *)p; + + kfree_rcu(a, rcu); +} + +#define au_kfree_rcu(_p) do { \ + typeof(_p) p = (_p); \ + BUILD_BUG_ON(sizeof(*p) < sizeof(struct rcu_head)); \ + if (p) \ + au_kfree_do_rcu(p); \ + } while (0) + +#define au_kfree_do_sz_test(sz) (sz >= sizeof(struct rcu_head)) +#define au_kfree_sz_test(p) (p && au_kfree_do_sz_test(ksize(p))) + +static inline void au_kfree_try_rcu(const void *p) +{ + if (!p) + return; + if (au_kfree_sz_test(p)) + au_kfree_do_rcu(p); + else + kfree(p); +} + +static inline void au_kfree_small(const void *p) +{ + if (!p) + return; + AuDebugOn(au_kfree_sz_test(p)); + kfree(p); +} + +static inline int au_kmidx_sub(size_t sz, size_t new_sz) +{ +#ifndef CONFIG_SLOB + return kmalloc_index(sz) - kmalloc_index(new_sz); +#else + return -1; /* SLOB is untested */ +#endif +} + +int au_seq_path(struct seq_file *seq, struct path *path); + +#ifdef CONFIG_PROC_FS +/* procfs.c */ +int __init au_procfs_init(void); +void au_procfs_fin(void); +#else +AuStubInt0(au_procfs_init, void); +AuStubVoid(au_procfs_fin, void); +#endif + +/* ---------------------------------------------------------------------- */ + +/* kmem cache */ +enum { + AuCache_DINFO, + AuCache_ICNTNR, + AuCache_FINFO, + AuCache_VDIR, + AuCache_DEHSTR, + AuCache_HNOTIFY, /* must be last */ + AuCache_Last +}; + +extern struct kmem_cache *au_cache[AuCache_Last]; + +#define AuCacheFlags (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD) +#define AuCache(type) KMEM_CACHE(type, AuCacheFlags) +#define AuCacheCtor(type, ctor) \ + kmem_cache_create(#type, sizeof(struct type), \ + __alignof__(struct type), AuCacheFlags, ctor) + +#define AuCacheFuncs(name, index) \ + static inline struct au_##name *au_cache_alloc_##name(void) \ + { return kmem_cache_alloc(au_cache[AuCache_##index], GFP_NOFS); } \ + static inline void au_cache_free_##name##_norcu(struct au_##name *p) \ + { kmem_cache_free(au_cache[AuCache_##index], p); } \ + \ + static inline void au_cache_free_##name##_rcu_cb(struct rcu_head *rcu) \ + { void *p = rcu; \ + p -= offsetof(struct au_##name, rcu); \ + kmem_cache_free(au_cache[AuCache_##index], p); } \ + static inline void au_cache_free_##name##_rcu(struct au_##name *p) \ + { BUILD_BUG_ON(sizeof(struct au_##name) < sizeof(struct rcu_head)); \ + call_rcu(&p->rcu, au_cache_free_##name##_rcu_cb); } \ + \ + static inline void au_cache_free_##name(struct au_##name *p) \ + { /* au_cache_free_##name##_norcu(p); */ \ + au_cache_free_##name##_rcu(p); } + +AuCacheFuncs(dinfo, DINFO); +AuCacheFuncs(icntnr, ICNTNR); +AuCacheFuncs(finfo, FINFO); +AuCacheFuncs(vdir, VDIR); +AuCacheFuncs(vdir_dehstr, DEHSTR); +#ifdef CONFIG_AUFS_HNOTIFY +AuCacheFuncs(hnotify, HNOTIFY); +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_MODULE_H__ */ diff --git a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c new file mode 100644 index 0000000000000..59ce2d700ec9c --- /dev/null +++ b/fs/aufs/mvdown.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * move-down, opposite of copy-up + */ + +#include "aufs.h" + +struct au_mvd_args { + struct { + struct super_block *h_sb; + struct dentry *h_parent; + struct au_hinode *hdir; + struct inode *h_dir, *h_inode; + struct au_pin pin; + } info[AUFS_MVDOWN_NARRAY]; + + struct aufs_mvdown mvdown; + struct dentry *dentry, *parent; + struct inode *inode, *dir; + struct super_block *sb; + aufs_bindex_t bopq, bwh, bfound; + unsigned char rename_lock; +}; + +#define mvd_errno mvdown.au_errno +#define mvd_bsrc mvdown.stbr[AUFS_MVDOWN_UPPER].bindex +#define mvd_src_brid mvdown.stbr[AUFS_MVDOWN_UPPER].brid +#define mvd_bdst mvdown.stbr[AUFS_MVDOWN_LOWER].bindex +#define mvd_dst_brid mvdown.stbr[AUFS_MVDOWN_LOWER].brid + +#define mvd_h_src_sb info[AUFS_MVDOWN_UPPER].h_sb +#define mvd_h_src_parent info[AUFS_MVDOWN_UPPER].h_parent +#define mvd_hdir_src info[AUFS_MVDOWN_UPPER].hdir +#define mvd_h_src_dir info[AUFS_MVDOWN_UPPER].h_dir +#define mvd_h_src_inode info[AUFS_MVDOWN_UPPER].h_inode +#define mvd_pin_src info[AUFS_MVDOWN_UPPER].pin + +#define mvd_h_dst_sb info[AUFS_MVDOWN_LOWER].h_sb +#define mvd_h_dst_parent info[AUFS_MVDOWN_LOWER].h_parent +#define mvd_hdir_dst info[AUFS_MVDOWN_LOWER].hdir +#define mvd_h_dst_dir info[AUFS_MVDOWN_LOWER].h_dir +#define mvd_h_dst_inode info[AUFS_MVDOWN_LOWER].h_inode +#define mvd_pin_dst info[AUFS_MVDOWN_LOWER].pin + +#define AU_MVD_PR(flag, ...) do { \ + if (flag) \ + pr_err(__VA_ARGS__); \ + } while (0) + +static int find_lower_writable(struct au_mvd_args *a) +{ + struct super_block *sb; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + sb = a->sb; + bindex = a->mvd_bsrc; + bbot = au_sbbot(sb); + if (a->mvdown.flags & AUFS_MVDOWN_FHSM_LOWER) + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm) + && !sb_rdonly(au_br_sb(br))) + return bindex; + } + else if (!(a->mvdown.flags & AUFS_MVDOWN_ROLOWER)) + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_rdonly(br)) + return bindex; + } + else + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!sb_rdonly(au_br_sb(br))) { + if (au_br_rdonly(br)) + a->mvdown.flags + |= AUFS_MVDOWN_ROLOWER_R; + return bindex; + } + } + + return -1; +} + +/* make the parent dir on bdst */ +static int au_do_mkdir(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = 0; + a->mvd_hdir_src = au_hi(a->dir, a->mvd_bsrc); + a->mvd_hdir_dst = au_hi(a->dir, a->mvd_bdst); + a->mvd_h_src_parent = au_h_dptr(a->parent, a->mvd_bsrc); + a->mvd_h_dst_parent = NULL; + if (au_dbbot(a->parent) >= a->mvd_bdst) + a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst); + if (!a->mvd_h_dst_parent) { + err = au_cpdown_dirs(a->dentry, a->mvd_bdst); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "cpdown_dirs failed\n"); + goto out; + } + a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst); + } + +out: + AuTraceErr(err); + return err; +} + +/* lock them all */ +static int au_do_lock(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct dentry *h_trap; + + a->mvd_h_src_sb = au_sbr_sb(a->sb, a->mvd_bsrc); + a->mvd_h_dst_sb = au_sbr_sb(a->sb, a->mvd_bdst); + err = au_pin(&a->mvd_pin_dst, a->dentry, a->mvd_bdst, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + AuTraceErr(err); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_dst failed\n"); + goto out; + } + + if (a->mvd_h_src_sb != a->mvd_h_dst_sb) { + a->rename_lock = 0; + au_pin_init(&a->mvd_pin_src, a->dentry, a->mvd_bsrc, + AuLsc_DI_PARENT, AuLsc_I_PARENT3, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + err = au_do_pin(&a->mvd_pin_src); + AuTraceErr(err); + a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_src failed\n"); + goto out_dst; + } + goto out; /* success */ + } + + a->rename_lock = 1; + au_pin_hdir_unlock(&a->mvd_pin_dst); + err = au_pin(&a->mvd_pin_src, a->dentry, a->mvd_bsrc, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + AuTraceErr(err); + a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_src failed\n"); + au_pin_hdir_lock(&a->mvd_pin_dst); + goto out_dst; + } + au_pin_hdir_unlock(&a->mvd_pin_src); + h_trap = vfsub_lock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + if (h_trap) { + err = (h_trap != a->mvd_h_src_parent); + if (err) + err = (h_trap != a->mvd_h_dst_parent); + } + BUG_ON(err); /* it should never happen */ + if (unlikely(a->mvd_h_src_dir != au_pinned_h_dir(&a->mvd_pin_src))) { + err = -EBUSY; + AuTraceErr(err); + vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + au_pin_hdir_lock(&a->mvd_pin_src); + au_unpin(&a->mvd_pin_src); + au_pin_hdir_lock(&a->mvd_pin_dst); + goto out_dst; + } + goto out; /* success */ + +out_dst: + au_unpin(&a->mvd_pin_dst); +out: + AuTraceErr(err); + return err; +} + +static void au_do_unlock(const unsigned char dmsg, struct au_mvd_args *a) +{ + if (!a->rename_lock) + au_unpin(&a->mvd_pin_src); + else { + vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + au_pin_hdir_lock(&a->mvd_pin_src); + au_unpin(&a->mvd_pin_src); + au_pin_hdir_lock(&a->mvd_pin_dst); + } + au_unpin(&a->mvd_pin_dst); +} + +/* copy-down the file */ +static int au_do_cpdown(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_cp_generic cpg = { + .dentry = a->dentry, + .bdst = a->mvd_bdst, + .bsrc = a->mvd_bsrc, + .len = -1, + .pin = &a->mvd_pin_dst, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + + AuDbg("b%d, b%d\n", cpg.bsrc, cpg.bdst); + if (a->mvdown.flags & AUFS_MVDOWN_OWLOWER) + au_fset_cpup(cpg.flags, OVERWRITE); + if (a->mvdown.flags & AUFS_MVDOWN_ROLOWER) + au_fset_cpup(cpg.flags, RWDST); + err = au_sio_cpdown_simple(&cpg); + if (unlikely(err)) + AU_MVD_PR(dmsg, "cpdown failed\n"); + + AuTraceErr(err); + return err; +} + +/* + * unlink the whiteout on bdst if exist which may be created by UDBA while we + * were sleeping + */ +static int au_do_unlink_wh(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct path h_path; + struct au_branch *br; + struct inode *delegated; + + br = au_sbr(a->sb, a->mvd_bdst); + h_path.dentry = au_wh_lkup(a->mvd_h_dst_parent, &a->dentry->d_name, br); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) { + AU_MVD_PR(dmsg, "wh_lkup failed\n"); + goto out; + } + + err = 0; + if (d_is_positive(h_path.dentry)) { + h_path.mnt = au_br_mnt(br); + delegated = NULL; + err = vfsub_unlink(d_inode(a->mvd_h_dst_parent), &h_path, + &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) + AU_MVD_PR(dmsg, "wh_unlink failed\n"); + } + dput(h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +/* + * unlink the topmost h_dentry + */ +static int au_do_unlink(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct path h_path; + struct inode *delegated; + + h_path.mnt = au_sbr_mnt(a->sb, a->mvd_bsrc); + h_path.dentry = au_h_dptr(a->dentry, a->mvd_bsrc); + delegated = NULL; + err = vfsub_unlink(a->mvd_h_src_dir, &h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) + AU_MVD_PR(dmsg, "unlink failed\n"); + + AuTraceErr(err); + return err; +} + +/* Since mvdown succeeded, we ignore an error of this function */ +static void au_do_stfs(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_branch *br; + + a->mvdown.flags |= AUFS_MVDOWN_STFS_FAILED; + br = au_sbr(a->sb, a->mvd_bsrc); + err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_UPPER].stfs); + if (!err) { + br = au_sbr(a->sb, a->mvd_bdst); + a->mvdown.stbr[AUFS_MVDOWN_LOWER].brid = br->br_id; + err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_LOWER].stfs); + } + if (!err) + a->mvdown.flags &= ~AUFS_MVDOWN_STFS_FAILED; + else + AU_MVD_PR(dmsg, "statfs failed (%d), ignored\n", err); +} + +/* + * copy-down the file and unlink the bsrc file. + * - unlink the bdst whout if exist + * - copy-down the file (with whtmp name and rename) + * - unlink the bsrc file + */ +static int au_do_mvdown(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = au_do_mkdir(dmsg, a); + if (!err) + err = au_do_lock(dmsg, a); + if (unlikely(err)) + goto out; + + /* + * do not revert the activities we made on bdst since they should be + * harmless in aufs. + */ + + err = au_do_cpdown(dmsg, a); + if (!err) + err = au_do_unlink_wh(dmsg, a); + if (!err && !(a->mvdown.flags & AUFS_MVDOWN_KUPPER)) + err = au_do_unlink(dmsg, a); + if (unlikely(err)) + goto out_unlock; + + AuDbg("%pd2, 0x%x, %d --> %d\n", + a->dentry, a->mvdown.flags, a->mvd_bsrc, a->mvd_bdst); + if (find_lower_writable(a) < 0) + a->mvdown.flags |= AUFS_MVDOWN_BOTTOM; + + if (a->mvdown.flags & AUFS_MVDOWN_STFS) + au_do_stfs(dmsg, a); + + /* maintain internal array */ + if (!(a->mvdown.flags & AUFS_MVDOWN_KUPPER)) { + au_set_h_dptr(a->dentry, a->mvd_bsrc, NULL); + au_set_dbtop(a->dentry, a->mvd_bdst); + au_set_h_iptr(a->inode, a->mvd_bsrc, NULL, /*flags*/0); + au_set_ibtop(a->inode, a->mvd_bdst); + } else { + /* hide the lower */ + au_set_h_dptr(a->dentry, a->mvd_bdst, NULL); + au_set_dbbot(a->dentry, a->mvd_bsrc); + au_set_h_iptr(a->inode, a->mvd_bdst, NULL, /*flags*/0); + au_set_ibbot(a->inode, a->mvd_bsrc); + } + if (au_dbbot(a->dentry) < a->mvd_bdst) + au_set_dbbot(a->dentry, a->mvd_bdst); + if (au_ibbot(a->inode) < a->mvd_bdst) + au_set_ibbot(a->inode, a->mvd_bdst); + +out_unlock: + au_do_unlock(dmsg, a); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* make sure the file is idle */ +static int au_mvd_args_busy(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err, plinked; + + err = 0; + plinked = !!au_opt_test(au_mntflags(a->sb), PLINK); + if (au_dbtop(a->dentry) == a->mvd_bsrc + && au_dcount(a->dentry) == 1 + && atomic_read(&a->inode->i_count) == 1 + /* && a->mvd_h_src_inode->i_nlink == 1 */ + && (!plinked || !au_plink_test(a->inode)) + && a->inode->i_nlink == 1) + goto out; + + err = -EBUSY; + AU_MVD_PR(dmsg, + "b%d, d{b%d, c%d?}, i{c%d?, l%u}, hi{l%u}, p{%d, %d}\n", + a->mvd_bsrc, au_dbtop(a->dentry), au_dcount(a->dentry), + atomic_read(&a->inode->i_count), a->inode->i_nlink, + a->mvd_h_src_inode->i_nlink, + plinked, plinked ? au_plink_test(a->inode) : 0); + +out: + AuTraceErr(err); + return err; +} + +/* make sure the parent dir is fine */ +static int au_mvd_args_parent(const unsigned char dmsg, + struct au_mvd_args *a) +{ + int err; + aufs_bindex_t bindex; + + err = 0; + if (unlikely(au_alive_dir(a->parent))) { + err = -ENOENT; + AU_MVD_PR(dmsg, "parent dir is dead\n"); + goto out; + } + + a->bopq = au_dbdiropq(a->parent); + bindex = au_wbr_nonopq(a->dentry, a->mvd_bdst); + AuDbg("b%d\n", bindex); + if (unlikely((bindex >= 0 && bindex < a->mvd_bdst) + || (a->bopq != -1 && a->bopq < a->mvd_bdst))) { + err = -EINVAL; + a->mvd_errno = EAU_MVDOWN_OPAQUE; + AU_MVD_PR(dmsg, "ancestor is opaque b%d, b%d\n", + a->bopq, a->mvd_bdst); + } + +out: + AuTraceErr(err); + return err; +} + +static int au_mvd_args_intermediate(const unsigned char dmsg, + struct au_mvd_args *a) +{ + int err; + struct au_dinfo *dinfo, *tmp; + + /* lookup the next lower positive entry */ + err = -ENOMEM; + tmp = au_di_alloc(a->sb, AuLsc_DI_TMP); + if (unlikely(!tmp)) + goto out; + + a->bfound = -1; + a->bwh = -1; + dinfo = au_di(a->dentry); + au_di_cp(tmp, dinfo); + au_di_swap(tmp, dinfo); + + /* returns the number of positive dentries */ + err = au_lkup_dentry(a->dentry, a->mvd_bsrc + 1, + /* AuLkup_IGNORE_PERM */ 0); + if (!err) + a->bwh = au_dbwh(a->dentry); + else if (err > 0) + a->bfound = au_dbtop(a->dentry); + + au_di_swap(tmp, dinfo); + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err < 0)) + AU_MVD_PR(dmsg, "failed look-up lower\n"); + + /* + * here, we have these cases. + * bfound == -1 + * no positive dentry under bsrc. there are more sub-cases. + * bwh < 0 + * there no whiteout, we can safely move-down. + * bwh <= bsrc + * impossible + * bsrc < bwh && bwh < bdst + * there is a whiteout on RO branch. cannot proceed. + * bwh == bdst + * there is a whiteout on the RW target branch. it should + * be removed. + * bdst < bwh + * there is a whiteout somewhere unrelated branch. + * -1 < bfound && bfound <= bsrc + * impossible. + * bfound < bdst + * found, but it is on RO branch between bsrc and bdst. cannot + * proceed. + * bfound == bdst + * found, replace it if AUFS_MVDOWN_FORCE is set. otherwise return + * error. + * bdst < bfound + * found, after we create the file on bdst, it will be hidden. + */ + + AuDebugOn(a->bfound == -1 + && a->bwh != -1 + && a->bwh <= a->mvd_bsrc); + AuDebugOn(-1 < a->bfound + && a->bfound <= a->mvd_bsrc); + + err = -EINVAL; + if (a->bfound == -1 + && a->mvd_bsrc < a->bwh + && a->bwh != -1 + && a->bwh < a->mvd_bdst) { + a->mvd_errno = EAU_MVDOWN_WHITEOUT; + AU_MVD_PR(dmsg, "bsrc %d, bdst %d, bfound %d, bwh %d\n", + a->mvd_bsrc, a->mvd_bdst, a->bfound, a->bwh); + goto out; + } else if (a->bfound != -1 && a->bfound < a->mvd_bdst) { + a->mvd_errno = EAU_MVDOWN_UPPER; + AU_MVD_PR(dmsg, "bdst %d, bfound %d\n", + a->mvd_bdst, a->bfound); + goto out; + } + + err = 0; /* success */ + +out: + AuTraceErr(err); + return err; +} + +static int au_mvd_args_exist(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = 0; + if (!(a->mvdown.flags & AUFS_MVDOWN_OWLOWER) + && a->bfound == a->mvd_bdst) + err = -EEXIST; + AuTraceErr(err); + return err; +} + +static int au_mvd_args(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_branch *br; + + err = -EISDIR; + if (unlikely(S_ISDIR(a->inode->i_mode))) + goto out; + + err = -EINVAL; + if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_UPPER)) + a->mvd_bsrc = au_ibtop(a->inode); + else { + a->mvd_bsrc = au_br_index(a->sb, a->mvd_src_brid); + if (unlikely(a->mvd_bsrc < 0 + || (a->mvd_bsrc < au_dbtop(a->dentry) + || au_dbbot(a->dentry) < a->mvd_bsrc + || !au_h_dptr(a->dentry, a->mvd_bsrc)) + || (a->mvd_bsrc < au_ibtop(a->inode) + || au_ibbot(a->inode) < a->mvd_bsrc + || !au_h_iptr(a->inode, a->mvd_bsrc)))) { + a->mvd_errno = EAU_MVDOWN_NOUPPER; + AU_MVD_PR(dmsg, "no upper\n"); + goto out; + } + } + if (unlikely(a->mvd_bsrc == au_sbbot(a->sb))) { + a->mvd_errno = EAU_MVDOWN_BOTTOM; + AU_MVD_PR(dmsg, "on the bottom\n"); + goto out; + } + a->mvd_h_src_inode = au_h_iptr(a->inode, a->mvd_bsrc); + br = au_sbr(a->sb, a->mvd_bsrc); + err = au_br_rdonly(br); + if (!(a->mvdown.flags & AUFS_MVDOWN_ROUPPER)) { + if (unlikely(err)) + goto out; + } else if (!(vfsub_native_ro(a->mvd_h_src_inode) + || IS_APPEND(a->mvd_h_src_inode))) { + if (err) + a->mvdown.flags |= AUFS_MVDOWN_ROUPPER_R; + /* go on */ + } else + goto out; + + err = -EINVAL; + if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_LOWER)) { + a->mvd_bdst = find_lower_writable(a); + if (unlikely(a->mvd_bdst < 0)) { + a->mvd_errno = EAU_MVDOWN_BOTTOM; + AU_MVD_PR(dmsg, "no writable lower branch\n"); + goto out; + } + } else { + a->mvd_bdst = au_br_index(a->sb, a->mvd_dst_brid); + if (unlikely(a->mvd_bdst < 0 + || au_sbbot(a->sb) < a->mvd_bdst)) { + a->mvd_errno = EAU_MVDOWN_NOLOWERBR; + AU_MVD_PR(dmsg, "no lower brid\n"); + goto out; + } + } + + err = au_mvd_args_busy(dmsg, a); + if (!err) + err = au_mvd_args_parent(dmsg, a); + if (!err) + err = au_mvd_args_intermediate(dmsg, a); + if (!err) + err = au_mvd_args_exist(dmsg, a); + if (!err) + AuDbg("b%d, b%d\n", a->mvd_bsrc, a->mvd_bdst); + +out: + AuTraceErr(err); + return err; +} + +int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *uarg) +{ + int err, e; + unsigned char dmsg; + struct au_mvd_args *args; + struct inode *inode; + + inode = d_inode(dentry); + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -ENOMEM; + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) + goto out; + + err = copy_from_user(&args->mvdown, uarg, sizeof(args->mvdown)); + if (!err) + err = !access_ok(VERIFY_WRITE, uarg, sizeof(*uarg)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out_free; + } + AuDbg("flags 0x%x\n", args->mvdown.flags); + args->mvdown.flags &= ~(AUFS_MVDOWN_ROLOWER_R | AUFS_MVDOWN_ROUPPER_R); + args->mvdown.au_errno = 0; + args->dentry = dentry; + args->inode = inode; + args->sb = dentry->d_sb; + + err = -ENOENT; + dmsg = !!(args->mvdown.flags & AUFS_MVDOWN_DMSG); + args->parent = dget_parent(dentry); + args->dir = d_inode(args->parent); + inode_lock_nested(args->dir, I_MUTEX_PARENT); + dput(args->parent); + if (unlikely(args->parent != dentry->d_parent)) { + AU_MVD_PR(dmsg, "parent dir is moved\n"); + goto out_dir; + } + + inode_lock_nested(inode, I_MUTEX_CHILD); + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_NOPLMW); + if (unlikely(err)) + goto out_inode; + + di_write_lock_parent(args->parent); + err = au_mvd_args(dmsg, args); + if (unlikely(err)) + goto out_parent; + + err = au_do_mvdown(dmsg, args); + if (unlikely(err)) + goto out_parent; + + au_cpup_attr_timesizes(args->dir); + au_cpup_attr_timesizes(inode); + if (!(args->mvdown.flags & AUFS_MVDOWN_KUPPER)) + au_cpup_igen(inode, au_h_iptr(inode, args->mvd_bdst)); + /* au_digen_dec(dentry); */ + +out_parent: + di_write_unlock(args->parent); + aufs_read_unlock(dentry, AuLock_DW); +out_inode: + inode_unlock(inode); +out_dir: + inode_unlock(args->dir); +out_free: + e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown)); + if (unlikely(e)) + err = -EFAULT; + au_kfree_rcu(args); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/opts.c b/fs/aufs/opts.c new file mode 100644 index 0000000000000..1663e790137a5 --- /dev/null +++ b/fs/aufs/opts.c @@ -0,0 +1,1880 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount options/flags + */ + +#include +#include /* a distribution requires */ +#include +#include "aufs.h" + +/* ---------------------------------------------------------------------- */ + +enum { + Opt_br, + Opt_add, Opt_del, Opt_mod, Opt_append, Opt_prepend, + Opt_idel, Opt_imod, + Opt_dirwh, Opt_rdcache, Opt_rdblk, Opt_rdhash, + Opt_rdblk_def, Opt_rdhash_def, + Opt_xino, Opt_noxino, + Opt_trunc_xino, Opt_trunc_xino_v, Opt_notrunc_xino, + Opt_trunc_xino_path, Opt_itrunc_xino, + Opt_trunc_xib, Opt_notrunc_xib, + Opt_shwh, Opt_noshwh, + Opt_plink, Opt_noplink, Opt_list_plink, + Opt_udba, + Opt_dio, Opt_nodio, + Opt_diropq_a, Opt_diropq_w, + Opt_warn_perm, Opt_nowarn_perm, + Opt_wbr_copyup, Opt_wbr_create, + Opt_fhsm_sec, + Opt_verbose, Opt_noverbose, + Opt_sum, Opt_nosum, Opt_wsum, + Opt_dirperm1, Opt_nodirperm1, + Opt_dirren, Opt_nodirren, + Opt_acl, Opt_noacl, + Opt_tail, Opt_ignore, Opt_ignore_silent, Opt_err +}; + +static match_table_t options = { + {Opt_br, "br=%s"}, + {Opt_br, "br:%s"}, + + {Opt_add, "add=%d:%s"}, + {Opt_add, "add:%d:%s"}, + {Opt_add, "ins=%d:%s"}, + {Opt_add, "ins:%d:%s"}, + {Opt_append, "append=%s"}, + {Opt_append, "append:%s"}, + {Opt_prepend, "prepend=%s"}, + {Opt_prepend, "prepend:%s"}, + + {Opt_del, "del=%s"}, + {Opt_del, "del:%s"}, + /* {Opt_idel, "idel:%d"}, */ + {Opt_mod, "mod=%s"}, + {Opt_mod, "mod:%s"}, + /* {Opt_imod, "imod:%d:%s"}, */ + + {Opt_dirwh, "dirwh=%d"}, + + {Opt_xino, "xino=%s"}, + {Opt_noxino, "noxino"}, + {Opt_trunc_xino, "trunc_xino"}, + {Opt_trunc_xino_v, "trunc_xino_v=%d:%d"}, + {Opt_notrunc_xino, "notrunc_xino"}, + {Opt_trunc_xino_path, "trunc_xino=%s"}, + {Opt_itrunc_xino, "itrunc_xino=%d"}, + /* {Opt_zxino, "zxino=%s"}, */ + {Opt_trunc_xib, "trunc_xib"}, + {Opt_notrunc_xib, "notrunc_xib"}, + +#ifdef CONFIG_PROC_FS + {Opt_plink, "plink"}, +#else + {Opt_ignore_silent, "plink"}, +#endif + + {Opt_noplink, "noplink"}, + +#ifdef CONFIG_AUFS_DEBUG + {Opt_list_plink, "list_plink"}, +#endif + + {Opt_udba, "udba=%s"}, + + {Opt_dio, "dio"}, + {Opt_nodio, "nodio"}, + +#ifdef CONFIG_AUFS_DIRREN + {Opt_dirren, "dirren"}, + {Opt_nodirren, "nodirren"}, +#else + {Opt_ignore, "dirren"}, + {Opt_ignore_silent, "nodirren"}, +#endif + +#ifdef CONFIG_AUFS_FHSM + {Opt_fhsm_sec, "fhsm_sec=%d"}, +#else + {Opt_ignore, "fhsm_sec=%d"}, +#endif + + {Opt_diropq_a, "diropq=always"}, + {Opt_diropq_a, "diropq=a"}, + {Opt_diropq_w, "diropq=whiteouted"}, + {Opt_diropq_w, "diropq=w"}, + + {Opt_warn_perm, "warn_perm"}, + {Opt_nowarn_perm, "nowarn_perm"}, + + /* keep them temporary */ + {Opt_ignore_silent, "nodlgt"}, + {Opt_ignore, "clean_plink"}, + +#ifdef CONFIG_AUFS_SHWH + {Opt_shwh, "shwh"}, +#endif + {Opt_noshwh, "noshwh"}, + + {Opt_dirperm1, "dirperm1"}, + {Opt_nodirperm1, "nodirperm1"}, + + {Opt_verbose, "verbose"}, + {Opt_verbose, "v"}, + {Opt_noverbose, "noverbose"}, + {Opt_noverbose, "quiet"}, + {Opt_noverbose, "q"}, + {Opt_noverbose, "silent"}, + + {Opt_sum, "sum"}, + {Opt_nosum, "nosum"}, + {Opt_wsum, "wsum"}, + + {Opt_rdcache, "rdcache=%d"}, + {Opt_rdblk, "rdblk=%d"}, + {Opt_rdblk_def, "rdblk=def"}, + {Opt_rdhash, "rdhash=%d"}, + {Opt_rdhash_def, "rdhash=def"}, + + {Opt_wbr_create, "create=%s"}, + {Opt_wbr_create, "create_policy=%s"}, + {Opt_wbr_copyup, "cpup=%s"}, + {Opt_wbr_copyup, "copyup=%s"}, + {Opt_wbr_copyup, "copyup_policy=%s"}, + + /* generic VFS flag */ +#ifdef CONFIG_FS_POSIX_ACL + {Opt_acl, "acl"}, + {Opt_noacl, "noacl"}, +#else + {Opt_ignore, "acl"}, + {Opt_ignore_silent, "noacl"}, +#endif + + /* internal use for the scripts */ + {Opt_ignore_silent, "si=%s"}, + + {Opt_br, "dirs=%s"}, + {Opt_ignore, "debug=%d"}, + {Opt_ignore, "delete=whiteout"}, + {Opt_ignore, "delete=all"}, + {Opt_ignore, "imap=%s"}, + + /* temporary workaround, due to old mount(8)? */ + {Opt_ignore_silent, "relatime"}, + + {Opt_err, NULL} +}; + +/* ---------------------------------------------------------------------- */ + +static const char *au_parser_pattern(int val, match_table_t tbl) +{ + struct match_token *p; + + p = tbl; + while (p->pattern) { + if (p->token == val) + return p->pattern; + p++; + } + BUG(); + return "??"; +} + +static const char *au_optstr(int *val, match_table_t tbl) +{ + struct match_token *p; + int v; + + v = *val; + if (!v) + goto out; + p = tbl; + while (p->pattern) { + if (p->token + && (v & p->token) == p->token) { + *val &= ~p->token; + return p->pattern; + } + p++; + } + +out: + return NULL; +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t brperm = { + {AuBrPerm_RO, AUFS_BRPERM_RO}, + {AuBrPerm_RR, AUFS_BRPERM_RR}, + {AuBrPerm_RW, AUFS_BRPERM_RW}, + {0, NULL} +}; + +static match_table_t brattr = { + /* general */ + {AuBrAttr_COO_REG, AUFS_BRATTR_COO_REG}, + {AuBrAttr_COO_ALL, AUFS_BRATTR_COO_ALL}, + /* 'unpin' attrib is meaningless since linux-3.18-rc1 */ + {AuBrAttr_UNPIN, AUFS_BRATTR_UNPIN}, +#ifdef CONFIG_AUFS_FHSM + {AuBrAttr_FHSM, AUFS_BRATTR_FHSM}, +#endif +#ifdef CONFIG_AUFS_XATTR + {AuBrAttr_ICEX, AUFS_BRATTR_ICEX}, + {AuBrAttr_ICEX_SEC, AUFS_BRATTR_ICEX_SEC}, + {AuBrAttr_ICEX_SYS, AUFS_BRATTR_ICEX_SYS}, + {AuBrAttr_ICEX_TR, AUFS_BRATTR_ICEX_TR}, + {AuBrAttr_ICEX_USR, AUFS_BRATTR_ICEX_USR}, + {AuBrAttr_ICEX_OTH, AUFS_BRATTR_ICEX_OTH}, +#endif + + /* ro/rr branch */ + {AuBrRAttr_WH, AUFS_BRRATTR_WH}, + + /* rw branch */ + {AuBrWAttr_MOO, AUFS_BRWATTR_MOO}, + {AuBrWAttr_NoLinkWH, AUFS_BRWATTR_NLWH}, + + {0, NULL} +}; + +static int br_attr_val(char *str, match_table_t table, substring_t args[]) +{ + int attr, v; + char *p; + + attr = 0; + do { + p = strchr(str, '+'); + if (p) + *p = 0; + v = match_token(str, table, args); + if (v) { + if (v & AuBrAttr_CMOO_Mask) + attr &= ~AuBrAttr_CMOO_Mask; + attr |= v; + } else { + if (p) + *p = '+'; + pr_warn("ignored branch attribute %s\n", str); + break; + } + if (p) + str = p + 1; + } while (p); + + return attr; +} + +static int au_do_optstr_br_attr(au_br_perm_str_t *str, int perm) +{ + int sz; + const char *p; + char *q; + + q = str->a; + *q = 0; + p = au_optstr(&perm, brattr); + if (p) { + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + } else + goto out; + + do { + p = au_optstr(&perm, brattr); + if (p) { + *q++ = '+'; + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + } + } while (p); + +out: + return q - str->a; +} + +static int noinline_for_stack br_perm_val(char *perm) +{ + int val, bad, sz; + char *p; + substring_t args[MAX_OPT_ARGS]; + au_br_perm_str_t attr; + + p = strchr(perm, '+'); + if (p) + *p = 0; + val = match_token(perm, brperm, args); + if (!val) { + if (p) + *p = '+'; + pr_warn("ignored branch permission %s\n", perm); + val = AuBrPerm_RO; + goto out; + } + if (!p) + goto out; + + val |= br_attr_val(p + 1, brattr, args); + + bad = 0; + switch (val & AuBrPerm_Mask) { + case AuBrPerm_RO: + case AuBrPerm_RR: + bad = val & AuBrWAttr_Mask; + val &= ~AuBrWAttr_Mask; + break; + case AuBrPerm_RW: + bad = val & AuBrRAttr_Mask; + val &= ~AuBrRAttr_Mask; + break; + } + + /* + * 'unpin' attrib becomes meaningless since linux-3.18-rc1, but aufs + * does not treat it as an error, just warning. + * this is a tiny guard for the user operation. + */ + if (val & AuBrAttr_UNPIN) { + bad |= AuBrAttr_UNPIN; + val &= ~AuBrAttr_UNPIN; + } + + if (unlikely(bad)) { + sz = au_do_optstr_br_attr(&attr, bad); + AuDebugOn(!sz); + pr_warn("ignored branch attribute %s\n", attr.a); + } + +out: + return val; +} + +void au_optstr_br_perm(au_br_perm_str_t *str, int perm) +{ + au_br_perm_str_t attr; + const char *p; + char *q; + int sz; + + q = str->a; + p = au_optstr(&perm, brperm); + AuDebugOn(!p || !*p); + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + + sz = au_do_optstr_br_attr(&attr, perm); + if (sz) { + *q++ = '+'; + memcpy(q, attr.a, sz + 1); + } + + AuDebugOn(strlen(str->a) >= sizeof(str->a)); +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t udbalevel = { + {AuOpt_UDBA_REVAL, "reval"}, + {AuOpt_UDBA_NONE, "none"}, +#ifdef CONFIG_AUFS_HNOTIFY + {AuOpt_UDBA_HNOTIFY, "notify"}, /* abstraction */ +#ifdef CONFIG_AUFS_HFSNOTIFY + {AuOpt_UDBA_HNOTIFY, "fsnotify"}, +#endif +#endif + {-1, NULL} +}; + +static int noinline_for_stack udba_val(char *str) +{ + substring_t args[MAX_OPT_ARGS]; + + return match_token(str, udbalevel, args); +} + +const char *au_optstr_udba(int udba) +{ + return au_parser_pattern(udba, udbalevel); +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t au_wbr_create_policy = { + {AuWbrCreate_TDP, "tdp"}, + {AuWbrCreate_TDP, "top-down-parent"}, + {AuWbrCreate_RR, "rr"}, + {AuWbrCreate_RR, "round-robin"}, + {AuWbrCreate_MFS, "mfs"}, + {AuWbrCreate_MFS, "most-free-space"}, + {AuWbrCreate_MFSV, "mfs:%d"}, + {AuWbrCreate_MFSV, "most-free-space:%d"}, + + /* top-down regardless the parent, and then mfs */ + {AuWbrCreate_TDMFS, "tdmfs:%d"}, + {AuWbrCreate_TDMFSV, "tdmfs:%d:%d"}, + + {AuWbrCreate_MFSRR, "mfsrr:%d"}, + {AuWbrCreate_MFSRRV, "mfsrr:%d:%d"}, + {AuWbrCreate_PMFS, "pmfs"}, + {AuWbrCreate_PMFSV, "pmfs:%d"}, + {AuWbrCreate_PMFSRR, "pmfsrr:%d"}, + {AuWbrCreate_PMFSRRV, "pmfsrr:%d:%d"}, + + {-1, NULL} +}; + +static int au_wbr_mfs_wmark(substring_t *arg, char *str, + struct au_opt_wbr_create *create) +{ + int err; + unsigned long long ull; + + err = 0; + if (!match_u64(arg, &ull)) + create->mfsrr_watermark = ull; + else { + pr_err("bad integer in %s\n", str); + err = -EINVAL; + } + + return err; +} + +static int au_wbr_mfs_sec(substring_t *arg, char *str, + struct au_opt_wbr_create *create) +{ + int n, err; + + err = 0; + if (!match_int(arg, &n) && 0 <= n && n <= AUFS_MFS_MAX_SEC) + create->mfs_second = n; + else { + pr_err("bad integer in %s\n", str); + err = -EINVAL; + } + + return err; +} + +static int noinline_for_stack +au_wbr_create_val(char *str, struct au_opt_wbr_create *create) +{ + int err, e; + substring_t args[MAX_OPT_ARGS]; + + err = match_token(str, au_wbr_create_policy, args); + create->wbr_create = err; + switch (err) { + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + e = au_wbr_mfs_wmark(&args[0], str, create); + if (!e) + e = au_wbr_mfs_sec(&args[1], str, create); + if (unlikely(e)) + err = e; + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_PMFSRR: + e = au_wbr_mfs_wmark(&args[0], str, create); + if (unlikely(e)) { + err = e; + break; + } + /*FALLTHROUGH*/ + case AuWbrCreate_MFS: + case AuWbrCreate_PMFS: + create->mfs_second = AUFS_MFS_DEF_SEC; + break; + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + e = au_wbr_mfs_sec(&args[0], str, create); + if (unlikely(e)) + err = e; + break; + } + + return err; +} + +const char *au_optstr_wbr_create(int wbr_create) +{ + return au_parser_pattern(wbr_create, au_wbr_create_policy); +} + +static match_table_t au_wbr_copyup_policy = { + {AuWbrCopyup_TDP, "tdp"}, + {AuWbrCopyup_TDP, "top-down-parent"}, + {AuWbrCopyup_BUP, "bup"}, + {AuWbrCopyup_BUP, "bottom-up-parent"}, + {AuWbrCopyup_BU, "bu"}, + {AuWbrCopyup_BU, "bottom-up"}, + {-1, NULL} +}; + +static int noinline_for_stack au_wbr_copyup_val(char *str) +{ + substring_t args[MAX_OPT_ARGS]; + + return match_token(str, au_wbr_copyup_policy, args); +} + +const char *au_optstr_wbr_copyup(int wbr_copyup) +{ + return au_parser_pattern(wbr_copyup, au_wbr_copyup_policy); +} + +/* ---------------------------------------------------------------------- */ + +static const int lkup_dirflags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + +static void dump_opts(struct au_opts *opts) +{ +#ifdef CONFIG_AUFS_DEBUG + /* reduce stack space */ + union { + struct au_opt_add *add; + struct au_opt_del *del; + struct au_opt_mod *mod; + struct au_opt_xino *xino; + struct au_opt_xino_itrunc *xino_itrunc; + struct au_opt_wbr_create *create; + } u; + struct au_opt *opt; + + opt = opts->opt; + while (opt->type != Opt_tail) { + switch (opt->type) { + case Opt_add: + u.add = &opt->add; + AuDbg("add {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_del: + case Opt_idel: + u.del = &opt->del; + AuDbg("del {%s, %p}\n", + u.del->pathname, u.del->h_path.dentry); + break; + case Opt_mod: + case Opt_imod: + u.mod = &opt->mod; + AuDbg("mod {%s, 0x%x, %p}\n", + u.mod->path, u.mod->perm, u.mod->h_root); + break; + case Opt_append: + u.add = &opt->add; + AuDbg("append {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_prepend: + u.add = &opt->add; + AuDbg("prepend {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_dirwh: + AuDbg("dirwh %d\n", opt->dirwh); + break; + case Opt_rdcache: + AuDbg("rdcache %d\n", opt->rdcache); + break; + case Opt_rdblk: + AuDbg("rdblk %u\n", opt->rdblk); + break; + case Opt_rdblk_def: + AuDbg("rdblk_def\n"); + break; + case Opt_rdhash: + AuDbg("rdhash %u\n", opt->rdhash); + break; + case Opt_rdhash_def: + AuDbg("rdhash_def\n"); + break; + case Opt_xino: + u.xino = &opt->xino; + AuDbg("xino {%s %pD}\n", u.xino->path, u.xino->file); + break; + case Opt_trunc_xino: + AuLabel(trunc_xino); + break; + case Opt_notrunc_xino: + AuLabel(notrunc_xino); + break; + case Opt_trunc_xino_path: + case Opt_itrunc_xino: + u.xino_itrunc = &opt->xino_itrunc; + AuDbg("trunc_xino %d\n", u.xino_itrunc->bindex); + break; + case Opt_noxino: + AuLabel(noxino); + break; + case Opt_trunc_xib: + AuLabel(trunc_xib); + break; + case Opt_notrunc_xib: + AuLabel(notrunc_xib); + break; + case Opt_shwh: + AuLabel(shwh); + break; + case Opt_noshwh: + AuLabel(noshwh); + break; + case Opt_dirperm1: + AuLabel(dirperm1); + break; + case Opt_nodirperm1: + AuLabel(nodirperm1); + break; + case Opt_plink: + AuLabel(plink); + break; + case Opt_noplink: + AuLabel(noplink); + break; + case Opt_list_plink: + AuLabel(list_plink); + break; + case Opt_udba: + AuDbg("udba %d, %s\n", + opt->udba, au_optstr_udba(opt->udba)); + break; + case Opt_dio: + AuLabel(dio); + break; + case Opt_nodio: + AuLabel(nodio); + break; + case Opt_diropq_a: + AuLabel(diropq_a); + break; + case Opt_diropq_w: + AuLabel(diropq_w); + break; + case Opt_warn_perm: + AuLabel(warn_perm); + break; + case Opt_nowarn_perm: + AuLabel(nowarn_perm); + break; + case Opt_verbose: + AuLabel(verbose); + break; + case Opt_noverbose: + AuLabel(noverbose); + break; + case Opt_sum: + AuLabel(sum); + break; + case Opt_nosum: + AuLabel(nosum); + break; + case Opt_wsum: + AuLabel(wsum); + break; + case Opt_wbr_create: + u.create = &opt->wbr_create; + AuDbg("create %d, %s\n", u.create->wbr_create, + au_optstr_wbr_create(u.create->wbr_create)); + switch (u.create->wbr_create) { + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + AuDbg("%d sec\n", u.create->mfs_second); + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + AuDbg("%llu watermark\n", + u.create->mfsrr_watermark); + break; + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + AuDbg("%llu watermark, %d sec\n", + u.create->mfsrr_watermark, + u.create->mfs_second); + break; + } + break; + case Opt_wbr_copyup: + AuDbg("copyup %d, %s\n", opt->wbr_copyup, + au_optstr_wbr_copyup(opt->wbr_copyup)); + break; + case Opt_fhsm_sec: + AuDbg("fhsm_sec %u\n", opt->fhsm_second); + break; + case Opt_dirren: + AuLabel(dirren); + break; + case Opt_nodirren: + AuLabel(nodirren); + break; + case Opt_acl: + AuLabel(acl); + break; + case Opt_noacl: + AuLabel(noacl); + break; + default: + BUG(); + } + opt++; + } +#endif +} + +void au_opts_free(struct au_opts *opts) +{ + struct au_opt *opt; + + opt = opts->opt; + while (opt->type != Opt_tail) { + switch (opt->type) { + case Opt_add: + case Opt_append: + case Opt_prepend: + path_put(&opt->add.path); + break; + case Opt_del: + case Opt_idel: + path_put(&opt->del.h_path); + break; + case Opt_mod: + case Opt_imod: + dput(opt->mod.h_root); + break; + case Opt_xino: + fput(opt->xino.file); + break; + } + opt++; + } +} + +static int opt_add(struct au_opt *opt, char *opt_str, unsigned long sb_flags, + aufs_bindex_t bindex) +{ + int err; + struct au_opt_add *add = &opt->add; + char *p; + + add->bindex = bindex; + add->perm = AuBrPerm_RO; + add->pathname = opt_str; + p = strchr(opt_str, '='); + if (p) { + *p++ = 0; + if (*p) + add->perm = br_perm_val(p); + } + + err = vfsub_kern_path(add->pathname, lkup_dirflags, &add->path); + if (!err) { + if (!p) { + add->perm = AuBrPerm_RO; + if (au_test_fs_rr(add->path.dentry->d_sb)) + add->perm = AuBrPerm_RR; + else if (!bindex && !(sb_flags & MS_RDONLY)) + add->perm = AuBrPerm_RW; + } + opt->type = Opt_add; + goto out; + } + pr_err("lookup failed %s (%d)\n", add->pathname, err); + err = -EINVAL; + +out: + return err; +} + +static int au_opts_parse_del(struct au_opt_del *del, substring_t args[]) +{ + int err; + + del->pathname = args[0].from; + AuDbg("del path %s\n", del->pathname); + + err = vfsub_kern_path(del->pathname, lkup_dirflags, &del->h_path); + if (unlikely(err)) + pr_err("lookup failed %s (%d)\n", del->pathname, err); + + return err; +} + +#if 0 /* reserved for future use */ +static int au_opts_parse_idel(struct super_block *sb, aufs_bindex_t bindex, + struct au_opt_del *del, substring_t args[]) +{ + int err; + struct dentry *root; + + err = -EINVAL; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + if (bindex < 0 || au_sbbot(sb) < bindex) { + pr_err("out of bounds, %d\n", bindex); + goto out; + } + + err = 0; + del->h_path.dentry = dget(au_h_dptr(root, bindex)); + del->h_path.mnt = mntget(au_sbr_mnt(sb, bindex)); + +out: + aufs_read_unlock(root, !AuLock_IR); + return err; +} +#endif + +static int noinline_for_stack +au_opts_parse_mod(struct au_opt_mod *mod, substring_t args[]) +{ + int err; + struct path path; + char *p; + + err = -EINVAL; + mod->path = args[0].from; + p = strchr(mod->path, '='); + if (unlikely(!p)) { + pr_err("no permission %s\n", args[0].from); + goto out; + } + + *p++ = 0; + err = vfsub_kern_path(mod->path, lkup_dirflags, &path); + if (unlikely(err)) { + pr_err("lookup failed %s (%d)\n", mod->path, err); + goto out; + } + + mod->perm = br_perm_val(p); + AuDbg("mod path %s, perm 0x%x, %s\n", mod->path, mod->perm, p); + mod->h_root = dget(path.dentry); + path_put(&path); + +out: + return err; +} + +#if 0 /* reserved for future use */ +static int au_opts_parse_imod(struct super_block *sb, aufs_bindex_t bindex, + struct au_opt_mod *mod, substring_t args[]) +{ + int err; + struct dentry *root; + + err = -EINVAL; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + if (bindex < 0 || au_sbbot(sb) < bindex) { + pr_err("out of bounds, %d\n", bindex); + goto out; + } + + err = 0; + mod->perm = br_perm_val(args[1].from); + AuDbg("mod path %s, perm 0x%x, %s\n", + mod->path, mod->perm, args[1].from); + mod->h_root = dget(au_h_dptr(root, bindex)); + +out: + aufs_read_unlock(root, !AuLock_IR); + return err; +} +#endif + +static int au_opts_parse_xino(struct super_block *sb, struct au_opt_xino *xino, + substring_t args[]) +{ + int err; + struct file *file; + + file = au_xino_create(sb, args[0].from, /*silent*/0, /*wbrtop*/0); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + + err = -EINVAL; + if (unlikely(file->f_path.dentry->d_sb == sb)) { + fput(file); + pr_err("%s must be outside\n", args[0].from); + goto out; + } + + err = 0; + xino->file = file; + xino->path = args[0].from; + +out: + return err; +} + +static int noinline_for_stack +au_opts_parse_xino_itrunc_path(struct super_block *sb, + struct au_opt_xino_itrunc *xino_itrunc, + substring_t args[]) +{ + int err; + aufs_bindex_t bbot, bindex; + struct path path; + struct dentry *root; + + err = vfsub_kern_path(args[0].from, lkup_dirflags, &path); + if (unlikely(err)) { + pr_err("lookup failed %s (%d)\n", args[0].from, err); + goto out; + } + + xino_itrunc->bindex = -1; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + if (au_h_dptr(root, bindex) == path.dentry) { + xino_itrunc->bindex = bindex; + break; + } + } + aufs_read_unlock(root, !AuLock_IR); + path_put(&path); + + if (unlikely(xino_itrunc->bindex < 0)) { + pr_err("no such branch %s\n", args[0].from); + err = -EINVAL; + } + +out: + return err; +} + +/* called without aufs lock */ +int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts) +{ + int err, n, token; + aufs_bindex_t bindex; + unsigned char skipped; + struct dentry *root; + struct au_opt *opt, *opt_tail; + char *opt_str; + /* reduce the stack space */ + union { + struct au_opt_xino_itrunc *xino_itrunc; + struct au_opt_wbr_create *create; + } u; + struct { + substring_t args[MAX_OPT_ARGS]; + } *a; + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + root = sb->s_root; + err = 0; + bindex = 0; + opt = opts->opt; + opt_tail = opt + opts->max_opt - 1; + opt->type = Opt_tail; + while (!err && (opt_str = strsep(&str, ",")) && *opt_str) { + err = -EINVAL; + skipped = 0; + token = match_token(opt_str, options, a->args); + switch (token) { + case Opt_br: + err = 0; + while (!err && (opt_str = strsep(&a->args[0].from, ":")) + && *opt_str) { + err = opt_add(opt, opt_str, opts->sb_flags, + bindex++); + if (unlikely(!err && ++opt > opt_tail)) { + err = -E2BIG; + break; + } + opt->type = Opt_tail; + skipped = 1; + } + break; + case Opt_add: + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + bindex = n; + err = opt_add(opt, a->args[1].from, opts->sb_flags, + bindex); + if (!err) + opt->type = token; + break; + case Opt_append: + err = opt_add(opt, a->args[0].from, opts->sb_flags, + /*dummy bindex*/1); + if (!err) + opt->type = token; + break; + case Opt_prepend: + err = opt_add(opt, a->args[0].from, opts->sb_flags, + /*bindex*/0); + if (!err) + opt->type = token; + break; + case Opt_del: + err = au_opts_parse_del(&opt->del, a->args); + if (!err) + opt->type = token; + break; +#if 0 /* reserved for future use */ + case Opt_idel: + del->pathname = "(indexed)"; + if (unlikely(match_int(&args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + err = au_opts_parse_idel(sb, n, &opt->del, a->args); + if (!err) + opt->type = token; + break; +#endif + case Opt_mod: + err = au_opts_parse_mod(&opt->mod, a->args); + if (!err) + opt->type = token; + break; +#ifdef IMOD /* reserved for future use */ + case Opt_imod: + u.mod->path = "(indexed)"; + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + err = au_opts_parse_imod(sb, n, &opt->mod, a->args); + if (!err) + opt->type = token; + break; +#endif + case Opt_xino: + err = au_opts_parse_xino(sb, &opt->xino, a->args); + if (!err) + opt->type = token; + break; + + case Opt_trunc_xino_path: + err = au_opts_parse_xino_itrunc_path + (sb, &opt->xino_itrunc, a->args); + if (!err) + opt->type = token; + break; + + case Opt_itrunc_xino: + u.xino_itrunc = &opt->xino_itrunc; + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + u.xino_itrunc->bindex = n; + aufs_read_lock(root, AuLock_FLUSH); + if (n < 0 || au_sbbot(sb) < n) { + pr_err("out of bounds, %d\n", n); + aufs_read_unlock(root, !AuLock_IR); + break; + } + aufs_read_unlock(root, !AuLock_IR); + err = 0; + opt->type = token; + break; + + case Opt_dirwh: + if (unlikely(match_int(&a->args[0], &opt->dirwh))) + break; + err = 0; + opt->type = token; + break; + + case Opt_rdcache: + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (unlikely(n > AUFS_RDCACHE_MAX)) { + pr_err("rdcache must be smaller than %d\n", + AUFS_RDCACHE_MAX); + break; + } + opt->rdcache = n; + err = 0; + opt->type = token; + break; + case Opt_rdblk: + if (unlikely(match_int(&a->args[0], &n) + || n < 0 + || n > KMALLOC_MAX_SIZE)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (unlikely(n && n < NAME_MAX)) { + pr_err("rdblk must be larger than %d\n", + NAME_MAX); + break; + } + opt->rdblk = n; + err = 0; + opt->type = token; + break; + case Opt_rdhash: + if (unlikely(match_int(&a->args[0], &n) + || n < 0 + || n * sizeof(struct hlist_head) + > KMALLOC_MAX_SIZE)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + opt->rdhash = n; + err = 0; + opt->type = token; + break; + + case Opt_trunc_xino: + case Opt_notrunc_xino: + case Opt_noxino: + case Opt_trunc_xib: + case Opt_notrunc_xib: + case Opt_shwh: + case Opt_noshwh: + case Opt_dirperm1: + case Opt_nodirperm1: + case Opt_plink: + case Opt_noplink: + case Opt_list_plink: + case Opt_dio: + case Opt_nodio: + case Opt_diropq_a: + case Opt_diropq_w: + case Opt_warn_perm: + case Opt_nowarn_perm: + case Opt_verbose: + case Opt_noverbose: + case Opt_sum: + case Opt_nosum: + case Opt_wsum: + case Opt_rdblk_def: + case Opt_rdhash_def: + case Opt_dirren: + case Opt_nodirren: + case Opt_acl: + case Opt_noacl: + err = 0; + opt->type = token; + break; + + case Opt_udba: + opt->udba = udba_val(a->args[0].from); + if (opt->udba >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + + case Opt_wbr_create: + u.create = &opt->wbr_create; + u.create->wbr_create + = au_wbr_create_val(a->args[0].from, u.create); + if (u.create->wbr_create >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + case Opt_wbr_copyup: + opt->wbr_copyup = au_wbr_copyup_val(a->args[0].from); + if (opt->wbr_copyup >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + + case Opt_fhsm_sec: + if (unlikely(match_int(&a->args[0], &n) + || n < 0)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (sysaufs_brs) { + opt->fhsm_second = n; + opt->type = token; + } else + pr_warn("ignored %s\n", opt_str); + err = 0; + break; + + case Opt_ignore: + pr_warn("ignored %s\n", opt_str); + /*FALLTHROUGH*/ + case Opt_ignore_silent: + skipped = 1; + err = 0; + break; + case Opt_err: + pr_err("unknown option %s\n", opt_str); + break; + } + + if (!err && !skipped) { + if (unlikely(++opt > opt_tail)) { + err = -E2BIG; + opt--; + opt->type = Opt_tail; + break; + } + opt->type = Opt_tail; + } + } + + au_kfree_rcu(a); + dump_opts(opts); + if (unlikely(err)) + au_opts_free(opts); + +out: + return err; +} + +static int au_opt_wbr_create(struct super_block *sb, + struct au_opt_wbr_create *create) +{ + int err; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = 1; /* handled */ + sbinfo = au_sbi(sb); + if (sbinfo->si_wbr_create_ops->fin) { + err = sbinfo->si_wbr_create_ops->fin(sb); + if (!err) + err = 1; + } + + sbinfo->si_wbr_create = create->wbr_create; + sbinfo->si_wbr_create_ops = au_wbr_create_ops + create->wbr_create; + switch (create->wbr_create) { + case AuWbrCreate_MFSRRV: + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRR: + case AuWbrCreate_PMFSRRV: + sbinfo->si_wbr_mfs.mfsrr_watermark = create->mfsrr_watermark; + /*FALLTHROUGH*/ + case AuWbrCreate_MFS: + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFS: + case AuWbrCreate_PMFSV: + sbinfo->si_wbr_mfs.mfs_expire + = msecs_to_jiffies(create->mfs_second * MSEC_PER_SEC); + break; + } + + if (sbinfo->si_wbr_create_ops->init) + sbinfo->si_wbr_create_ops->init(sb); /* ignore */ + + return err; +} + +/* + * returns, + * plus: processed without an error + * zero: unprocessed + */ +static int au_opt_simple(struct super_block *sb, struct au_opt *opt, + struct au_opts *opts) +{ + int err; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = 1; /* handled */ + sbinfo = au_sbi(sb); + switch (opt->type) { + case Opt_udba: + sbinfo->si_mntflags &= ~AuOptMask_UDBA; + sbinfo->si_mntflags |= opt->udba; + opts->given_udba |= opt->udba; + break; + + case Opt_plink: + au_opt_set(sbinfo->si_mntflags, PLINK); + break; + case Opt_noplink: + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_put(sb, /*verbose*/1); + au_opt_clr(sbinfo->si_mntflags, PLINK); + break; + case Opt_list_plink: + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_list(sb); + break; + + case Opt_dio: + au_opt_set(sbinfo->si_mntflags, DIO); + au_fset_opts(opts->flags, REFRESH_DYAOP); + break; + case Opt_nodio: + au_opt_clr(sbinfo->si_mntflags, DIO); + au_fset_opts(opts->flags, REFRESH_DYAOP); + break; + + case Opt_fhsm_sec: + au_fhsm_set(sbinfo, opt->fhsm_second); + break; + + case Opt_diropq_a: + au_opt_set(sbinfo->si_mntflags, ALWAYS_DIROPQ); + break; + case Opt_diropq_w: + au_opt_clr(sbinfo->si_mntflags, ALWAYS_DIROPQ); + break; + + case Opt_warn_perm: + au_opt_set(sbinfo->si_mntflags, WARN_PERM); + break; + case Opt_nowarn_perm: + au_opt_clr(sbinfo->si_mntflags, WARN_PERM); + break; + + case Opt_verbose: + au_opt_set(sbinfo->si_mntflags, VERBOSE); + break; + case Opt_noverbose: + au_opt_clr(sbinfo->si_mntflags, VERBOSE); + break; + + case Opt_sum: + au_opt_set(sbinfo->si_mntflags, SUM); + break; + case Opt_wsum: + au_opt_clr(sbinfo->si_mntflags, SUM); + au_opt_set(sbinfo->si_mntflags, SUM_W); + break; + case Opt_nosum: + au_opt_clr(sbinfo->si_mntflags, SUM); + au_opt_clr(sbinfo->si_mntflags, SUM_W); + break; + + case Opt_wbr_create: + err = au_opt_wbr_create(sb, &opt->wbr_create); + break; + case Opt_wbr_copyup: + sbinfo->si_wbr_copyup = opt->wbr_copyup; + sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + opt->wbr_copyup; + break; + + case Opt_dirwh: + sbinfo->si_dirwh = opt->dirwh; + break; + + case Opt_rdcache: + sbinfo->si_rdcache + = msecs_to_jiffies(opt->rdcache * MSEC_PER_SEC); + break; + case Opt_rdblk: + sbinfo->si_rdblk = opt->rdblk; + break; + case Opt_rdblk_def: + sbinfo->si_rdblk = AUFS_RDBLK_DEF; + break; + case Opt_rdhash: + sbinfo->si_rdhash = opt->rdhash; + break; + case Opt_rdhash_def: + sbinfo->si_rdhash = AUFS_RDHASH_DEF; + break; + + case Opt_shwh: + au_opt_set(sbinfo->si_mntflags, SHWH); + break; + case Opt_noshwh: + au_opt_clr(sbinfo->si_mntflags, SHWH); + break; + + case Opt_dirperm1: + au_opt_set(sbinfo->si_mntflags, DIRPERM1); + break; + case Opt_nodirperm1: + au_opt_clr(sbinfo->si_mntflags, DIRPERM1); + break; + + case Opt_trunc_xino: + au_opt_set(sbinfo->si_mntflags, TRUNC_XINO); + break; + case Opt_notrunc_xino: + au_opt_clr(sbinfo->si_mntflags, TRUNC_XINO); + break; + + case Opt_trunc_xino_path: + case Opt_itrunc_xino: + err = au_xino_trunc(sb, opt->xino_itrunc.bindex, + /*idx_begin*/0); + if (!err) + err = 1; + break; + + case Opt_trunc_xib: + au_fset_opts(opts->flags, TRUNC_XIB); + break; + case Opt_notrunc_xib: + au_fclr_opts(opts->flags, TRUNC_XIB); + break; + + case Opt_dirren: + err = 1; + if (!au_opt_test(sbinfo->si_mntflags, DIRREN)) { + err = au_dr_opt_set(sb); + if (!err) + err = 1; + } + if (err == 1) + au_opt_set(sbinfo->si_mntflags, DIRREN); + break; + case Opt_nodirren: + err = 1; + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) { + err = au_dr_opt_clr(sb, au_ftest_opts(opts->flags, + DR_FLUSHED)); + if (!err) + err = 1; + } + if (err == 1) + au_opt_clr(sbinfo->si_mntflags, DIRREN); + break; + + case Opt_acl: + sb->s_flags |= MS_POSIXACL; + break; + case Opt_noacl: + sb->s_flags &= ~MS_POSIXACL; + break; + + default: + err = 0; + break; + } + + return err; +} + +/* + * returns tri-state. + * plus: processed without an error + * zero: unprocessed + * minus: error + */ +static int au_opt_br(struct super_block *sb, struct au_opt *opt, + struct au_opts *opts) +{ + int err, do_refresh; + + err = 0; + switch (opt->type) { + case Opt_append: + opt->add.bindex = au_sbbot(sb) + 1; + if (opt->add.bindex < 0) + opt->add.bindex = 0; + goto add; + /* Always goto add, not fallthrough */ + case Opt_prepend: + opt->add.bindex = 0; + /* fallthrough */ + add: /* indented label */ + case Opt_add: + err = au_br_add(sb, &opt->add, + au_ftest_opts(opts->flags, REMOUNT)); + if (!err) { + err = 1; + au_fset_opts(opts->flags, REFRESH); + } + break; + + case Opt_del: + case Opt_idel: + err = au_br_del(sb, &opt->del, + au_ftest_opts(opts->flags, REMOUNT)); + if (!err) { + err = 1; + au_fset_opts(opts->flags, TRUNC_XIB); + au_fset_opts(opts->flags, REFRESH); + } + break; + + case Opt_mod: + case Opt_imod: + err = au_br_mod(sb, &opt->mod, + au_ftest_opts(opts->flags, REMOUNT), + &do_refresh); + if (!err) { + err = 1; + if (do_refresh) + au_fset_opts(opts->flags, REFRESH); + } + break; + } + return err; +} + +static int au_opt_xino(struct super_block *sb, struct au_opt *opt, + struct au_opt_xino **opt_xino, + struct au_opts *opts) +{ + int err; + + err = 0; + switch (opt->type) { + case Opt_xino: + err = au_xino_set(sb, &opt->xino, + !!au_ftest_opts(opts->flags, REMOUNT)); + if (unlikely(err)) + break; + + *opt_xino = &opt->xino; + break; + + case Opt_noxino: + au_xino_clr(sb); + *opt_xino = (void *)-1; + break; + } + + return err; +} + +int au_opts_verify(struct super_block *sb, unsigned long sb_flags, + unsigned int pending) +{ + int err, fhsm; + aufs_bindex_t bindex, bbot; + unsigned char do_plink, skip, do_free, can_no_dreval; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *root, *dentry; + struct inode *dir, *h_dir; + struct au_sbinfo *sbinfo; + struct au_hinode *hdir; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!(sbinfo->si_mntflags & AuOptMask_UDBA)); + + if (!(sb_flags & MS_RDONLY)) { + if (unlikely(!au_br_writable(au_sbr_perm(sb, 0)))) + pr_warn("first branch should be rw\n"); + if (unlikely(au_opt_test(sbinfo->si_mntflags, SHWH))) + pr_warn_once("shwh should be used with ro\n"); + } + + if (au_opt_test((sbinfo->si_mntflags | pending), UDBA_HNOTIFY) + && !au_opt_test(sbinfo->si_mntflags, XINO)) + pr_warn_once("udba=*notify requires xino\n"); + + if (au_opt_test(sbinfo->si_mntflags, DIRPERM1)) + pr_warn_once("dirperm1 breaks the protection" + " by the permission bits on the lower branch\n"); + + err = 0; + fhsm = 0; + root = sb->s_root; + dir = d_inode(root); + do_plink = !!au_opt_test(sbinfo->si_mntflags, PLINK); + can_no_dreval = !!au_opt_test((sbinfo->si_mntflags | pending), + UDBA_NONE); + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + skip = 0; + h_dir = au_h_iptr(dir, bindex); + br = au_sbr(sb, bindex); + + if ((br->br_perm & AuBrAttr_ICEX) + && !h_dir->i_op->listxattr) + br->br_perm &= ~AuBrAttr_ICEX; +#if 0 + if ((br->br_perm & AuBrAttr_ICEX_SEC) + && (au_br_sb(br)->s_flags & MS_NOSEC)) + br->br_perm &= ~AuBrAttr_ICEX_SEC; +#endif + + do_free = 0; + wbr = br->br_wbr; + if (wbr) + wbr_wh_read_lock(wbr); + + if (!au_br_writable(br->br_perm)) { + do_free = !!wbr; + skip = (!wbr + || (!wbr->wbr_whbase + && !wbr->wbr_plink + && !wbr->wbr_orph)); + } else if (!au_br_wh_linkable(br->br_perm)) { + /* skip = (!br->br_whbase && !br->br_orph); */ + skip = (!wbr || !wbr->wbr_whbase); + if (skip && wbr) { + if (do_plink) + skip = !!wbr->wbr_plink; + else + skip = !wbr->wbr_plink; + } + } else { + /* skip = (br->br_whbase && br->br_ohph); */ + skip = (wbr && wbr->wbr_whbase); + if (skip) { + if (do_plink) + skip = !!wbr->wbr_plink; + else + skip = !wbr->wbr_plink; + } + } + if (wbr) + wbr_wh_read_unlock(wbr); + + if (can_no_dreval) { + dentry = br->br_path.dentry; + spin_lock(&dentry->d_lock); + if (dentry->d_flags & + (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE)) + can_no_dreval = 0; + spin_unlock(&dentry->d_lock); + } + + if (au_br_fhsm(br->br_perm)) { + fhsm++; + AuDebugOn(!br->br_fhsm); + } + + if (skip) + continue; + + hdir = au_hi(dir, bindex); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + if (wbr) + wbr_wh_write_lock(wbr); + err = au_wh_init(br, sb); + if (wbr) + wbr_wh_write_unlock(wbr); + au_hn_inode_unlock(hdir); + + if (!err && do_free) { + au_kfree_rcu(wbr); + br->br_wbr = NULL; + } + } + + if (can_no_dreval) + au_fset_si(sbinfo, NO_DREVAL); + else + au_fclr_si(sbinfo, NO_DREVAL); + + if (fhsm >= 2) { + au_fset_si(sbinfo, FHSM); + for (bindex = bbot; bindex >= 0; bindex--) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) { + au_fhsm_set_bottom(sb, bindex); + break; + } + } + } else { + au_fclr_si(sbinfo, FHSM); + au_fhsm_set_bottom(sb, -1); + } + + return err; +} + +int au_opts_mount(struct super_block *sb, struct au_opts *opts) +{ + int err; + unsigned int tmp; + aufs_bindex_t bindex, bbot; + struct au_opt *opt; + struct au_opt_xino *opt_xino, xino; + struct au_sbinfo *sbinfo; + struct au_branch *br; + struct inode *dir; + + SiMustWriteLock(sb); + + err = 0; + opt_xino = NULL; + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) + err = au_opt_simple(sb, opt++, opts); + if (err > 0) + err = 0; + else if (unlikely(err < 0)) + goto out; + + /* disable xino and udba temporary */ + sbinfo = au_sbi(sb); + tmp = sbinfo->si_mntflags; + au_opt_clr(sbinfo->si_mntflags, XINO); + au_opt_set_udba(sbinfo->si_mntflags, UDBA_REVAL); + + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) + err = au_opt_br(sb, opt++, opts); + if (err > 0) + err = 0; + else if (unlikely(err < 0)) + goto out; + + bbot = au_sbbot(sb); + if (unlikely(bbot < 0)) { + err = -EINVAL; + pr_err("no branches\n"); + goto out; + } + + if (au_opt_test(tmp, XINO)) + au_opt_set(sbinfo->si_mntflags, XINO); + opt = opts->opt; + while (!err && opt->type != Opt_tail) + err = au_opt_xino(sb, opt++, &opt_xino, opts); + if (unlikely(err)) + goto out; + + err = au_opts_verify(sb, sb->s_flags, tmp); + if (unlikely(err)) + goto out; + + /* restore xino */ + if (au_opt_test(tmp, XINO) && !opt_xino) { + xino.file = au_xino_def(sb); + err = PTR_ERR(xino.file); + if (IS_ERR(xino.file)) + goto out; + + err = au_xino_set(sb, &xino, /*remount*/0); + fput(xino.file); + if (unlikely(err)) + goto out; + } + + /* restore udba */ + tmp &= AuOptMask_UDBA; + sbinfo->si_mntflags &= ~AuOptMask_UDBA; + sbinfo->si_mntflags |= tmp; + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_hnotify_reset_br(tmp, br, br->br_perm); + if (unlikely(err)) + AuIOErr("hnotify failed on br %d, %d, ignored\n", + bindex, err); + /* go on even if err */ + } + if (au_opt_test(tmp, UDBA_HNOTIFY)) { + dir = d_inode(sb->s_root); + au_hn_reset(dir, au_hi_flags(dir, /*isdir*/1) & ~AuHi_XINO); + } + +out: + return err; +} + +int au_opts_remount(struct super_block *sb, struct au_opts *opts) +{ + int err, rerr; + unsigned char no_dreval; + struct inode *dir; + struct au_opt_xino *opt_xino; + struct au_opt *opt; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = au_dr_opt_flush(sb); + if (unlikely(err)) + goto out; + au_fset_opts(opts->flags, DR_FLUSHED); + + dir = d_inode(sb->s_root); + sbinfo = au_sbi(sb); + opt_xino = NULL; + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) { + err = au_opt_simple(sb, opt, opts); + if (!err) + err = au_opt_br(sb, opt, opts); + if (!err) + err = au_opt_xino(sb, opt, &opt_xino, opts); + opt++; + } + if (err > 0) + err = 0; + AuTraceErr(err); + /* go on even err */ + + no_dreval = !!au_ftest_si(sbinfo, NO_DREVAL); + rerr = au_opts_verify(sb, opts->sb_flags, /*pending*/0); + if (unlikely(rerr && !err)) + err = rerr; + + if (no_dreval != !!au_ftest_si(sbinfo, NO_DREVAL)) + au_fset_opts(opts->flags, REFRESH_IDOP); + + if (au_ftest_opts(opts->flags, TRUNC_XIB)) { + rerr = au_xib_trunc(sb); + if (unlikely(rerr && !err)) + err = rerr; + } + + /* will be handled by the caller */ + if (!au_ftest_opts(opts->flags, REFRESH) + && (opts->given_udba + || au_opt_test(sbinfo->si_mntflags, XINO) + || au_ftest_opts(opts->flags, REFRESH_IDOP) + )) + au_fset_opts(opts->flags, REFRESH); + + AuDbg("status 0x%x\n", opts->flags); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +unsigned int au_opt_udba(struct super_block *sb) +{ + return au_mntflags(sb) & AuOptMask_UDBA; +} diff --git a/fs/aufs/opts.h b/fs/aufs/opts.h new file mode 100644 index 0000000000000..6326b54340c78 --- /dev/null +++ b/fs/aufs/opts.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount options/flags + */ + +#ifndef __AUFS_OPTS_H__ +#define __AUFS_OPTS_H__ + +#ifdef __KERNEL__ + +#include + +struct file; + +/* ---------------------------------------------------------------------- */ + +/* mount flags */ +#define AuOpt_XINO 1 /* external inode number bitmap + and translation table */ +#define AuOpt_TRUNC_XINO (1 << 1) /* truncate xino files */ +#define AuOpt_UDBA_NONE (1 << 2) /* users direct branch access */ +#define AuOpt_UDBA_REVAL (1 << 3) +#define AuOpt_UDBA_HNOTIFY (1 << 4) +#define AuOpt_SHWH (1 << 5) /* show whiteout */ +#define AuOpt_PLINK (1 << 6) /* pseudo-link */ +#define AuOpt_DIRPERM1 (1 << 7) /* ignore the lower dir's perm + bits */ +#define AuOpt_ALWAYS_DIROPQ (1 << 9) /* policy to creating diropq */ +#define AuOpt_SUM (1 << 10) /* summation for statfs(2) */ +#define AuOpt_SUM_W (1 << 11) /* unimplemented */ +#define AuOpt_WARN_PERM (1 << 12) /* warn when add-branch */ +#define AuOpt_VERBOSE (1 << 13) /* print the cause of error */ +#define AuOpt_DIO (1 << 14) /* direct io */ +#define AuOpt_DIRREN (1 << 15) /* directory rename */ + +#ifndef CONFIG_AUFS_HNOTIFY +#undef AuOpt_UDBA_HNOTIFY +#define AuOpt_UDBA_HNOTIFY 0 +#endif +#ifndef CONFIG_AUFS_DIRREN +#undef AuOpt_DIRREN +#define AuOpt_DIRREN 0 +#endif +#ifndef CONFIG_AUFS_SHWH +#undef AuOpt_SHWH +#define AuOpt_SHWH 0 +#endif + +#define AuOpt_Def (AuOpt_XINO \ + | AuOpt_UDBA_REVAL \ + | AuOpt_PLINK \ + /* | AuOpt_DIRPERM1 */ \ + | AuOpt_WARN_PERM) +#define AuOptMask_UDBA (AuOpt_UDBA_NONE \ + | AuOpt_UDBA_REVAL \ + | AuOpt_UDBA_HNOTIFY) + +#define au_opt_test(flags, name) (flags & AuOpt_##name) +#define au_opt_set(flags, name) do { \ + BUILD_BUG_ON(AuOpt_##name & AuOptMask_UDBA); \ + ((flags) |= AuOpt_##name); \ +} while (0) +#define au_opt_set_udba(flags, name) do { \ + (flags) &= ~AuOptMask_UDBA; \ + ((flags) |= AuOpt_##name); \ +} while (0) +#define au_opt_clr(flags, name) do { \ + ((flags) &= ~AuOpt_##name); \ +} while (0) + +static inline unsigned int au_opts_plink(unsigned int mntflags) +{ +#ifdef CONFIG_PROC_FS + return mntflags; +#else + return mntflags & ~AuOpt_PLINK; +#endif +} + +/* ---------------------------------------------------------------------- */ + +/* policies to select one among multiple writable branches */ +enum { + AuWbrCreate_TDP, /* top down parent */ + AuWbrCreate_RR, /* round robin */ + AuWbrCreate_MFS, /* most free space */ + AuWbrCreate_MFSV, /* mfs with seconds */ + AuWbrCreate_MFSRR, /* mfs then rr */ + AuWbrCreate_MFSRRV, /* mfs then rr with seconds */ + AuWbrCreate_TDMFS, /* top down regardless parent and mfs */ + AuWbrCreate_TDMFSV, /* top down regardless parent and mfs */ + AuWbrCreate_PMFS, /* parent and mfs */ + AuWbrCreate_PMFSV, /* parent and mfs with seconds */ + AuWbrCreate_PMFSRR, /* parent, mfs and round-robin */ + AuWbrCreate_PMFSRRV, /* plus seconds */ + + AuWbrCreate_Def = AuWbrCreate_TDP +}; + +enum { + AuWbrCopyup_TDP, /* top down parent */ + AuWbrCopyup_BUP, /* bottom up parent */ + AuWbrCopyup_BU, /* bottom up */ + + AuWbrCopyup_Def = AuWbrCopyup_TDP +}; + +/* ---------------------------------------------------------------------- */ + +struct au_opt_add { + aufs_bindex_t bindex; + char *pathname; + int perm; + struct path path; +}; + +struct au_opt_del { + char *pathname; + struct path h_path; +}; + +struct au_opt_mod { + char *path; + int perm; + struct dentry *h_root; +}; + +struct au_opt_xino { + char *path; + struct file *file; +}; + +struct au_opt_xino_itrunc { + aufs_bindex_t bindex; +}; + +struct au_opt_wbr_create { + int wbr_create; + int mfs_second; + unsigned long long mfsrr_watermark; +}; + +struct au_opt { + int type; + union { + struct au_opt_xino xino; + struct au_opt_xino_itrunc xino_itrunc; + struct au_opt_add add; + struct au_opt_del del; + struct au_opt_mod mod; + int dirwh; + int rdcache; + unsigned int rdblk; + unsigned int rdhash; + int udba; + struct au_opt_wbr_create wbr_create; + int wbr_copyup; + unsigned int fhsm_second; + }; +}; + +/* opts flags */ +#define AuOpts_REMOUNT 1 +#define AuOpts_REFRESH (1 << 1) +#define AuOpts_TRUNC_XIB (1 << 2) +#define AuOpts_REFRESH_DYAOP (1 << 3) +#define AuOpts_REFRESH_IDOP (1 << 4) +#define AuOpts_DR_FLUSHED (1 << 5) +#define au_ftest_opts(flags, name) ((flags) & AuOpts_##name) +#define au_fset_opts(flags, name) \ + do { (flags) |= AuOpts_##name; } while (0) +#define au_fclr_opts(flags, name) \ + do { (flags) &= ~AuOpts_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuOpts_DR_FLUSHED +#define AuOpts_DR_FLUSHED 0 +#endif + +struct au_opts { + struct au_opt *opt; + int max_opt; + + unsigned int given_udba; + unsigned int flags; + unsigned long sb_flags; +}; + +/* ---------------------------------------------------------------------- */ + +/* opts.c */ +void au_optstr_br_perm(au_br_perm_str_t *str, int perm); +const char *au_optstr_udba(int udba); +const char *au_optstr_wbr_copyup(int wbr_copyup); +const char *au_optstr_wbr_create(int wbr_create); + +void au_opts_free(struct au_opts *opts); +struct super_block; +int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts); +int au_opts_verify(struct super_block *sb, unsigned long sb_flags, + unsigned int pending); +int au_opts_mount(struct super_block *sb, struct au_opts *opts); +int au_opts_remount(struct super_block *sb, struct au_opts *opts); + +unsigned int au_opt_udba(struct super_block *sb); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_OPTS_H__ */ diff --git a/fs/aufs/plink.c b/fs/aufs/plink.c new file mode 100644 index 0000000000000..8a3bf96b6b34f --- /dev/null +++ b/fs/aufs/plink.c @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * pseudo-link + */ + +#include "aufs.h" + +/* + * the pseudo-link maintenance mode. + * during a user process maintains the pseudo-links, + * prohibit adding a new plink and branch manipulation. + * + * Flags + * NOPLM: + * For entry functions which will handle plink, and i_mutex is already held + * in VFS. + * They cannot wait and should return an error at once. + * Callers has to check the error. + * NOPLMW: + * For entry functions which will handle plink, but i_mutex is not held + * in VFS. + * They can wait the plink maintenance mode to finish. + * + * They behave like F_SETLK and F_SETLKW. + * If the caller never handle plink, then both flags are unnecessary. + */ + +int au_plink_maint(struct super_block *sb, int flags) +{ + int err; + pid_t pid, ppid; + struct task_struct *parent, *prev; + struct au_sbinfo *sbi; + + SiMustAnyLock(sb); + + err = 0; + if (!au_opt_test(au_mntflags(sb), PLINK)) + goto out; + + sbi = au_sbi(sb); + pid = sbi->si_plink_maint_pid; + if (!pid || pid == current->pid) + goto out; + + /* todo: it highly depends upon /sbin/mount.aufs */ + prev = NULL; + parent = current; + ppid = 0; + rcu_read_lock(); + while (1) { + parent = rcu_dereference(parent->real_parent); + if (parent == prev) + break; + ppid = task_pid_vnr(parent); + if (pid == ppid) { + rcu_read_unlock(); + goto out; + } + prev = parent; + } + rcu_read_unlock(); + + if (au_ftest_lock(flags, NOPLMW)) { + /* if there is no i_mutex lock in VFS, we don't need to wait */ + /* AuDebugOn(!lockdep_depth(current)); */ + while (sbi->si_plink_maint_pid) { + si_read_unlock(sb); + /* gave up wake_up_bit() */ + wait_event(sbi->si_plink_wq, !sbi->si_plink_maint_pid); + + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&sbi->si_nowait); + si_noflush_read_lock(sb); + } + } else if (au_ftest_lock(flags, NOPLM)) { + AuDbg("ppid %d, pid %d\n", ppid, pid); + err = -EAGAIN; + } + +out: + return err; +} + +void au_plink_maint_leave(struct au_sbinfo *sbinfo) +{ + spin_lock(&sbinfo->si_plink_maint_lock); + sbinfo->si_plink_maint_pid = 0; + spin_unlock(&sbinfo->si_plink_maint_lock); + wake_up_all(&sbinfo->si_plink_wq); +} + +int au_plink_maint_enter(struct super_block *sb) +{ + int err; + struct au_sbinfo *sbinfo; + + err = 0; + sbinfo = au_sbi(sb); + /* make sure i am the only one in this fs */ + si_write_lock(sb, AuLock_FLUSH); + if (au_opt_test(au_mntflags(sb), PLINK)) { + spin_lock(&sbinfo->si_plink_maint_lock); + if (!sbinfo->si_plink_maint_pid) + sbinfo->si_plink_maint_pid = current->pid; + else + err = -EBUSY; + spin_unlock(&sbinfo->si_plink_maint_lock); + } + si_write_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_DEBUG +void au_plink_list(struct super_block *sb) +{ + int i; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) + AuDbg("%lu\n", icntnr->vfs_inode.i_ino); + hlist_bl_unlock(hbl); + } +} +#endif + +/* is the inode pseudo-linked? */ +int au_plink_test(struct inode *inode) +{ + int found, i; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + + sbinfo = au_sbi(inode->i_sb); + AuRwMustAnyLock(&sbinfo->si_rwsem); + AuDebugOn(!au_opt_test(au_mntflags(inode->i_sb), PLINK)); + AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM)); + + found = 0; + i = au_plink_hash(inode->i_ino); + hbl = sbinfo->si_plink + i; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) + if (&icntnr->vfs_inode == inode) { + found = 1; + break; + } + hlist_bl_unlock(hbl); + return found; +} + +/* ---------------------------------------------------------------------- */ + +/* + * generate a name for plink. + * the file will be stored under AUFS_WH_PLINKDIR. + */ +/* 20 is max digits length of ulong 64 */ +#define PLINK_NAME_LEN ((20 + 1) * 2) + +static int plink_name(char *name, int len, struct inode *inode, + aufs_bindex_t bindex) +{ + int rlen; + struct inode *h_inode; + + h_inode = au_h_iptr(inode, bindex); + rlen = snprintf(name, len, "%lu.%lu", inode->i_ino, h_inode->i_ino); + return rlen; +} + +struct au_do_plink_lkup_args { + struct dentry **errp; + struct qstr *tgtname; + struct dentry *h_parent; + struct au_branch *br; +}; + +static struct dentry *au_do_plink_lkup(struct qstr *tgtname, + struct dentry *h_parent, + struct au_branch *br) +{ + struct dentry *h_dentry; + struct inode *h_inode; + + h_inode = d_inode(h_parent); + vfsub_inode_lock_shared_nested(h_inode, AuLsc_I_CHILD2); + h_dentry = vfsub_lkup_one(tgtname, h_parent); + inode_unlock_shared(h_inode); + return h_dentry; +} + +static void au_call_do_plink_lkup(void *args) +{ + struct au_do_plink_lkup_args *a = args; + *a->errp = au_do_plink_lkup(a->tgtname, a->h_parent, a->br); +} + +/* lookup the plink-ed @inode under the branch at @bindex */ +struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex) +{ + struct dentry *h_dentry, *h_parent; + struct au_branch *br; + int wkq_err; + char a[PLINK_NAME_LEN]; + struct qstr tgtname = QSTR_INIT(a, 0); + + AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM)); + + br = au_sbr(inode->i_sb, bindex); + h_parent = br->br_wbr->wbr_plink; + tgtname.len = plink_name(a, sizeof(a), inode, bindex); + + if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) { + struct au_do_plink_lkup_args args = { + .errp = &h_dentry, + .tgtname = &tgtname, + .h_parent = h_parent, + .br = br + }; + + wkq_err = au_wkq_wait(au_call_do_plink_lkup, &args); + if (unlikely(wkq_err)) + h_dentry = ERR_PTR(wkq_err); + } else + h_dentry = au_do_plink_lkup(&tgtname, h_parent, br); + + return h_dentry; +} + +/* create a pseudo-link */ +static int do_whplink(struct qstr *tgt, struct dentry *h_parent, + struct dentry *h_dentry, struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + struct inode *h_dir, *delegated; + + h_dir = d_inode(h_parent); + inode_lock_nested(h_dir, AuLsc_I_CHILD2); +again: + h_path.dentry = vfsub_lkup_one(tgt, h_parent); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + err = 0; + /* wh.plink dir is not monitored */ + /* todo: is it really safe? */ + if (d_is_positive(h_path.dentry) + && d_inode(h_path.dentry) != d_inode(h_dentry)) { + delegated = NULL; + err = vfsub_unlink(h_dir, &h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + dput(h_path.dentry); + h_path.dentry = NULL; + if (!err) + goto again; + } + if (!err && d_is_negative(h_path.dentry)) { + delegated = NULL; + err = vfsub_link(h_dentry, h_dir, &h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + dput(h_path.dentry); + +out: + inode_unlock(h_dir); + return err; +} + +struct do_whplink_args { + int *errp; + struct qstr *tgt; + struct dentry *h_parent; + struct dentry *h_dentry; + struct au_branch *br; +}; + +static void call_do_whplink(void *args) +{ + struct do_whplink_args *a = args; + *a->errp = do_whplink(a->tgt, a->h_parent, a->h_dentry, a->br); +} + +static int whplink(struct dentry *h_dentry, struct inode *inode, + aufs_bindex_t bindex, struct au_branch *br) +{ + int err, wkq_err; + struct au_wbr *wbr; + struct dentry *h_parent; + char a[PLINK_NAME_LEN]; + struct qstr tgtname = QSTR_INIT(a, 0); + + wbr = au_sbr(inode->i_sb, bindex)->br_wbr; + h_parent = wbr->wbr_plink; + tgtname.len = plink_name(a, sizeof(a), inode, bindex); + + /* always superio. */ + if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) { + struct do_whplink_args args = { + .errp = &err, + .tgt = &tgtname, + .h_parent = h_parent, + .h_dentry = h_dentry, + .br = br + }; + wkq_err = au_wkq_wait(call_do_whplink, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } else + err = do_whplink(&tgtname, h_parent, h_dentry, br); + + return err; +} + +/* + * create a new pseudo-link for @h_dentry on @bindex. + * the linked inode is held in aufs @inode. + */ +void au_plink_append(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry) +{ + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + int found, err, cnt, i; + + sb = inode->i_sb; + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + found = au_plink_test(inode); + if (found) + return; + + i = au_plink_hash(inode->i_ino); + hbl = sbinfo->si_plink + i; + au_igrab(inode); + + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) { + if (&icntnr->vfs_inode == inode) { + found = 1; + break; + } + } + if (!found) { + icntnr = container_of(inode, struct au_icntnr, vfs_inode); + hlist_bl_add_head(&icntnr->plink, hbl); + } + hlist_bl_unlock(hbl); + if (!found) { + cnt = au_hbl_count(hbl); +#define msg "unexpectedly unbalanced or too many pseudo-links" + if (cnt > AUFS_PLINK_WARN) + AuWarn1(msg ", %d\n", cnt); +#undef msg + err = whplink(h_dentry, inode, bindex, au_sbr(sb, bindex)); + if (unlikely(err)) { + pr_warn("err %d, damaged pseudo link.\n", err); + au_hbl_del(&icntnr->plink, hbl); + iput(&icntnr->vfs_inode); + } + } else + iput(&icntnr->vfs_inode); +} + +/* free all plinks */ +void au_plink_put(struct super_block *sb, int verbose) +{ + int i, warned; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_icntnr *icntnr; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + /* no spin_lock since sbinfo is write-locked */ + warned = 0; + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + if (!warned && verbose && !hlist_bl_empty(hbl)) { + pr_warn("pseudo-link is not flushed"); + warned = 1; + } + hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink) + iput(&icntnr->vfs_inode); + INIT_HLIST_BL_HEAD(hbl); + } +} + +void au_plink_clean(struct super_block *sb, int verbose) +{ + struct dentry *root; + + root = sb->s_root; + aufs_write_lock(root); + if (au_opt_test(au_mntflags(sb), PLINK)) + au_plink_put(sb, verbose); + aufs_write_unlock(root); +} + +static int au_plink_do_half_refresh(struct inode *inode, aufs_bindex_t br_id) +{ + int do_put; + aufs_bindex_t btop, bbot, bindex; + + do_put = 0; + btop = au_ibtop(inode); + bbot = au_ibbot(inode); + if (btop >= 0) { + for (bindex = btop; bindex <= bbot; bindex++) { + if (!au_h_iptr(inode, bindex) + || au_ii_br_id(inode, bindex) != br_id) + continue; + au_set_h_iptr(inode, bindex, NULL, 0); + do_put = 1; + break; + } + if (do_put) + for (bindex = btop; bindex <= bbot; bindex++) + if (au_h_iptr(inode, bindex)) { + do_put = 0; + break; + } + } else + do_put = 1; + + return do_put; +} + +/* free the plinks on a branch specified by @br_id */ +void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id) +{ + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_icntnr *icntnr; + struct inode *inode; + int i, do_put; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + /* no bit_lock since sbinfo is write-locked */ + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink) { + inode = au_igrab(&icntnr->vfs_inode); + ii_write_lock_child(inode); + do_put = au_plink_do_half_refresh(inode, br_id); + if (do_put) { + hlist_bl_del(&icntnr->plink); + iput(inode); + } + ii_write_unlock(inode); + iput(inode); + } + } +} diff --git a/fs/aufs/poll.c b/fs/aufs/poll.c new file mode 100644 index 0000000000000..d25202722ef6e --- /dev/null +++ b/fs/aufs/poll.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * poll operation + * There is only one filesystem which implements ->poll operation, currently. + */ + +#include "aufs.h" + +unsigned int aufs_poll(struct file *file, poll_table *wait) +{ + unsigned int mask; + struct file *h_file; + struct super_block *sb; + + /* We should pretend an error happened. */ + mask = POLLERR /* | POLLIN | POLLOUT */; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + if (IS_ERR(h_file)) { + AuDbg("h_file %ld\n", PTR_ERR(h_file)); + goto out; + } + + /* it is not an error if h_file has no operation */ + mask = DEFAULT_POLLMASK; + if (h_file->f_op->poll) + mask = h_file->f_op->poll(h_file, wait); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + if (mask & POLLERR) + AuDbg("mask 0x%x\n", mask); + return mask; +} diff --git a/fs/aufs/posix_acl.c b/fs/aufs/posix_acl.c new file mode 100644 index 0000000000000..64c504002290e --- /dev/null +++ b/fs/aufs/posix_acl.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * posix acl operations + */ + +#include +#include "aufs.h" + +struct posix_acl *aufs_get_acl(struct inode *inode, int type) +{ + struct posix_acl *acl; + int err; + aufs_bindex_t bindex; + struct inode *h_inode; + struct super_block *sb; + + acl = NULL; + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH); + ii_read_lock_child(inode); + if (!(sb->s_flags & MS_POSIXACL)) + goto out; + + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (unlikely(!h_inode + || ((h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT)))) { + err = au_busy_or_stale(); + acl = ERR_PTR(err); + goto out; + } + + /* always topmost only */ + acl = get_acl(h_inode, type); + if (!IS_ERR_OR_NULL(acl)) + set_cached_acl(inode, type, acl); + +out: + ii_read_unlock(inode); + si_read_unlock(sb); + + AuTraceErrPtr(acl); + return acl; +} + +int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + int err; + ssize_t ssz; + struct dentry *dentry; + struct au_sxattr arg = { + .type = AU_ACL_SET, + .u.acl_set = { + .acl = acl, + .type = type + }, + }; + + IMustLock(inode); + + if (inode->i_ino == AUFS_ROOT_INO) + dentry = dget(inode->i_sb->s_root); + else { + dentry = d_find_alias(inode); + if (!dentry) + dentry = d_find_any_alias(inode); + if (!dentry) { + pr_warn("cannot handle this inode, " + "please report to aufs-users ML\n"); + err = -ENOENT; + goto out; + } + } + + ssz = au_sxattr(dentry, inode, &arg); + dput(dentry); + err = ssz; + if (ssz >= 0) { + err = 0; + set_cached_acl(inode, type, acl); + } + +out: + return err; +} diff --git a/fs/aufs/procfs.c b/fs/aufs/procfs.c new file mode 100644 index 0000000000000..582dd486fcd30 --- /dev/null +++ b/fs/aufs/procfs.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * procfs interfaces + */ + +#include +#include "aufs.h" + +static int au_procfs_plm_release(struct inode *inode, struct file *file) +{ + struct au_sbinfo *sbinfo; + + sbinfo = file->private_data; + if (sbinfo) { + au_plink_maint_leave(sbinfo); + kobject_put(&sbinfo->si_kobj); + } + + return 0; +} + +static void au_procfs_plm_write_clean(struct file *file) +{ + struct au_sbinfo *sbinfo; + + sbinfo = file->private_data; + if (sbinfo) + au_plink_clean(sbinfo->si_sb, /*verbose*/0); +} + +static int au_procfs_plm_write_si(struct file *file, unsigned long id) +{ + int err; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct hlist_bl_node *pos; + + err = -EBUSY; + if (unlikely(file->private_data)) + goto out; + + sb = NULL; + /* don't use au_sbilist_lock() here */ + hlist_bl_lock(&au_sbilist); + hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list) + if (id == sysaufs_si_id(sbinfo)) { + if (kobject_get_unless_zero(&sbinfo->si_kobj)) + sb = sbinfo->si_sb; + break; + } + hlist_bl_unlock(&au_sbilist); + + err = -EINVAL; + if (unlikely(!sb)) + goto out; + + err = au_plink_maint_enter(sb); + if (!err) + /* keep kobject_get() */ + file->private_data = sbinfo; + else + kobject_put(&sbinfo->si_kobj); +out: + return err; +} + +/* + * Accept a valid "si=xxxx" only. + * Once it is accepted successfully, accept "clean" too. + */ +static ssize_t au_procfs_plm_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + ssize_t err; + unsigned long id; + /* last newline is allowed */ + char buf[3 + sizeof(unsigned long) * 2 + 1]; + + err = -EACCES; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -EINVAL; + if (unlikely(count > sizeof(buf))) + goto out; + + err = copy_from_user(buf, ubuf, count); + if (unlikely(err)) { + err = -EFAULT; + goto out; + } + buf[count] = 0; + + err = -EINVAL; + if (!strcmp("clean", buf)) { + au_procfs_plm_write_clean(file); + goto out_success; + } else if (unlikely(strncmp("si=", buf, 3))) + goto out; + + err = kstrtoul(buf + 3, 16, &id); + if (unlikely(err)) + goto out; + + err = au_procfs_plm_write_si(file, id); + if (unlikely(err)) + goto out; + +out_success: + err = count; /* success */ +out: + return err; +} + +static const struct file_operations au_procfs_plm_fop = { + .write = au_procfs_plm_write, + .release = au_procfs_plm_release, + .owner = THIS_MODULE +}; + +/* ---------------------------------------------------------------------- */ + +static struct proc_dir_entry *au_procfs_dir; + +void au_procfs_fin(void) +{ + remove_proc_entry(AUFS_PLINK_MAINT_NAME, au_procfs_dir); + remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL); +} + +int __init au_procfs_init(void) +{ + int err; + struct proc_dir_entry *entry; + + err = -ENOMEM; + au_procfs_dir = proc_mkdir(AUFS_PLINK_MAINT_DIR, NULL); + if (unlikely(!au_procfs_dir)) + goto out; + + entry = proc_create(AUFS_PLINK_MAINT_NAME, S_IFREG | 0200, + au_procfs_dir, &au_procfs_plm_fop); + if (unlikely(!entry)) + goto out_dir; + + err = 0; + goto out; /* success */ + + +out_dir: + remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL); +out: + return err; +} diff --git a/fs/aufs/rdu.c b/fs/aufs/rdu.c new file mode 100644 index 0000000000000..bf0233a2887dc --- /dev/null +++ b/fs/aufs/rdu.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * readdir in userspace. + */ + +#include +#include +#include +#include "aufs.h" + +/* bits for struct aufs_rdu.flags */ +#define AuRdu_CALLED 1 +#define AuRdu_CONT (1 << 1) +#define AuRdu_FULL (1 << 2) +#define au_ftest_rdu(flags, name) ((flags) & AuRdu_##name) +#define au_fset_rdu(flags, name) \ + do { (flags) |= AuRdu_##name; } while (0) +#define au_fclr_rdu(flags, name) \ + do { (flags) &= ~AuRdu_##name; } while (0) + +struct au_rdu_arg { + struct dir_context ctx; + struct aufs_rdu *rdu; + union au_rdu_ent_ul ent; + unsigned long end; + + struct super_block *sb; + int err; +}; + +static int au_rdu_fill(struct dir_context *ctx, const char *name, int nlen, + loff_t offset, u64 h_ino, unsigned int d_type) +{ + int err, len; + struct au_rdu_arg *arg = container_of(ctx, struct au_rdu_arg, ctx); + struct aufs_rdu *rdu = arg->rdu; + struct au_rdu_ent ent; + + err = 0; + arg->err = 0; + au_fset_rdu(rdu->cookie.flags, CALLED); + len = au_rdu_len(nlen); + if (arg->ent.ul + len < arg->end) { + ent.ino = h_ino; + ent.bindex = rdu->cookie.bindex; + ent.type = d_type; + ent.nlen = nlen; + if (unlikely(nlen > AUFS_MAX_NAMELEN)) + ent.type = DT_UNKNOWN; + + /* unnecessary to support mmap_sem since this is a dir */ + err = -EFAULT; + if (copy_to_user(arg->ent.e, &ent, sizeof(ent))) + goto out; + if (copy_to_user(arg->ent.e->name, name, nlen)) + goto out; + /* the terminating NULL */ + if (__put_user(0, arg->ent.e->name + nlen)) + goto out; + err = 0; + /* AuDbg("%p, %.*s\n", arg->ent.p, nlen, name); */ + arg->ent.ul += len; + rdu->rent++; + } else { + err = -EFAULT; + au_fset_rdu(rdu->cookie.flags, FULL); + rdu->full = 1; + rdu->tail = arg->ent; + } + +out: + /* AuTraceErr(err); */ + return err; +} + +static int au_rdu_do(struct file *h_file, struct au_rdu_arg *arg) +{ + int err; + loff_t offset; + struct au_rdu_cookie *cookie = &arg->rdu->cookie; + + /* we don't have to care (FMODE_32BITHASH | FMODE_64BITHASH) for ext4 */ + offset = vfsub_llseek(h_file, cookie->h_pos, SEEK_SET); + err = offset; + if (unlikely(offset != cookie->h_pos)) + goto out; + + err = 0; + do { + arg->err = 0; + au_fclr_rdu(cookie->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(h_file, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err + && au_ftest_rdu(cookie->flags, CALLED) + && !au_ftest_rdu(cookie->flags, FULL)); + cookie->h_pos = h_file->f_pos; + +out: + AuTraceErr(err); + return err; +} + +static int au_rdu(struct file *file, struct aufs_rdu *rdu) +{ + int err; + aufs_bindex_t bbot; + struct au_rdu_arg arg = { + .ctx = { + .actor = au_rdu_fill + } + }; + struct dentry *dentry; + struct inode *inode; + struct file *h_file; + struct au_rdu_cookie *cookie = &rdu->cookie; + + err = !access_ok(VERIFY_WRITE, rdu->ent.e, rdu->sz); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + rdu->rent = 0; + rdu->tail = rdu->ent; + rdu->full = 0; + arg.rdu = rdu; + arg.ent = rdu->ent; + arg.end = arg.ent.ul; + arg.end += rdu->sz; + + err = -ENOTDIR; + if (unlikely(!file->f_op->iterate && !file->f_op->iterate_shared)) + goto out; + + err = security_file_permission(file, MAY_READ); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + inode = d_inode(dentry); + inode_lock_shared(inode); + + arg.sb = inode->i_sb; + err = si_read_lock(arg.sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_mtx; + err = au_alive_dir(dentry); + if (unlikely(err)) + goto out_si; + /* todo: reval? */ + fi_read_lock(file); + + err = -EAGAIN; + if (unlikely(au_ftest_rdu(cookie->flags, CONT) + && cookie->generation != au_figen(file))) + goto out_unlock; + + err = 0; + if (!rdu->blk) { + rdu->blk = au_sbi(arg.sb)->si_rdblk; + if (!rdu->blk) + rdu->blk = au_dir_size(file, /*dentry*/NULL); + } + bbot = au_fbtop(file); + if (cookie->bindex < bbot) + cookie->bindex = bbot; + bbot = au_fbbot_dir(file); + /* AuDbg("b%d, b%d\n", cookie->bindex, bbot); */ + for (; !err && cookie->bindex <= bbot; + cookie->bindex++, cookie->h_pos = 0) { + h_file = au_hf_dir(file, cookie->bindex); + if (!h_file) + continue; + + au_fclr_rdu(cookie->flags, FULL); + err = au_rdu_do(h_file, &arg); + AuTraceErr(err); + if (unlikely(au_ftest_rdu(cookie->flags, FULL) || err)) + break; + } + AuDbg("rent %llu\n", rdu->rent); + + if (!err && !au_ftest_rdu(cookie->flags, CONT)) { + rdu->shwh = !!au_opt_test(au_sbi(arg.sb)->si_mntflags, SHWH); + au_fset_rdu(cookie->flags, CONT); + cookie->generation = au_figen(file); + } + + ii_read_lock_child(inode); + fsstack_copy_attr_atime(inode, au_h_iptr(inode, au_ibtop(inode))); + ii_read_unlock(inode); + +out_unlock: + fi_read_unlock(file); +out_si: + si_read_unlock(arg.sb); +out_mtx: + inode_unlock_shared(inode); +out: + AuTraceErr(err); + return err; +} + +static int au_rdu_ino(struct file *file, struct aufs_rdu *rdu) +{ + int err; + ino_t ino; + unsigned long long nent; + union au_rdu_ent_ul *u; + struct au_rdu_ent ent; + struct super_block *sb; + + err = 0; + nent = rdu->nent; + u = &rdu->ent; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + while (nent-- > 0) { + /* unnecessary to support mmap_sem since this is a dir */ + err = copy_from_user(&ent, u->e, sizeof(ent)); + if (!err) + err = !access_ok(VERIFY_WRITE, &u->e->ino, sizeof(ino)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + break; + } + + /* AuDbg("b%d, i%llu\n", ent.bindex, ent.ino); */ + if (!ent.wh) + err = au_ino(sb, ent.bindex, ent.ino, ent.type, &ino); + else + err = au_wh_ino(sb, ent.bindex, ent.ino, ent.type, + &ino); + if (unlikely(err)) { + AuTraceErr(err); + break; + } + + err = __put_user(ino, &u->e->ino); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + break; + } + u->ul += au_rdu_len(ent.nlen); + } + si_read_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_rdu_verify(struct aufs_rdu *rdu) +{ + AuDbg("rdu{%llu, %p, %u | %u | %llu, %u, %u | " + "%llu, b%d, 0x%x, g%u}\n", + rdu->sz, rdu->ent.e, rdu->verify[AufsCtlRduV_SZ], + rdu->blk, + rdu->rent, rdu->shwh, rdu->full, + rdu->cookie.h_pos, rdu->cookie.bindex, rdu->cookie.flags, + rdu->cookie.generation); + + if (rdu->verify[AufsCtlRduV_SZ] == sizeof(*rdu)) + return 0; + + AuDbg("%u:%u\n", + rdu->verify[AufsCtlRduV_SZ], (unsigned int)sizeof(*rdu)); + return -EINVAL; +} + +long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err, e; + struct aufs_rdu rdu; + void __user *p = (void __user *)arg; + + err = copy_from_user(&rdu, p, sizeof(rdu)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + err = au_rdu_verify(&rdu); + if (unlikely(err)) + goto out; + + switch (cmd) { + case AUFS_CTL_RDU: + err = au_rdu(file, &rdu); + if (unlikely(err)) + break; + + e = copy_to_user(p, &rdu, sizeof(rdu)); + if (unlikely(e)) { + err = -EFAULT; + AuTraceErr(err); + } + break; + case AUFS_CTL_RDU_INO: + err = au_rdu_ino(file, &rdu); + break; + + default: + /* err = -ENOTTY; */ + err = -EINVAL; + } + +out: + AuTraceErr(err); + return err; +} + +#ifdef CONFIG_COMPAT +long au_rdu_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err, e; + struct aufs_rdu rdu; + void __user *p = compat_ptr(arg); + + /* todo: get_user()? */ + err = copy_from_user(&rdu, p, sizeof(rdu)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + rdu.ent.e = compat_ptr(rdu.ent.ul); + err = au_rdu_verify(&rdu); + if (unlikely(err)) + goto out; + + switch (cmd) { + case AUFS_CTL_RDU: + err = au_rdu(file, &rdu); + if (unlikely(err)) + break; + + rdu.ent.ul = ptr_to_compat(rdu.ent.e); + rdu.tail.ul = ptr_to_compat(rdu.tail.e); + e = copy_to_user(p, &rdu, sizeof(rdu)); + if (unlikely(e)) { + err = -EFAULT; + AuTraceErr(err); + } + break; + case AUFS_CTL_RDU_INO: + err = au_rdu_ino(file, &rdu); + break; + + default: + /* err = -ENOTTY; */ + err = -EINVAL; + } + +out: + AuTraceErr(err); + return err; +} +#endif diff --git a/fs/aufs/rwsem.h b/fs/aufs/rwsem.h new file mode 100644 index 0000000000000..2f8988fa1bda5 --- /dev/null +++ b/fs/aufs/rwsem.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * simple read-write semaphore wrappers + */ + +#ifndef __AUFS_RWSEM_H__ +#define __AUFS_RWSEM_H__ + +#ifdef __KERNEL__ + +#include "debug.h" + +/* in the future, the name 'au_rwsem' will be totally gone */ +#define au_rwsem rw_semaphore + +/* to debug easier, do not make them inlined functions */ +#define AuRwMustNoWaiters(rw) AuDebugOn(rwsem_is_contended(rw)) +/* rwsem_is_locked() is unusable */ +#define AuRwMustReadLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held_type(rw, 1)) +#define AuRwMustWriteLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held_type(rw, 0)) +#define AuRwMustAnyLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held(rw)) +#define AuRwDestroy(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && lockdep_is_held(rw)) + +#define au_rw_init(rw) init_rwsem(rw) + +#define au_rw_init_wlock(rw) do { \ + au_rw_init(rw); \ + down_write(rw); \ + } while (0) + +#define au_rw_init_wlock_nested(rw, lsc) do { \ + au_rw_init(rw); \ + down_write_nested(rw, lsc); \ + } while (0) + +#define au_rw_read_lock(rw) down_read(rw) +#define au_rw_read_lock_nested(rw, lsc) down_read_nested(rw, lsc) +#define au_rw_read_unlock(rw) up_read(rw) +#define au_rw_dgrade_lock(rw) downgrade_write(rw) +#define au_rw_write_lock(rw) down_write(rw) +#define au_rw_write_lock_nested(rw, lsc) down_write_nested(rw, lsc) +#define au_rw_write_unlock(rw) up_write(rw) +/* why is not _nested version defined? */ +#define au_rw_read_trylock(rw) down_read_trylock(rw) +#define au_rw_write_trylock(rw) down_write_trylock(rw) + +#endif /* __KERNEL__ */ +#endif /* __AUFS_RWSEM_H__ */ diff --git a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c new file mode 100644 index 0000000000000..7318df0fe3d6d --- /dev/null +++ b/fs/aufs/sbinfo.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * superblock private data + */ + +#include "aufs.h" + +/* + * they are necessary regardless sysfs is disabled. + */ +void au_si_free(struct kobject *kobj) +{ + int i; + struct au_sbinfo *sbinfo; + char *locked __maybe_unused; /* debug only */ + + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + for (i = 0; i < AuPlink_NHASH; i++) + AuDebugOn(!hlist_bl_empty(sbinfo->si_plink + i)); + AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len)); + + AuLCntZero(au_lcnt_read(&sbinfo->si_ninodes, /*do_rev*/0)); + au_lcnt_fin(&sbinfo->si_ninodes, /*do_sync*/0); + AuLCntZero(au_lcnt_read(&sbinfo->si_nfiles, /*do_rev*/0)); + au_lcnt_fin(&sbinfo->si_nfiles, /*do_sync*/0); + + dbgaufs_si_fin(sbinfo); + au_rw_write_lock(&sbinfo->si_rwsem); + au_br_free(sbinfo); + au_rw_write_unlock(&sbinfo->si_rwsem); + + au_kfree_try_rcu(sbinfo->si_branch); + mutex_destroy(&sbinfo->si_xib_mtx); + AuRwDestroy(&sbinfo->si_rwsem); + + au_lcnt_wait_for_fin(&sbinfo->si_ninodes); + /* si_nfiles is waited too */ + au_kfree_rcu(sbinfo); +} + +int au_si_alloc(struct super_block *sb) +{ + int err, i; + struct au_sbinfo *sbinfo; + + err = -ENOMEM; + sbinfo = kzalloc(sizeof(*sbinfo), GFP_NOFS); + if (unlikely(!sbinfo)) + goto out; + + /* will be reallocated separately */ + sbinfo->si_branch = kzalloc(sizeof(*sbinfo->si_branch), GFP_NOFS); + if (unlikely(!sbinfo->si_branch)) + goto out_sbinfo; + + err = sysaufs_si_init(sbinfo); + if (!err) { + dbgaufs_si_null(sbinfo); + err = dbgaufs_si_init(sbinfo); + if (unlikely(err)) + kobject_put(&sbinfo->si_kobj); + } + if (unlikely(err)) + goto out_br; + + au_nwt_init(&sbinfo->si_nowait); + au_rw_init_wlock(&sbinfo->si_rwsem); + + au_lcnt_init(&sbinfo->si_ninodes, /*release*/NULL); + au_lcnt_init(&sbinfo->si_nfiles, /*release*/NULL); + + sbinfo->si_bbot = -1; + sbinfo->si_last_br_id = AUFS_BRANCH_MAX / 2; + + sbinfo->si_wbr_copyup = AuWbrCopyup_Def; + sbinfo->si_wbr_create = AuWbrCreate_Def; + sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + sbinfo->si_wbr_copyup; + sbinfo->si_wbr_create_ops = au_wbr_create_ops + sbinfo->si_wbr_create; + + au_fhsm_init(sbinfo); + + sbinfo->si_mntflags = au_opts_plink(AuOpt_Def); + + sbinfo->si_xino_jiffy = jiffies; + sbinfo->si_xino_expire + = msecs_to_jiffies(AUFS_XINO_DEF_SEC * MSEC_PER_SEC); + mutex_init(&sbinfo->si_xib_mtx); + /* leave si_xib_last_pindex and si_xib_next_bit */ + + INIT_HLIST_BL_HEAD(&sbinfo->si_aopen); + + sbinfo->si_rdcache = msecs_to_jiffies(AUFS_RDCACHE_DEF * MSEC_PER_SEC); + sbinfo->si_rdblk = AUFS_RDBLK_DEF; + sbinfo->si_rdhash = AUFS_RDHASH_DEF; + sbinfo->si_dirwh = AUFS_DIRWH_DEF; + + for (i = 0; i < AuPlink_NHASH; i++) + INIT_HLIST_BL_HEAD(sbinfo->si_plink + i); + init_waitqueue_head(&sbinfo->si_plink_wq); + spin_lock_init(&sbinfo->si_plink_maint_lock); + + INIT_HLIST_BL_HEAD(&sbinfo->si_files); + + /* with getattr by default */ + sbinfo->si_iop_array = aufs_iop; + + /* leave other members for sysaufs and si_mnt. */ + sbinfo->si_sb = sb; + sb->s_fs_info = sbinfo; + si_pid_set(sb); + return 0; /* success */ + +out_br: + au_kfree_try_rcu(sbinfo->si_branch); +out_sbinfo: + au_kfree_rcu(sbinfo); +out: + return err; +} + +int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_branch **brp; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + err = -ENOMEM; + sz = sizeof(*brp) * (sbinfo->si_bbot + 1); + if (unlikely(!sz)) + sz = sizeof(*brp); + brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS, + may_shrink); + if (brp) { + sbinfo->si_branch = brp; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +unsigned int au_sigen_inc(struct super_block *sb) +{ + unsigned int gen; + struct inode *inode; + + SiMustWriteLock(sb); + + gen = ++au_sbi(sb)->si_generation; + au_update_digen(sb->s_root); + inode = d_inode(sb->s_root); + au_update_iigen(inode, /*half*/0); + inode->i_version++; + return gen; +} + +aufs_bindex_t au_new_br_id(struct super_block *sb) +{ + aufs_bindex_t br_id; + int i; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + for (i = 0; i <= AUFS_BRANCH_MAX; i++) { + br_id = ++sbinfo->si_last_br_id; + AuDebugOn(br_id < 0); + if (br_id && au_br_index(sb, br_id) < 0) + return br_id; + } + + return -1; +} + +/* ---------------------------------------------------------------------- */ + +/* it is ok that new 'nwt' tasks are appended while we are sleeping */ +int si_read_lock(struct super_block *sb, int flags) +{ + int err; + + err = 0; + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + + si_noflush_read_lock(sb); + err = au_plink_maint(sb, flags); + if (unlikely(err)) + si_read_unlock(sb); + + return err; +} + +int si_write_lock(struct super_block *sb, int flags) +{ + int err; + + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + + si_noflush_write_lock(sb); + err = au_plink_maint(sb, flags); + if (unlikely(err)) + si_write_unlock(sb); + + return err; +} + +/* dentry and super_block lock. call at entry point */ +int aufs_read_lock(struct dentry *dentry, int flags) +{ + int err; + struct super_block *sb; + + sb = dentry->d_sb; + err = si_read_lock(sb, flags); + if (unlikely(err)) + goto out; + + if (au_ftest_lock(flags, DW)) + di_write_lock_child(dentry); + else + di_read_lock_child(dentry, flags); + + if (au_ftest_lock(flags, GEN)) { + err = au_digen_test(dentry, au_sigen(sb)); + if (!au_opt_test(au_mntflags(sb), UDBA_NONE)) + AuDebugOn(!err && au_dbrange_test(dentry)); + else if (!err) + err = au_dbrange_test(dentry); + if (unlikely(err)) + aufs_read_unlock(dentry, flags); + } + +out: + return err; +} + +void aufs_read_unlock(struct dentry *dentry, int flags) +{ + if (au_ftest_lock(flags, DW)) + di_write_unlock(dentry); + else + di_read_unlock(dentry, flags); + si_read_unlock(dentry->d_sb); +} + +void aufs_write_lock(struct dentry *dentry) +{ + si_write_lock(dentry->d_sb, AuLock_FLUSH | AuLock_NOPLMW); + di_write_lock_child(dentry); +} + +void aufs_write_unlock(struct dentry *dentry) +{ + di_write_unlock(dentry); + si_write_unlock(dentry->d_sb); +} + +int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags) +{ + int err; + unsigned int sigen; + struct super_block *sb; + + sb = d1->d_sb; + err = si_read_lock(sb, flags); + if (unlikely(err)) + goto out; + + di_write_lock2_child(d1, d2, au_ftest_lock(flags, DIRS)); + + if (au_ftest_lock(flags, GEN)) { + sigen = au_sigen(sb); + err = au_digen_test(d1, sigen); + AuDebugOn(!err && au_dbrange_test(d1)); + if (!err) { + err = au_digen_test(d2, sigen); + AuDebugOn(!err && au_dbrange_test(d2)); + } + if (unlikely(err)) + aufs_read_and_write_unlock2(d1, d2); + } + +out: + return err; +} + +void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2) +{ + di_write_unlock2(d1, d2); + si_read_unlock(d1->d_sb); +} diff --git a/fs/aufs/super.c b/fs/aufs/super.c new file mode 100644 index 0000000000000..25d8b27ecee5c --- /dev/null +++ b/fs/aufs/super.c @@ -0,0 +1,1043 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount and super_block operations + */ + +#include +#include +#include +#include +#include "aufs.h" + +/* + * super_operations + */ +static struct inode *aufs_alloc_inode(struct super_block *sb __maybe_unused) +{ + struct au_icntnr *c; + + c = au_cache_alloc_icntnr(); + if (c) { + au_icntnr_init(c); + c->vfs_inode.i_version = 1; /* sigen(sb); */ + c->iinfo.ii_hinode = NULL; + return &c->vfs_inode; + } + return NULL; +} + +static void aufs_destroy_inode_cb(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + + au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); +} + +static void aufs_destroy_inode(struct inode *inode) +{ + if (!au_is_bad_inode(inode)) + au_iinfo_fin(inode); + call_rcu(&inode->i_rcu, aufs_destroy_inode_cb); +} + +struct inode *au_iget_locked(struct super_block *sb, ino_t ino) +{ + struct inode *inode; + int err; + + inode = iget_locked(sb, ino); + if (unlikely(!inode)) { + inode = ERR_PTR(-ENOMEM); + goto out; + } + if (!(inode->i_state & I_NEW)) + goto out; + + err = au_xigen_new(inode); + if (!err) + err = au_iinfo_init(inode); + if (!err) + inode->i_version++; + else { + iget_failed(inode); + inode = ERR_PTR(err); + } + +out: + /* never return NULL */ + AuDebugOn(!inode); + AuTraceErrPtr(inode); + return inode; +} + +/* lock free root dinfo */ +static int au_show_brs(struct seq_file *seq, struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct path path; + struct au_hdentry *hdp; + struct au_branch *br; + au_br_perm_str_t perm; + + err = 0; + bbot = au_sbbot(sb); + bindex = 0; + hdp = au_hdentry(au_di(sb->s_root), bindex); + for (; !err && bindex <= bbot; bindex++, hdp++) { + br = au_sbr(sb, bindex); + path.mnt = au_br_mnt(br); + path.dentry = hdp->hd_dentry; + err = au_seq_path(seq, &path); + if (!err) { + au_optstr_br_perm(&perm, br->br_perm); + seq_printf(seq, "=%s", perm.a); + if (bindex != bbot) + seq_putc(seq, ':'); + } + } + if (unlikely(err || seq_has_overflowed(seq))) + err = -E2BIG; + + return err; +} + +static void au_gen_fmt(char *fmt, int len __maybe_unused, const char *pat, + const char *append) +{ + char *p; + + p = fmt; + while (*pat != ':') + *p++ = *pat++; + *p++ = *pat++; + strcpy(p, append); + AuDebugOn(strlen(fmt) >= len); +} + +static void au_show_wbr_create(struct seq_file *m, int v, + struct au_sbinfo *sbinfo) +{ + const char *pat; + char fmt[32]; + struct au_wbr_mfs *mfs; + + AuRwMustAnyLock(&sbinfo->si_rwsem); + + seq_puts(m, ",create="); + pat = au_optstr_wbr_create(v); + mfs = &sbinfo->si_wbr_mfs; + switch (v) { + case AuWbrCreate_TDP: + case AuWbrCreate_RR: + case AuWbrCreate_MFS: + case AuWbrCreate_PMFS: + seq_puts(m, pat); + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_PMFSRR: + au_gen_fmt(fmt, sizeof(fmt), pat, "%llu"); + seq_printf(m, fmt, mfs->mfsrr_watermark); + break; + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + au_gen_fmt(fmt, sizeof(fmt), pat, "%lu"); + seq_printf(m, fmt, + jiffies_to_msecs(mfs->mfs_expire) + / MSEC_PER_SEC); + break; + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + au_gen_fmt(fmt, sizeof(fmt), pat, "%llu:%lu"); + seq_printf(m, fmt, mfs->mfsrr_watermark, + jiffies_to_msecs(mfs->mfs_expire) / MSEC_PER_SEC); + break; + default: + BUG(); + } +} + +static int au_show_xino(struct seq_file *seq, struct super_block *sb) +{ +#ifdef CONFIG_SYSFS + return 0; +#else + int err; + const int len = sizeof(AUFS_XINO_FNAME) - 1; + aufs_bindex_t bindex, brid; + struct qstr *name; + struct file *f; + struct dentry *d, *h_root; + struct au_branch *br; + + AuRwMustAnyLock(&sbinfo->si_rwsem); + + err = 0; + f = au_sbi(sb)->si_xib; + if (!f) + goto out; + + /* stop printing the default xino path on the first writable branch */ + h_root = NULL; + bindex = au_xi_root(sb, f->f_path.dentry); + if (bindex >= 0) { + br = au_sbr_sb(sb, bindex); + h_root = au_br_dentry(br); + } + + d = f->f_path.dentry; + name = &d->d_name; + /* safe ->d_parent because the file is unlinked */ + if (d->d_parent == h_root + && name->len == len + && !memcmp(name->name, AUFS_XINO_FNAME, len)) + goto out; + + seq_puts(seq, ",xino="); + err = au_xino_path(seq, f); + +out: + return err; +#endif +} + +/* seq_file will re-call me in case of too long string */ +static int aufs_show_options(struct seq_file *m, struct dentry *dentry) +{ + int err; + unsigned int mnt_flags, v; + struct super_block *sb; + struct au_sbinfo *sbinfo; + +#define AuBool(name, str) do { \ + v = au_opt_test(mnt_flags, name); \ + if (v != au_opt_test(AuOpt_Def, name)) \ + seq_printf(m, ",%s" #str, v ? "" : "no"); \ +} while (0) + +#define AuStr(name, str) do { \ + v = mnt_flags & AuOptMask_##name; \ + if (v != (AuOpt_Def & AuOptMask_##name)) \ + seq_printf(m, "," #str "=%s", au_optstr_##str(v)); \ +} while (0) + +#define AuUInt(name, str, val) do { \ + if (val != AUFS_##name##_DEF) \ + seq_printf(m, "," #str "=%u", val); \ +} while (0) + + sb = dentry->d_sb; + if (sb->s_flags & MS_POSIXACL) + seq_puts(m, ",acl"); + + /* lock free root dinfo */ + si_noflush_read_lock(sb); + sbinfo = au_sbi(sb); + seq_printf(m, ",si=%lx", sysaufs_si_id(sbinfo)); + + mnt_flags = au_mntflags(sb); + if (au_opt_test(mnt_flags, XINO)) { + err = au_show_xino(m, sb); + if (unlikely(err)) + goto out; + } else + seq_puts(m, ",noxino"); + + AuBool(TRUNC_XINO, trunc_xino); + AuStr(UDBA, udba); + AuBool(SHWH, shwh); + AuBool(PLINK, plink); + AuBool(DIO, dio); + AuBool(DIRPERM1, dirperm1); + + v = sbinfo->si_wbr_create; + if (v != AuWbrCreate_Def) + au_show_wbr_create(m, v, sbinfo); + + v = sbinfo->si_wbr_copyup; + if (v != AuWbrCopyup_Def) + seq_printf(m, ",cpup=%s", au_optstr_wbr_copyup(v)); + + v = au_opt_test(mnt_flags, ALWAYS_DIROPQ); + if (v != au_opt_test(AuOpt_Def, ALWAYS_DIROPQ)) + seq_printf(m, ",diropq=%c", v ? 'a' : 'w'); + + AuUInt(DIRWH, dirwh, sbinfo->si_dirwh); + + v = jiffies_to_msecs(sbinfo->si_rdcache) / MSEC_PER_SEC; + AuUInt(RDCACHE, rdcache, v); + + AuUInt(RDBLK, rdblk, sbinfo->si_rdblk); + AuUInt(RDHASH, rdhash, sbinfo->si_rdhash); + + au_fhsm_show(m, sbinfo); + + AuBool(DIRREN, dirren); + AuBool(SUM, sum); + /* AuBool(SUM_W, wsum); */ + AuBool(WARN_PERM, warn_perm); + AuBool(VERBOSE, verbose); + +out: + /* be sure to print "br:" last */ + if (!sysaufs_brs) { + seq_puts(m, ",br:"); + au_show_brs(m, sb); + } + si_read_unlock(sb); + return 0; + +#undef AuBool +#undef AuStr +#undef AuUInt +} + +/* ---------------------------------------------------------------------- */ + +/* sum mode which returns the summation for statfs(2) */ + +static u64 au_add_till_max(u64 a, u64 b) +{ + u64 old; + + old = a; + a += b; + if (old <= a) + return a; + return ULLONG_MAX; +} + +static u64 au_mul_till_max(u64 a, long mul) +{ + u64 old; + + old = a; + a *= mul; + if (old <= a) + return a; + return ULLONG_MAX; +} + +static int au_statfs_sum(struct super_block *sb, struct kstatfs *buf) +{ + int err; + long bsize, factor; + u64 blocks, bfree, bavail, files, ffree; + aufs_bindex_t bbot, bindex, i; + unsigned char shared; + struct path h_path; + struct super_block *h_sb; + + err = 0; + bsize = LONG_MAX; + files = 0; + ffree = 0; + blocks = 0; + bfree = 0; + bavail = 0; + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + h_path.mnt = au_sbr_mnt(sb, bindex); + h_sb = h_path.mnt->mnt_sb; + shared = 0; + for (i = 0; !shared && i < bindex; i++) + shared = (au_sbr_sb(sb, i) == h_sb); + if (shared) + continue; + + /* sb->s_root for NFS is unreliable */ + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, buf); + if (unlikely(err)) + goto out; + + if (bsize > buf->f_bsize) { + /* + * we will reduce bsize, so we have to expand blocks + * etc. to match them again + */ + factor = (bsize / buf->f_bsize); + blocks = au_mul_till_max(blocks, factor); + bfree = au_mul_till_max(bfree, factor); + bavail = au_mul_till_max(bavail, factor); + bsize = buf->f_bsize; + } + + factor = (buf->f_bsize / bsize); + blocks = au_add_till_max(blocks, + au_mul_till_max(buf->f_blocks, factor)); + bfree = au_add_till_max(bfree, + au_mul_till_max(buf->f_bfree, factor)); + bavail = au_add_till_max(bavail, + au_mul_till_max(buf->f_bavail, factor)); + files = au_add_till_max(files, buf->f_files); + ffree = au_add_till_max(ffree, buf->f_ffree); + } + + buf->f_bsize = bsize; + buf->f_blocks = blocks; + buf->f_bfree = bfree; + buf->f_bavail = bavail; + buf->f_files = files; + buf->f_ffree = ffree; + buf->f_frsize = 0; + +out: + return err; +} + +static int aufs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + int err; + struct path h_path; + struct super_block *sb; + + /* lock free root dinfo */ + sb = dentry->d_sb; + si_noflush_read_lock(sb); + if (!au_opt_test(au_mntflags(sb), SUM)) { + /* sb->s_root for NFS is unreliable */ + h_path.mnt = au_sbr_mnt(sb, 0); + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, buf); + } else + err = au_statfs_sum(sb, buf); + si_read_unlock(sb); + + if (!err) { + buf->f_type = AUFS_SUPER_MAGIC; + buf->f_namelen = AUFS_MAX_NAMELEN; + memset(&buf->f_fsid, 0, sizeof(buf->f_fsid)); + } + /* buf->f_bsize = buf->f_blocks = buf->f_bfree = buf->f_bavail = -1; */ + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_sync_fs(struct super_block *sb, int wait) +{ + int err, e; + aufs_bindex_t bbot, bindex; + struct au_branch *br; + struct super_block *h_sb; + + err = 0; + si_noflush_read_lock(sb); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_writable(br->br_perm)) + continue; + + h_sb = au_sbr_sb(sb, bindex); + e = vfsub_sync_filesystem(h_sb, wait); + if (unlikely(e && !err)) + err = e; + /* go on even if an error happens */ + } + si_read_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* final actions when unmounting a file system */ +static void aufs_put_super(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + sbinfo = au_sbi(sb); + if (sbinfo) + kobject_put(&sbinfo->si_kobj); +} + +/* ---------------------------------------------------------------------- */ + +void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb, + struct super_block *sb, void *arg) +{ + void *array; + unsigned long long n, sz; + + array = NULL; + n = 0; + if (!*hint) + goto out; + + if (*hint > ULLONG_MAX / sizeof(array)) { + array = ERR_PTR(-EMFILE); + pr_err("hint %llu\n", *hint); + goto out; + } + + sz = sizeof(array) * *hint; + array = kzalloc(sz, GFP_NOFS); + if (unlikely(!array)) + array = vzalloc(sz); + if (unlikely(!array)) { + array = ERR_PTR(-ENOMEM); + goto out; + } + + n = cb(sb, array, *hint, arg); + AuDebugOn(n > *hint); + +out: + *hint = n; + return array; +} + +static unsigned long long au_iarray_cb(struct super_block *sb, void *a, + unsigned long long max __maybe_unused, + void *arg) +{ + unsigned long long n; + struct inode **p, *inode; + struct list_head *head; + + n = 0; + p = a; + head = arg; + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, head, i_sb_list) { + if (!au_is_bad_inode(inode) + && au_ii(inode)->ii_btop >= 0) { + spin_lock(&inode->i_lock); + if (atomic_read(&inode->i_count)) { + au_igrab(inode); + *p++ = inode; + n++; + AuDebugOn(n > max); + } + spin_unlock(&inode->i_lock); + } + } + spin_unlock(&sb->s_inode_list_lock); + + return n; +} + +struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max) +{ + struct au_sbinfo *sbi; + + sbi = au_sbi(sb); + *max = au_lcnt_read(&sbi->si_ninodes, /*do_rev*/1); + return au_array_alloc(max, au_iarray_cb, sb, &sb->s_inodes); +} + +void au_iarray_free(struct inode **a, unsigned long long max) +{ + unsigned long long ull; + + for (ull = 0; ull < max; ull++) + iput(a[ull]); + kvfree(a); +} + +/* ---------------------------------------------------------------------- */ + +/* + * refresh dentry and inode at remount time. + */ +/* todo: consolidate with simple_reval_dpath() and au_reval_for_attr() */ +static int au_do_refresh(struct dentry *dentry, unsigned int dir_flags, + struct dentry *parent) +{ + int err; + + di_write_lock_child(dentry); + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(dentry, parent); + if (!err && dir_flags) + au_hn_reset(d_inode(dentry), dir_flags); + di_read_unlock(parent, AuLock_IR); + di_write_unlock(dentry); + + return err; +} + +static int au_do_refresh_d(struct dentry *dentry, unsigned int sigen, + struct au_sbinfo *sbinfo, + const unsigned int dir_flags, unsigned int do_idop) +{ + int err; + struct dentry *parent; + + err = 0; + parent = dget_parent(dentry); + if (!au_digen_test(parent, sigen) && au_digen_test(dentry, sigen)) { + if (d_really_is_positive(dentry)) { + if (!d_is_dir(dentry)) + err = au_do_refresh(dentry, /*dir_flags*/0, + parent); + else { + err = au_do_refresh(dentry, dir_flags, parent); + if (unlikely(err)) + au_fset_si(sbinfo, FAILED_REFRESH_DIR); + } + } else + err = au_do_refresh(dentry, /*dir_flags*/0, parent); + AuDbgDentry(dentry); + } + dput(parent); + + if (!err) { + if (do_idop) + au_refresh_dop(dentry, /*force_reval*/0); + } else + au_refresh_dop(dentry, /*force_reval*/1); + + AuTraceErr(err); + return err; +} + +static int au_refresh_d(struct super_block *sb, unsigned int do_idop) +{ + int err, i, j, ndentry, e; + unsigned int sigen; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries, *d; + struct au_sbinfo *sbinfo; + struct dentry *root = sb->s_root; + const unsigned int dir_flags = au_hi_flags(d_inode(root), /*isdir*/1); + + if (do_idop) + au_refresh_dop(root, /*force_reval*/0); + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, root, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + sigen = au_sigen(sb); + sbinfo = au_sbi(sb); + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + d = dentries[j]; + e = au_do_refresh_d(d, sigen, sbinfo, dir_flags, + do_idop); + if (unlikely(e && !err)) + err = e; + /* go on even err */ + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static int au_refresh_i(struct super_block *sb, unsigned int do_idop) +{ + int err, e; + unsigned int sigen; + unsigned long long max, ull; + struct inode *inode, **array; + + array = au_iarray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + sigen = au_sigen(sb); + for (ull = 0; ull < max; ull++) { + inode = array[ull]; + if (unlikely(!inode)) + break; + + e = 0; + ii_write_lock_child(inode); + if (au_iigen(inode, NULL) != sigen) { + e = au_refresh_hinode_self(inode); + if (unlikely(e)) { + au_refresh_iop(inode, /*force_getattr*/1); + pr_err("error %d, i%lu\n", e, inode->i_ino); + if (!err) + err = e; + /* go on even if err */ + } + } + if (!e && do_idop) + au_refresh_iop(inode, /*force_getattr*/0); + ii_write_unlock(inode); + } + + au_iarray_free(array, max); + +out: + return err; +} + +static void au_remount_refresh(struct super_block *sb, unsigned int do_idop) +{ + int err, e; + unsigned int udba; + aufs_bindex_t bindex, bbot; + struct dentry *root; + struct inode *inode; + struct au_branch *br; + struct au_sbinfo *sbi; + + au_sigen_inc(sb); + sbi = au_sbi(sb); + au_fclr_si(sbi, FAILED_REFRESH_DIR); + + root = sb->s_root; + DiMustNoWaiters(root); + inode = d_inode(root); + IiMustNoWaiters(inode); + + udba = au_opt_udba(sb); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_hnotify_reset_br(udba, br, br->br_perm); + if (unlikely(err)) + AuIOErr("hnotify failed on br %d, %d, ignored\n", + bindex, err); + /* go on even if err */ + } + au_hn_reset(inode, au_hi_flags(inode, /*isdir*/1)); + + if (do_idop) { + if (au_ftest_si(sbi, NO_DREVAL)) { + AuDebugOn(sb->s_d_op == &aufs_dop_noreval); + sb->s_d_op = &aufs_dop_noreval; + AuDebugOn(sbi->si_iop_array == aufs_iop_nogetattr); + sbi->si_iop_array = aufs_iop_nogetattr; + } else { + AuDebugOn(sb->s_d_op == &aufs_dop); + sb->s_d_op = &aufs_dop; + AuDebugOn(sbi->si_iop_array == aufs_iop); + sbi->si_iop_array = aufs_iop; + } + pr_info("reset to %pf and %pf\n", + sb->s_d_op, sbi->si_iop_array); + } + + di_write_unlock(root); + err = au_refresh_d(sb, do_idop); + e = au_refresh_i(sb, do_idop); + if (unlikely(e && !err)) + err = e; + /* aufs_write_lock() calls ..._child() */ + di_write_lock_child(root); + + au_cpup_attr_all(inode, /*force*/1); + + if (unlikely(err)) + AuIOErr("refresh failed, ignored, %d\n", err); +} + +/* stop extra interpretation of errno in mount(8), and strange error messages */ +static int cvt_err(int err) +{ + AuTraceErr(err); + + switch (err) { + case -ENOENT: + case -ENOTDIR: + case -EEXIST: + case -EIO: + err = -EINVAL; + } + return err; +} + +static int aufs_remount_fs(struct super_block *sb, int *flags, char *data) +{ + int err, do_dx; + unsigned int mntflags; + struct au_opts opts = { + .opt = NULL + }; + struct dentry *root; + struct inode *inode; + struct au_sbinfo *sbinfo; + + err = 0; + root = sb->s_root; + if (!data || !*data) { + err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (!err) { + di_write_lock_child(root); + err = au_opts_verify(sb, *flags, /*pending*/0); + aufs_write_unlock(root); + } + goto out; + } + + err = -ENOMEM; + opts.opt = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!opts.opt)) + goto out; + opts.max_opt = PAGE_SIZE / sizeof(*opts.opt); + opts.flags = AuOpts_REMOUNT; + opts.sb_flags = *flags; + + /* parse it before aufs lock */ + err = au_opts_parse(sb, data, &opts); + if (unlikely(err)) + goto out_opts; + + sbinfo = au_sbi(sb); + inode = d_inode(root); + inode_lock(inode); + err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_mtx; + di_write_lock_child(root); + + /* au_opts_remount() may return an error */ + err = au_opts_remount(sb, &opts); + au_opts_free(&opts); + + if (au_ftest_opts(opts.flags, REFRESH)) + au_remount_refresh(sb, au_ftest_opts(opts.flags, REFRESH_IDOP)); + + if (au_ftest_opts(opts.flags, REFRESH_DYAOP)) { + mntflags = au_mntflags(sb); + do_dx = !!au_opt_test(mntflags, DIO); + au_dy_arefresh(do_dx); + } + + au_fhsm_wrote_all(sb, /*force*/1); /* ?? */ + aufs_write_unlock(root); + +out_mtx: + inode_unlock(inode); +out_opts: + free_page((unsigned long)opts.opt); +out: + err = cvt_err(err); + AuTraceErr(err); + return err; +} + +static const struct super_operations aufs_sop = { + .alloc_inode = aufs_alloc_inode, + .destroy_inode = aufs_destroy_inode, + /* always deleting, no clearing */ + .drop_inode = generic_delete_inode, + .show_options = aufs_show_options, + .statfs = aufs_statfs, + .put_super = aufs_put_super, + .sync_fs = aufs_sync_fs, + .remount_fs = aufs_remount_fs +}; + +/* ---------------------------------------------------------------------- */ + +static int alloc_root(struct super_block *sb) +{ + int err; + struct inode *inode; + struct dentry *root; + + err = -ENOMEM; + inode = au_iget_locked(sb, AUFS_ROOT_INO); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out; + + inode->i_op = aufs_iop + AuIop_DIR; /* with getattr by default */ + inode->i_fop = &aufs_dir_fop; + inode->i_mode = S_IFDIR; + set_nlink(inode, 2); + unlock_new_inode(inode); + + root = d_make_root(inode); + if (unlikely(!root)) + goto out; + err = PTR_ERR(root); + if (IS_ERR(root)) + goto out; + + err = au_di_init(root); + if (!err) { + sb->s_root = root; + return 0; /* success */ + } + dput(root); + +out: + return err; +} + +static int aufs_fill_super(struct super_block *sb, void *raw_data, + int silent __maybe_unused) +{ + int err; + struct au_opts opts = { + .opt = NULL + }; + struct au_sbinfo *sbinfo; + struct dentry *root; + struct inode *inode; + char *arg = raw_data; + + if (unlikely(!arg || !*arg)) { + err = -EINVAL; + pr_err("no arg\n"); + goto out; + } + + err = -ENOMEM; + opts.opt = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!opts.opt)) + goto out; + opts.max_opt = PAGE_SIZE / sizeof(*opts.opt); + opts.sb_flags = sb->s_flags; + + err = au_si_alloc(sb); + if (unlikely(err)) + goto out_opts; + sbinfo = au_sbi(sb); + + /* all timestamps always follow the ones on the branch */ + sb->s_flags |= MS_NOATIME | MS_NODIRATIME; + sb->s_op = &aufs_sop; + sb->s_d_op = &aufs_dop; + sb->s_magic = AUFS_SUPER_MAGIC; + sb->s_maxbytes = 0; + sb->s_stack_depth = 1; + au_export_init(sb); + au_xattr_init(sb); + + err = alloc_root(sb); + if (unlikely(err)) { + si_write_unlock(sb); + goto out_info; + } + root = sb->s_root; + inode = d_inode(root); + + /* + * actually we can parse options regardless aufs lock here. + * but at remount time, parsing must be done before aufs lock. + * so we follow the same rule. + */ + ii_write_lock_parent(inode); + aufs_write_unlock(root); + err = au_opts_parse(sb, arg, &opts); + if (unlikely(err)) + goto out_root; + + /* lock vfs_inode first, then aufs. */ + inode_lock(inode); + aufs_write_lock(root); + err = au_opts_mount(sb, &opts); + au_opts_free(&opts); + if (!err && au_ftest_si(sbinfo, NO_DREVAL)) { + sb->s_d_op = &aufs_dop_noreval; + pr_info("%pf\n", sb->s_d_op); + au_refresh_dop(root, /*force_reval*/0); + sbinfo->si_iop_array = aufs_iop_nogetattr; + au_refresh_iop(inode, /*force_getattr*/0); + } + aufs_write_unlock(root); + inode_unlock(inode); + if (!err) + goto out_opts; /* success */ + +out_root: + dput(root); + sb->s_root = NULL; +out_info: + kobject_put(&sbinfo->si_kobj); + sb->s_fs_info = NULL; +out_opts: + free_page((unsigned long)opts.opt); +out: + AuTraceErr(err); + err = cvt_err(err); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *aufs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name __maybe_unused, + void *raw_data) +{ + struct dentry *root; + + /* all timestamps always follow the ones on the branch */ + /* mnt->mnt_flags |= MNT_NOATIME | MNT_NODIRATIME; */ + root = mount_nodev(fs_type, flags, raw_data, aufs_fill_super); + if (IS_ERR(root)) + goto out; + + au_sbilist_add(root->d_sb); + +out: + return root; +} + +static void aufs_kill_sb(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + sbinfo = au_sbi(sb); + if (sbinfo) { + au_sbilist_del(sb); + aufs_write_lock(sb->s_root); + au_fhsm_fin(sb); + if (sbinfo->si_wbr_create_ops->fin) + sbinfo->si_wbr_create_ops->fin(sb); + if (au_opt_test(sbinfo->si_mntflags, UDBA_HNOTIFY)) { + au_opt_set_udba(sbinfo->si_mntflags, UDBA_NONE); + au_remount_refresh(sb, /*do_idop*/0); + } + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_put(sb, /*verbose*/1); + au_xino_clr(sb); + au_dr_opt_flush(sb); + sbinfo->si_sb = NULL; + aufs_write_unlock(sb->s_root); + au_nwt_flush(&sbinfo->si_nowait); + } + kill_anon_super(sb); +} + +struct file_system_type aufs_fs_type = { + .name = AUFS_FSTYPE, + /* a race between rename and others */ + .fs_flags = FS_RENAME_DOES_D_MOVE, + .mount = aufs_mount, + .kill_sb = aufs_kill_sb, + /* no need to __module_get() and module_put(). */ + .owner = THIS_MODULE, +}; diff --git a/fs/aufs/super.h b/fs/aufs/super.h new file mode 100644 index 0000000000000..74571acf3eb9e --- /dev/null +++ b/fs/aufs/super.h @@ -0,0 +1,589 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * super_block operations + */ + +#ifndef __AUFS_SUPER_H__ +#define __AUFS_SUPER_H__ + +#ifdef __KERNEL__ + +#include +#include +#include "hbl.h" +#include "lcnt.h" +#include "rwsem.h" +#include "wkq.h" + +/* policies to select one among multiple writable branches */ +struct au_wbr_copyup_operations { + int (*copyup)(struct dentry *dentry); +}; + +#define AuWbr_DIR 1 /* target is a dir */ +#define AuWbr_PARENT (1 << 1) /* always require a parent */ + +#define au_ftest_wbr(flags, name) ((flags) & AuWbr_##name) +#define au_fset_wbr(flags, name) { (flags) |= AuWbr_##name; } +#define au_fclr_wbr(flags, name) { (flags) &= ~AuWbr_##name; } + +struct au_wbr_create_operations { + int (*create)(struct dentry *dentry, unsigned int flags); + int (*init)(struct super_block *sb); + int (*fin)(struct super_block *sb); +}; + +struct au_wbr_mfs { + struct mutex mfs_lock; /* protect this structure */ + unsigned long mfs_jiffy; + unsigned long mfs_expire; + aufs_bindex_t mfs_bindex; + + unsigned long long mfsrr_bytes; + unsigned long long mfsrr_watermark; +}; + +#define AuPlink_NHASH 100 +static inline int au_plink_hash(ino_t ino) +{ + return ino % AuPlink_NHASH; +} + +/* File-based Hierarchical Storage Management */ +struct au_fhsm { +#ifdef CONFIG_AUFS_FHSM + /* allow only one process who can receive the notification */ + spinlock_t fhsm_spin; + pid_t fhsm_pid; + wait_queue_head_t fhsm_wqh; + atomic_t fhsm_readable; + + /* these are protected by si_rwsem */ + unsigned long fhsm_expire; + aufs_bindex_t fhsm_bottom; +#endif +}; + +struct au_branch; +struct au_sbinfo { + /* nowait tasks in the system-wide workqueue */ + struct au_nowait_tasks si_nowait; + + /* + * tried sb->s_umount, but failed due to the dependency between i_mutex. + * rwsem for au_sbinfo is necessary. + */ + struct au_rwsem si_rwsem; + + /* + * dirty approach to protect sb->sb_inodes and ->s_files (gone) from + * remount. + */ + au_lcnt_t si_ninodes, si_nfiles; + + /* branch management */ + unsigned int si_generation; + + /* see AuSi_ flags */ + unsigned char au_si_status; + + aufs_bindex_t si_bbot; + + /* dirty trick to keep br_id plus */ + unsigned int si_last_br_id : + sizeof(aufs_bindex_t) * BITS_PER_BYTE - 1; + struct au_branch **si_branch; + + /* policy to select a writable branch */ + unsigned char si_wbr_copyup; + unsigned char si_wbr_create; + struct au_wbr_copyup_operations *si_wbr_copyup_ops; + struct au_wbr_create_operations *si_wbr_create_ops; + + /* round robin */ + atomic_t si_wbr_rr_next; + + /* most free space */ + struct au_wbr_mfs si_wbr_mfs; + + /* File-based Hierarchical Storage Management */ + struct au_fhsm si_fhsm; + + /* mount flags */ + /* include/asm-ia64/siginfo.h defines a macro named si_flags */ + unsigned int si_mntflags; + + /* external inode number (bitmap and translation table) */ + vfs_readf_t si_xread; + vfs_writef_t si_xwrite; + loff_t si_ximaxent; /* max entries in a xino */ + + struct file *si_xib; + struct mutex si_xib_mtx; /* protect xib members */ + unsigned long *si_xib_buf; + unsigned long si_xib_last_pindex; + int si_xib_next_bit; + + unsigned long si_xino_jiffy; + unsigned long si_xino_expire; + /* reserved for future use */ + /* unsigned long long si_xib_limit; */ /* Max xib file size */ + +#ifdef CONFIG_AUFS_EXPORT + /* i_generation */ + /* todo: make xigen file an array to support many inode numbers */ + struct file *si_xigen; + atomic_t si_xigen_next; +#endif + + /* dirty trick to support atomic_open */ + struct hlist_bl_head si_aopen; + + /* vdir parameters */ + unsigned long si_rdcache; /* max cache time in jiffies */ + unsigned int si_rdblk; /* deblk size */ + unsigned int si_rdhash; /* hash size */ + + /* + * If the number of whiteouts are larger than si_dirwh, leave all of + * them after au_whtmp_ren to reduce the cost of rmdir(2). + * future fsck.aufs or kernel thread will remove them later. + * Otherwise, remove all whiteouts and the dir in rmdir(2). + */ + unsigned int si_dirwh; + + /* pseudo_link list */ + struct hlist_bl_head si_plink[AuPlink_NHASH]; + wait_queue_head_t si_plink_wq; + spinlock_t si_plink_maint_lock; + pid_t si_plink_maint_pid; + + /* file list */ + struct hlist_bl_head si_files; + + /* with/without getattr, brother of sb->s_d_op */ + struct inode_operations *si_iop_array; + + /* + * sysfs and lifetime management. + * this is not a small structure and it may be a waste of memory in case + * of sysfs is disabled, particularly when many aufs-es are mounted. + * but using sysfs is majority. + */ + struct kobject si_kobj; +#ifdef CONFIG_DEBUG_FS + struct dentry *si_dbgaufs; + struct dentry *si_dbgaufs_plink; + struct dentry *si_dbgaufs_xib; +#ifdef CONFIG_AUFS_EXPORT + struct dentry *si_dbgaufs_xigen; +#endif +#endif + +#ifdef CONFIG_AUFS_SBILIST + struct hlist_bl_node si_list; +#endif + + /* dirty, necessary for unmounting, sysfs and sysrq */ + struct super_block *si_sb; +}; + +/* sbinfo status flags */ +/* + * set true when refresh_dirs() failed at remount time. + * then try refreshing dirs at access time again. + * if it is false, refreshing dirs at access time is unnecessary + */ +#define AuSi_FAILED_REFRESH_DIR 1 +#define AuSi_FHSM (1 << 1) /* fhsm is active now */ +#define AuSi_NO_DREVAL (1 << 2) /* disable all d_revalidate */ + +#ifndef CONFIG_AUFS_FHSM +#undef AuSi_FHSM +#define AuSi_FHSM 0 +#endif + +static inline unsigned char au_do_ftest_si(struct au_sbinfo *sbi, + unsigned int flag) +{ + AuRwMustAnyLock(&sbi->si_rwsem); + return sbi->au_si_status & flag; +} +#define au_ftest_si(sbinfo, name) au_do_ftest_si(sbinfo, AuSi_##name) +#define au_fset_si(sbinfo, name) do { \ + AuRwMustWriteLock(&(sbinfo)->si_rwsem); \ + (sbinfo)->au_si_status |= AuSi_##name; \ +} while (0) +#define au_fclr_si(sbinfo, name) do { \ + AuRwMustWriteLock(&(sbinfo)->si_rwsem); \ + (sbinfo)->au_si_status &= ~AuSi_##name; \ +} while (0) + +/* ---------------------------------------------------------------------- */ + +/* policy to select one among writable branches */ +#define AuWbrCopyup(sbinfo, ...) \ + ((sbinfo)->si_wbr_copyup_ops->copyup(__VA_ARGS__)) +#define AuWbrCreate(sbinfo, ...) \ + ((sbinfo)->si_wbr_create_ops->create(__VA_ARGS__)) + +/* flags for si_read_lock()/aufs_read_lock()/di_read_lock() */ +#define AuLock_DW 1 /* write-lock dentry */ +#define AuLock_IR (1 << 1) /* read-lock inode */ +#define AuLock_IW (1 << 2) /* write-lock inode */ +#define AuLock_FLUSH (1 << 3) /* wait for 'nowait' tasks */ +#define AuLock_DIRS (1 << 4) /* target is a pair of dirs */ + /* except RENAME_EXCHANGE */ +#define AuLock_NOPLM (1 << 5) /* return err in plm mode */ +#define AuLock_NOPLMW (1 << 6) /* wait for plm mode ends */ +#define AuLock_GEN (1 << 7) /* test digen/iigen */ +#define au_ftest_lock(flags, name) ((flags) & AuLock_##name) +#define au_fset_lock(flags, name) \ + do { (flags) |= AuLock_##name; } while (0) +#define au_fclr_lock(flags, name) \ + do { (flags) &= ~AuLock_##name; } while (0) + +/* ---------------------------------------------------------------------- */ + +/* super.c */ +extern struct file_system_type aufs_fs_type; +struct inode *au_iget_locked(struct super_block *sb, ino_t ino); +typedef unsigned long long (*au_arraycb_t)(struct super_block *sb, void *array, + unsigned long long max, void *arg); +void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb, + struct super_block *sb, void *arg); +struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max); +void au_iarray_free(struct inode **a, unsigned long long max); + +/* sbinfo.c */ +void au_si_free(struct kobject *kobj); +int au_si_alloc(struct super_block *sb); +int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink); + +unsigned int au_sigen_inc(struct super_block *sb); +aufs_bindex_t au_new_br_id(struct super_block *sb); + +int si_read_lock(struct super_block *sb, int flags); +int si_write_lock(struct super_block *sb, int flags); +int aufs_read_lock(struct dentry *dentry, int flags); +void aufs_read_unlock(struct dentry *dentry, int flags); +void aufs_write_lock(struct dentry *dentry); +void aufs_write_unlock(struct dentry *dentry); +int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags); +void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2); + +/* wbr_policy.c */ +extern struct au_wbr_copyup_operations au_wbr_copyup_ops[]; +extern struct au_wbr_create_operations au_wbr_create_ops[]; +int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst); +int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex); +int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop); + +/* mvdown.c */ +int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *arg); + +#ifdef CONFIG_AUFS_FHSM +/* fhsm.c */ + +static inline pid_t au_fhsm_pid(struct au_fhsm *fhsm) +{ + pid_t pid; + + spin_lock(&fhsm->fhsm_spin); + pid = fhsm->fhsm_pid; + spin_unlock(&fhsm->fhsm_spin); + + return pid; +} + +void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force); +void au_fhsm_wrote_all(struct super_block *sb, int force); +int au_fhsm_fd(struct super_block *sb, int oflags); +int au_fhsm_br_alloc(struct au_branch *br); +void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex); +void au_fhsm_fin(struct super_block *sb); +void au_fhsm_init(struct au_sbinfo *sbinfo); +void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec); +void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo); +#else +AuStubVoid(au_fhsm_wrote, struct super_block *sb, aufs_bindex_t bindex, + int force) +AuStubVoid(au_fhsm_wrote_all, struct super_block *sb, int force) +AuStub(int, au_fhsm_fd, return -EOPNOTSUPP, struct super_block *sb, int oflags) +AuStub(pid_t, au_fhsm_pid, return 0, struct au_fhsm *fhsm) +AuStubInt0(au_fhsm_br_alloc, struct au_branch *br) +AuStubVoid(au_fhsm_set_bottom, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(au_fhsm_fin, struct super_block *sb) +AuStubVoid(au_fhsm_init, struct au_sbinfo *sbinfo) +AuStubVoid(au_fhsm_set, struct au_sbinfo *sbinfo, unsigned int sec) +AuStubVoid(au_fhsm_show, struct seq_file *seq, struct au_sbinfo *sbinfo) +#endif + +/* ---------------------------------------------------------------------- */ + +static inline struct au_sbinfo *au_sbi(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_EXPORT +int au_test_nfsd(void); +void au_export_init(struct super_block *sb); +void au_xigen_inc(struct inode *inode); +int au_xigen_new(struct inode *inode); +int au_xigen_set(struct super_block *sb, struct path *path); +void au_xigen_clr(struct super_block *sb); + +static inline int au_busy_or_stale(void) +{ + if (!au_test_nfsd()) + return -EBUSY; + return -ESTALE; +} +#else +AuStubInt0(au_test_nfsd, void) +AuStubVoid(au_export_init, struct super_block *sb) +AuStubVoid(au_xigen_inc, struct inode *inode) +AuStubInt0(au_xigen_new, struct inode *inode) +AuStubInt0(au_xigen_set, struct super_block *sb, struct path *path) +AuStubVoid(au_xigen_clr, struct super_block *sb) +AuStub(int, au_busy_or_stale, return -EBUSY, void) +#endif /* CONFIG_AUFS_EXPORT */ + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_SBILIST +/* module.c */ +extern struct hlist_bl_head au_sbilist; + +static inline void au_sbilist_init(void) +{ + INIT_HLIST_BL_HEAD(&au_sbilist); +} + +static inline void au_sbilist_add(struct super_block *sb) +{ + au_hbl_add(&au_sbi(sb)->si_list, &au_sbilist); +} + +static inline void au_sbilist_del(struct super_block *sb) +{ + au_hbl_del(&au_sbi(sb)->si_list, &au_sbilist); +} + +#ifdef CONFIG_AUFS_MAGIC_SYSRQ +static inline void au_sbilist_lock(void) +{ + hlist_bl_lock(&au_sbilist); +} + +static inline void au_sbilist_unlock(void) +{ + hlist_bl_unlock(&au_sbilist); +} +#define AuGFP_SBILIST GFP_ATOMIC +#else +AuStubVoid(au_sbilist_lock, void) +AuStubVoid(au_sbilist_unlock, void) +#define AuGFP_SBILIST GFP_NOFS +#endif /* CONFIG_AUFS_MAGIC_SYSRQ */ +#else +AuStubVoid(au_sbilist_init, void) +AuStubVoid(au_sbilist_add, struct super_block *sb) +AuStubVoid(au_sbilist_del, struct super_block *sb) +AuStubVoid(au_sbilist_lock, void) +AuStubVoid(au_sbilist_unlock, void) +#define AuGFP_SBILIST GFP_NOFS +#endif + +/* ---------------------------------------------------------------------- */ + +static inline void dbgaufs_si_null(struct au_sbinfo *sbinfo) +{ + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ +#ifdef CONFIG_DEBUG_FS + sbinfo->si_dbgaufs = NULL; + sbinfo->si_dbgaufs_plink = NULL; + sbinfo->si_dbgaufs_xib = NULL; +#ifdef CONFIG_AUFS_EXPORT + sbinfo->si_dbgaufs_xigen = NULL; +#endif +#endif +} + +/* ---------------------------------------------------------------------- */ + +/* current->atomic_flags */ +/* this value should never corrupt the ones defined in linux/sched.h */ +#define PFA_AUFS 7 + +TASK_PFA_TEST(AUFS, test_aufs) /* task_test_aufs */ +TASK_PFA_SET(AUFS, aufs) /* task_set_aufs */ +TASK_PFA_CLEAR(AUFS, aufs) /* task_clear_aufs */ + +static inline int si_pid_test(struct super_block *sb) +{ + return !!task_test_aufs(current); +} + +static inline void si_pid_clr(struct super_block *sb) +{ + AuDebugOn(!task_test_aufs(current)); + task_clear_aufs(current); +} + +static inline void si_pid_set(struct super_block *sb) +{ + AuDebugOn(task_test_aufs(current)); + task_set_aufs(current); +} + +/* ---------------------------------------------------------------------- */ + +/* lock superblock. mainly for entry point functions */ +#define __si_read_lock(sb) au_rw_read_lock(&au_sbi(sb)->si_rwsem) +#define __si_write_lock(sb) au_rw_write_lock(&au_sbi(sb)->si_rwsem) +#define __si_read_trylock(sb) au_rw_read_trylock(&au_sbi(sb)->si_rwsem) +#define __si_write_trylock(sb) au_rw_write_trylock(&au_sbi(sb)->si_rwsem) +/* +#define __si_read_trylock_nested(sb) \ + au_rw_read_trylock_nested(&au_sbi(sb)->si_rwsem) +#define __si_write_trylock_nested(sb) \ + au_rw_write_trylock_nested(&au_sbi(sb)->si_rwsem) +*/ + +#define __si_read_unlock(sb) au_rw_read_unlock(&au_sbi(sb)->si_rwsem) +#define __si_write_unlock(sb) au_rw_write_unlock(&au_sbi(sb)->si_rwsem) +#define __si_downgrade_lock(sb) au_rw_dgrade_lock(&au_sbi(sb)->si_rwsem) + +#define SiMustNoWaiters(sb) AuRwMustNoWaiters(&au_sbi(sb)->si_rwsem) +#define SiMustAnyLock(sb) AuRwMustAnyLock(&au_sbi(sb)->si_rwsem) +#define SiMustWriteLock(sb) AuRwMustWriteLock(&au_sbi(sb)->si_rwsem) + +static inline void si_noflush_read_lock(struct super_block *sb) +{ + __si_read_lock(sb); + si_pid_set(sb); +} + +static inline int si_noflush_read_trylock(struct super_block *sb) +{ + int locked; + + locked = __si_read_trylock(sb); + if (locked) + si_pid_set(sb); + return locked; +} + +static inline void si_noflush_write_lock(struct super_block *sb) +{ + __si_write_lock(sb); + si_pid_set(sb); +} + +static inline int si_noflush_write_trylock(struct super_block *sb) +{ + int locked; + + locked = __si_write_trylock(sb); + if (locked) + si_pid_set(sb); + return locked; +} + +#if 0 /* reserved */ +static inline int si_read_trylock(struct super_block *sb, int flags) +{ + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + return si_noflush_read_trylock(sb); +} +#endif + +static inline void si_read_unlock(struct super_block *sb) +{ + si_pid_clr(sb); + __si_read_unlock(sb); +} + +#if 0 /* reserved */ +static inline int si_write_trylock(struct super_block *sb, int flags) +{ + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + return si_noflush_write_trylock(sb); +} +#endif + +static inline void si_write_unlock(struct super_block *sb) +{ + si_pid_clr(sb); + __si_write_unlock(sb); +} + +#if 0 /* reserved */ +static inline void si_downgrade_lock(struct super_block *sb) +{ + __si_downgrade_lock(sb); +} +#endif + +/* ---------------------------------------------------------------------- */ + +static inline aufs_bindex_t au_sbbot(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_bbot; +} + +static inline unsigned int au_mntflags(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_mntflags; +} + +static inline unsigned int au_sigen(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_generation; +} + +static inline struct au_branch *au_sbr(struct super_block *sb, + aufs_bindex_t bindex) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_branch[0 + bindex]; +} + +static inline loff_t au_xi_maxent(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_ximaxent; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_SUPER_H__ */ diff --git a/fs/aufs/sysaufs.c b/fs/aufs/sysaufs.c new file mode 100644 index 0000000000000..32a0811c54de0 --- /dev/null +++ b/fs/aufs/sysaufs.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface and lifetime management + * they are necessary regardless sysfs is disabled. + */ + +#include +#include "aufs.h" + +unsigned long sysaufs_si_mask; +struct kset *sysaufs_kset; + +#define AuSiAttr(_name) { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = sysaufs_si_##_name, \ +} + +static struct sysaufs_si_attr sysaufs_si_attr_xi_path = AuSiAttr(xi_path); +struct attribute *sysaufs_si_attrs[] = { + &sysaufs_si_attr_xi_path.attr, + NULL, +}; + +static const struct sysfs_ops au_sbi_ops = { + .show = sysaufs_si_show +}; + +static struct kobj_type au_sbi_ktype = { + .release = au_si_free, + .sysfs_ops = &au_sbi_ops, + .default_attrs = sysaufs_si_attrs +}; + +/* ---------------------------------------------------------------------- */ + +int sysaufs_si_init(struct au_sbinfo *sbinfo) +{ + int err; + + sbinfo->si_kobj.kset = sysaufs_kset; + /* cf. sysaufs_name() */ + err = kobject_init_and_add + (&sbinfo->si_kobj, &au_sbi_ktype, /*&sysaufs_kset->kobj*/NULL, + SysaufsSiNamePrefix "%lx", sysaufs_si_id(sbinfo)); + + return err; +} + +void sysaufs_fin(void) +{ + sysfs_remove_group(&sysaufs_kset->kobj, sysaufs_attr_group); + kset_unregister(sysaufs_kset); +} + +int __init sysaufs_init(void) +{ + int err; + + do { + get_random_bytes(&sysaufs_si_mask, sizeof(sysaufs_si_mask)); + } while (!sysaufs_si_mask); + + err = -EINVAL; + sysaufs_kset = kset_create_and_add(AUFS_NAME, NULL, fs_kobj); + if (unlikely(!sysaufs_kset)) + goto out; + err = PTR_ERR(sysaufs_kset); + if (IS_ERR(sysaufs_kset)) + goto out; + err = sysfs_create_group(&sysaufs_kset->kobj, sysaufs_attr_group); + if (unlikely(err)) + kset_unregister(sysaufs_kset); + +out: + return err; +} diff --git a/fs/aufs/sysaufs.h b/fs/aufs/sysaufs.h new file mode 100644 index 0000000000000..195fe279b820d --- /dev/null +++ b/fs/aufs/sysaufs.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface and mount lifetime management + */ + +#ifndef __SYSAUFS_H__ +#define __SYSAUFS_H__ + +#ifdef __KERNEL__ + +#include +#include "module.h" + +struct super_block; +struct au_sbinfo; + +struct sysaufs_si_attr { + struct attribute attr; + int (*show)(struct seq_file *seq, struct super_block *sb); +}; + +/* ---------------------------------------------------------------------- */ + +/* sysaufs.c */ +extern unsigned long sysaufs_si_mask; +extern struct kset *sysaufs_kset; +extern struct attribute *sysaufs_si_attrs[]; +int sysaufs_si_init(struct au_sbinfo *sbinfo); +int __init sysaufs_init(void); +void sysaufs_fin(void); + +/* ---------------------------------------------------------------------- */ + +/* some people doesn't like to show a pointer in kernel */ +static inline unsigned long sysaufs_si_id(struct au_sbinfo *sbinfo) +{ + return sysaufs_si_mask ^ (unsigned long)sbinfo; +} + +#define SysaufsSiNamePrefix "si_" +#define SysaufsSiNameLen (sizeof(SysaufsSiNamePrefix) + 16) +static inline void sysaufs_name(struct au_sbinfo *sbinfo, char *name) +{ + snprintf(name, SysaufsSiNameLen, SysaufsSiNamePrefix "%lx", + sysaufs_si_id(sbinfo)); +} + +struct au_branch; +#ifdef CONFIG_SYSFS +/* sysfs.c */ +extern struct attribute_group *sysaufs_attr_group; + +int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb); +ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr, + char *buf); +long au_brinfo_ioctl(struct file *file, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_brinfo_compat_ioctl(struct file *file, unsigned long arg); +#endif + +void sysaufs_br_init(struct au_branch *br); +void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex); +void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex); + +#define sysaufs_brs_init() do {} while (0) + +#else +#define sysaufs_attr_group NULL + +AuStubInt0(sysaufs_si_xi_path, struct seq_file *seq, struct super_block *sb) +AuStub(ssize_t, sysaufs_si_show, return 0, struct kobject *kobj, + struct attribute *attr, char *buf) +AuStubVoid(sysaufs_br_init, struct au_branch *br) +AuStubVoid(sysaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(sysaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex) + +static inline void sysaufs_brs_init(void) +{ + sysaufs_brs = 0; +} + +#endif /* CONFIG_SYSFS */ + +#endif /* __KERNEL__ */ +#endif /* __SYSAUFS_H__ */ diff --git a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c new file mode 100644 index 0000000000000..927f6b252c752 --- /dev/null +++ b/fs/aufs/sysfs.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface + */ + +#include +#include +#include "aufs.h" + +#ifdef CONFIG_AUFS_FS_MODULE +/* this entry violates the "one line per file" policy of sysfs */ +static ssize_t config_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + ssize_t err; + static char *conf = +/* this file is generated at compiling */ +#include "conf.str" + ; + + err = snprintf(buf, PAGE_SIZE, conf); + if (unlikely(err >= PAGE_SIZE)) + err = -EFBIG; + return err; +} + +static struct kobj_attribute au_config_attr = __ATTR_RO(config); +#endif + +static struct attribute *au_attr[] = { +#ifdef CONFIG_AUFS_FS_MODULE + &au_config_attr.attr, +#endif + NULL, /* need to NULL terminate the list of attributes */ +}; + +static struct attribute_group sysaufs_attr_group_body = { + .attrs = au_attr +}; + +struct attribute_group *sysaufs_attr_group = &sysaufs_attr_group_body; + +/* ---------------------------------------------------------------------- */ + +int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb) +{ + int err; + + SiMustAnyLock(sb); + + err = 0; + if (au_opt_test(au_mntflags(sb), XINO)) { + err = au_xino_path(seq, au_sbi(sb)->si_xib); + seq_putc(seq, '\n'); + } + return err; +} + +/* + * the lifetime of branch is independent from the entry under sysfs. + * sysfs handles the lifetime of the entry, and never call ->show() after it is + * unlinked. + */ +static int sysaufs_si_br(struct seq_file *seq, struct super_block *sb, + aufs_bindex_t bindex, int idx) +{ + int err; + struct path path; + struct dentry *root; + struct au_branch *br; + au_br_perm_str_t perm; + + AuDbg("b%d\n", bindex); + + err = 0; + root = sb->s_root; + di_read_lock_parent(root, !AuLock_IR); + br = au_sbr(sb, bindex); + + switch (idx) { + case AuBrSysfs_BR: + path.mnt = au_br_mnt(br); + path.dentry = au_h_dptr(root, bindex); + err = au_seq_path(seq, &path); + if (!err) { + au_optstr_br_perm(&perm, br->br_perm); + seq_printf(seq, "=%s\n", perm.a); + } + break; + case AuBrSysfs_BRID: + seq_printf(seq, "%d\n", br->br_id); + break; + } + di_read_unlock(root, !AuLock_IR); + if (unlikely(err || seq_has_overflowed(seq))) + err = -E2BIG; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct seq_file *au_seq(char *p, ssize_t len) +{ + struct seq_file *seq; + + seq = kzalloc(sizeof(*seq), GFP_NOFS); + if (seq) { + /* mutex_init(&seq.lock); */ + seq->buf = p; + seq->size = len; + return seq; /* success */ + } + + seq = ERR_PTR(-ENOMEM); + return seq; +} + +#define SysaufsBr_PREFIX "br" +#define SysaufsBrid_PREFIX "brid" + +/* todo: file size may exceed PAGE_SIZE */ +ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + ssize_t err; + int idx; + long l; + aufs_bindex_t bbot; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct seq_file *seq; + char *name; + struct attribute **cattr; + + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + sb = sbinfo->si_sb; + + /* + * prevent a race condition between sysfs and aufs. + * for instance, sysfs_file_read() calls sysfs_get_active_two() which + * prohibits maintaining the sysfs entries. + * hew we acquire read lock after sysfs_get_active_two(). + * on the other hand, the remount process may maintain the sysfs/aufs + * entries after acquiring write lock. + * it can cause a deadlock. + * simply we gave up processing read here. + */ + err = -EBUSY; + if (unlikely(!si_noflush_read_trylock(sb))) + goto out; + + seq = au_seq(buf, PAGE_SIZE); + err = PTR_ERR(seq); + if (IS_ERR(seq)) + goto out_unlock; + + name = (void *)attr->name; + cattr = sysaufs_si_attrs; + while (*cattr) { + if (!strcmp(name, (*cattr)->name)) { + err = container_of(*cattr, struct sysaufs_si_attr, attr) + ->show(seq, sb); + goto out_seq; + } + cattr++; + } + + if (!strncmp(name, SysaufsBrid_PREFIX, + sizeof(SysaufsBrid_PREFIX) - 1)) { + idx = AuBrSysfs_BRID; + name += sizeof(SysaufsBrid_PREFIX) - 1; + } else if (!strncmp(name, SysaufsBr_PREFIX, + sizeof(SysaufsBr_PREFIX) - 1)) { + idx = AuBrSysfs_BR; + name += sizeof(SysaufsBr_PREFIX) - 1; + } else + BUG(); + + err = kstrtol(name, 10, &l); + if (!err) { + bbot = au_sbbot(sb); + if (l <= bbot) + err = sysaufs_si_br(seq, sb, (aufs_bindex_t)l, idx); + else + err = -ENOENT; + } + +out_seq: + if (!err) { + err = seq->count; + /* sysfs limit */ + if (unlikely(err == PAGE_SIZE)) + err = -EFBIG; + } + au_kfree_rcu(seq); +out_unlock: + si_read_unlock(sb); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_brinfo(struct super_block *sb, union aufs_brinfo __user *arg) +{ + int err; + int16_t brid; + aufs_bindex_t bindex, bbot; + size_t sz; + char *buf; + struct seq_file *seq; + struct au_branch *br; + + si_read_lock(sb, AuLock_FLUSH); + bbot = au_sbbot(sb); + err = bbot + 1; + if (!arg) + goto out; + + err = -ENOMEM; + buf = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!buf)) + goto out; + + seq = au_seq(buf, PAGE_SIZE); + err = PTR_ERR(seq); + if (IS_ERR(seq)) + goto out_buf; + + sz = sizeof(*arg) - offsetof(union aufs_brinfo, path); + for (bindex = 0; bindex <= bbot; bindex++, arg++) { + err = !access_ok(VERIFY_WRITE, arg, sizeof(*arg)); + if (unlikely(err)) + break; + + br = au_sbr(sb, bindex); + brid = br->br_id; + BUILD_BUG_ON(sizeof(brid) != sizeof(arg->id)); + err = __put_user(brid, &arg->id); + if (unlikely(err)) + break; + + BUILD_BUG_ON(sizeof(br->br_perm) != sizeof(arg->perm)); + err = __put_user(br->br_perm, &arg->perm); + if (unlikely(err)) + break; + + err = au_seq_path(seq, &br->br_path); + if (unlikely(err)) + break; + seq_putc(seq, '\0'); + if (!seq_has_overflowed(seq)) { + err = copy_to_user(arg->path, seq->buf, seq->count); + seq->count = 0; + if (unlikely(err)) + break; + } else { + err = -E2BIG; + goto out_seq; + } + } + if (unlikely(err)) + err = -EFAULT; + +out_seq: + au_kfree_rcu(seq); +out_buf: + free_page((unsigned long)buf); +out: + si_read_unlock(sb); + return err; +} + +long au_brinfo_ioctl(struct file *file, unsigned long arg) +{ + return au_brinfo(file->f_path.dentry->d_sb, (void __user *)arg); +} + +#ifdef CONFIG_COMPAT +long au_brinfo_compat_ioctl(struct file *file, unsigned long arg) +{ + return au_brinfo(file->f_path.dentry->d_sb, compat_ptr(arg)); +} +#endif + +/* ---------------------------------------------------------------------- */ + +void sysaufs_br_init(struct au_branch *br) +{ + int i; + struct au_brsysfs *br_sysfs; + struct attribute *attr; + + br_sysfs = br->br_sysfs; + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + attr = &br_sysfs->attr; + sysfs_attr_init(attr); + attr->name = br_sysfs->name; + attr->mode = 0444; + br_sysfs++; + } +} + +void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex) +{ + struct au_branch *br; + struct kobject *kobj; + struct au_brsysfs *br_sysfs; + int i; + aufs_bindex_t bbot; + + if (!sysaufs_brs) + return; + + kobj = &au_sbi(sb)->si_kobj; + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + br_sysfs = br->br_sysfs; + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + sysfs_remove_file(kobj, &br_sysfs->attr); + br_sysfs++; + } + } +} + +void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex) +{ + int err, i; + aufs_bindex_t bbot; + struct kobject *kobj; + struct au_branch *br; + struct au_brsysfs *br_sysfs; + + if (!sysaufs_brs) + return; + + kobj = &au_sbi(sb)->si_kobj; + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + br_sysfs = br->br_sysfs; + snprintf(br_sysfs[AuBrSysfs_BR].name, sizeof(br_sysfs->name), + SysaufsBr_PREFIX "%d", bindex); + snprintf(br_sysfs[AuBrSysfs_BRID].name, sizeof(br_sysfs->name), + SysaufsBrid_PREFIX "%d", bindex); + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + err = sysfs_create_file(kobj, &br_sysfs->attr); + if (unlikely(err)) + pr_warn("failed %s under sysfs(%d)\n", + br_sysfs->name, err); + br_sysfs++; + } + } +} diff --git a/fs/aufs/sysrq.c b/fs/aufs/sysrq.c new file mode 100644 index 0000000000000..db10ec528f5d6 --- /dev/null +++ b/fs/aufs/sysrq.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * magic sysrq handler + */ + +/* #include */ +#include +#include "aufs.h" + +/* ---------------------------------------------------------------------- */ + +static void sysrq_sb(struct super_block *sb) +{ + char *plevel; + struct au_sbinfo *sbinfo; + struct file *file; + struct hlist_bl_head *files; + struct hlist_bl_node *pos; + struct au_finfo *finfo; + + plevel = au_plevel; + au_plevel = KERN_WARNING; + + /* since we define pr_fmt, call printk directly */ +#define pr(str) printk(KERN_WARNING AUFS_NAME ": " str) + + sbinfo = au_sbi(sb); + printk(KERN_WARNING "si=%lx\n", sysaufs_si_id(sbinfo)); + pr("superblock\n"); + au_dpri_sb(sb); + +#if 0 + pr("root dentry\n"); + au_dpri_dentry(sb->s_root); + pr("root inode\n"); + au_dpri_inode(d_inode(sb->s_root)); +#endif + +#if 0 + do { + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + + err = au_dpages_init(&dpages, GFP_ATOMIC); + if (unlikely(err)) + break; + err = au_dcsub_pages(&dpages, sb->s_root, NULL, NULL); + if (!err) + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) + au_dpri_dentry(dpage->dentries[j]); + } + au_dpages_free(&dpages); + } while (0); +#endif + +#if 1 + { + struct inode *i; + + pr("isolated inode\n"); + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(i, &sb->s_inodes, i_sb_list) { + spin_lock(&i->i_lock); + if (1 || hlist_empty(&i->i_dentry)) + au_dpri_inode(i); + spin_unlock(&i->i_lock); + } + spin_unlock(&sb->s_inode_list_lock); + } +#endif + pr("files\n"); + files = &au_sbi(sb)->si_files; + hlist_bl_lock(files); + hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) { + umode_t mode; + + file = finfo->fi_file; + mode = file_inode(file)->i_mode; + if (!special_file(mode)) + au_dpri_file(file); + } + hlist_bl_unlock(files); + pr("done\n"); + +#undef pr + au_plevel = plevel; +} + +/* ---------------------------------------------------------------------- */ + +/* module parameter */ +static char *aufs_sysrq_key = "a"; +module_param_named(sysrq, aufs_sysrq_key, charp, 0444); +MODULE_PARM_DESC(sysrq, "MagicSysRq key for " AUFS_NAME); + +static void au_sysrq(int key __maybe_unused) +{ + struct au_sbinfo *sbinfo; + struct hlist_bl_node *pos; + + lockdep_off(); + au_sbilist_lock(); + hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list) + sysrq_sb(sbinfo->si_sb); + au_sbilist_unlock(); + lockdep_on(); +} + +static struct sysrq_key_op au_sysrq_op = { + .handler = au_sysrq, + .help_msg = "Aufs", + .action_msg = "Aufs", + .enable_mask = SYSRQ_ENABLE_DUMP +}; + +/* ---------------------------------------------------------------------- */ + +int __init au_sysrq_init(void) +{ + int err; + char key; + + err = -1; + key = *aufs_sysrq_key; + if ('a' <= key && key <= 'z') + err = register_sysrq_key(key, &au_sysrq_op); + if (unlikely(err)) + pr_err("err %d, sysrq=%c\n", err, key); + return err; +} + +void au_sysrq_fin(void) +{ + int err; + + err = unregister_sysrq_key(*aufs_sysrq_key, &au_sysrq_op); + if (unlikely(err)) + pr_err("err %d (ignored)\n", err); +} diff --git a/fs/aufs/vdir.c b/fs/aufs/vdir.c new file mode 100644 index 0000000000000..13ec60bb4cccb --- /dev/null +++ b/fs/aufs/vdir.c @@ -0,0 +1,895 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * virtual or vertical directory + */ + +#include "aufs.h" + +static unsigned int calc_size(int nlen) +{ + return ALIGN(sizeof(struct au_vdir_de) + nlen, sizeof(ino_t)); +} + +static int set_deblk_end(union au_vdir_deblk_p *p, + union au_vdir_deblk_p *deblk_end) +{ + if (calc_size(0) <= deblk_end->deblk - p->deblk) { + p->de->de_str.len = 0; + /* smp_mb(); */ + return 0; + } + return -1; /* error */ +} + +/* returns true or false */ +static int is_deblk_end(union au_vdir_deblk_p *p, + union au_vdir_deblk_p *deblk_end) +{ + if (calc_size(0) <= deblk_end->deblk - p->deblk) + return !p->de->de_str.len; + return 1; +} + +static unsigned char *last_deblk(struct au_vdir *vdir) +{ + return vdir->vd_deblk[vdir->vd_nblk - 1]; +} + +/* ---------------------------------------------------------------------- */ + +/* estimate the appropriate size for name hash table */ +unsigned int au_rdhash_est(loff_t sz) +{ + unsigned int n; + + n = UINT_MAX; + sz >>= 10; + if (sz < n) + n = sz; + if (sz < AUFS_RDHASH_DEF) + n = AUFS_RDHASH_DEF; + /* pr_info("n %u\n", n); */ + return n; +} + +/* + * the allocated memory has to be freed by + * au_nhash_wh_free() or au_nhash_de_free(). + */ +int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp) +{ + struct hlist_head *head; + unsigned int u; + size_t sz; + + sz = sizeof(*nhash->nh_head) * num_hash; + head = kmalloc(sz, gfp); + if (head) { + nhash->nh_num = num_hash; + nhash->nh_head = head; + for (u = 0; u < num_hash; u++) + INIT_HLIST_HEAD(head++); + return 0; /* success */ + } + + return -ENOMEM; +} + +static void nhash_count(struct hlist_head *head) +{ +#if 0 + unsigned long n; + struct hlist_node *pos; + + n = 0; + hlist_for_each(pos, head) + n++; + pr_info("%lu\n", n); +#endif +} + +static void au_nhash_wh_do_free(struct hlist_head *head) +{ + struct au_vdir_wh *pos; + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, wh_hash) + au_kfree_rcu(pos); +} + +static void au_nhash_de_do_free(struct hlist_head *head) +{ + struct au_vdir_dehstr *pos; + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, hash) + au_cache_free_vdir_dehstr(pos); +} + +static void au_nhash_do_free(struct au_nhash *nhash, + void (*free)(struct hlist_head *head)) +{ + unsigned int n; + struct hlist_head *head; + + n = nhash->nh_num; + if (!n) + return; + + head = nhash->nh_head; + while (n-- > 0) { + nhash_count(head); + free(head++); + } + au_kfree_try_rcu(nhash->nh_head); +} + +void au_nhash_wh_free(struct au_nhash *whlist) +{ + au_nhash_do_free(whlist, au_nhash_wh_do_free); +} + +static void au_nhash_de_free(struct au_nhash *delist) +{ + au_nhash_do_free(delist, au_nhash_de_do_free); +} + +/* ---------------------------------------------------------------------- */ + +int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt, + int limit) +{ + int num; + unsigned int u, n; + struct hlist_head *head; + struct au_vdir_wh *pos; + + num = 0; + n = whlist->nh_num; + head = whlist->nh_head; + for (u = 0; u < n; u++, head++) + hlist_for_each_entry(pos, head, wh_hash) + if (pos->wh_bindex == btgt && ++num > limit) + return 1; + return 0; +} + +static struct hlist_head *au_name_hash(struct au_nhash *nhash, + unsigned char *name, + unsigned int len) +{ + unsigned int v; + /* const unsigned int magic_bit = 12; */ + + AuDebugOn(!nhash->nh_num || !nhash->nh_head); + + v = 0; + if (len > 8) + len = 8; + while (len--) + v += *name++; + /* v = hash_long(v, magic_bit); */ + v %= nhash->nh_num; + return nhash->nh_head + v; +} + +static int au_nhash_test_name(struct au_vdir_destr *str, const char *name, + int nlen) +{ + return str->len == nlen && !memcmp(str->name, name, nlen); +} + +/* returns found or not */ +int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen) +{ + struct hlist_head *head; + struct au_vdir_wh *pos; + struct au_vdir_destr *str; + + head = au_name_hash(whlist, name, nlen); + hlist_for_each_entry(pos, head, wh_hash) { + str = &pos->wh_str; + AuDbg("%.*s\n", str->len, str->name); + if (au_nhash_test_name(str, name, nlen)) + return 1; + } + return 0; +} + +/* returns found(true) or not */ +static int test_known(struct au_nhash *delist, char *name, int nlen) +{ + struct hlist_head *head; + struct au_vdir_dehstr *pos; + struct au_vdir_destr *str; + + head = au_name_hash(delist, name, nlen); + hlist_for_each_entry(pos, head, hash) { + str = pos->str; + AuDbg("%.*s\n", str->len, str->name); + if (au_nhash_test_name(str, name, nlen)) + return 1; + } + return 0; +} + +static void au_shwh_init_wh(struct au_vdir_wh *wh, ino_t ino, + unsigned char d_type) +{ +#ifdef CONFIG_AUFS_SHWH + wh->wh_ino = ino; + wh->wh_type = d_type; +#endif +} + +/* ---------------------------------------------------------------------- */ + +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh) +{ + int err; + struct au_vdir_destr *str; + struct au_vdir_wh *wh; + + AuDbg("%.*s\n", nlen, name); + AuDebugOn(!whlist->nh_num || !whlist->nh_head); + + err = -ENOMEM; + wh = kmalloc(sizeof(*wh) + nlen, GFP_NOFS); + if (unlikely(!wh)) + goto out; + + err = 0; + wh->wh_bindex = bindex; + if (shwh) + au_shwh_init_wh(wh, ino, d_type); + str = &wh->wh_str; + str->len = nlen; + memcpy(str->name, name, nlen); + hlist_add_head(&wh->wh_hash, au_name_hash(whlist, name, nlen)); + /* smp_mb(); */ + +out: + return err; +} + +static int append_deblk(struct au_vdir *vdir) +{ + int err; + unsigned long ul; + const unsigned int deblk_sz = vdir->vd_deblk_sz; + union au_vdir_deblk_p p, deblk_end; + unsigned char **o; + + err = -ENOMEM; + o = au_krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), + GFP_NOFS, /*may_shrink*/0); + if (unlikely(!o)) + goto out; + + vdir->vd_deblk = o; + p.deblk = kmalloc(deblk_sz, GFP_NOFS); + if (p.deblk) { + ul = vdir->vd_nblk++; + vdir->vd_deblk[ul] = p.deblk; + vdir->vd_last.ul = ul; + vdir->vd_last.p.deblk = p.deblk; + deblk_end.deblk = p.deblk + deblk_sz; + err = set_deblk_end(&p, &deblk_end); + } + +out: + return err; +} + +static int append_de(struct au_vdir *vdir, char *name, int nlen, ino_t ino, + unsigned int d_type, struct au_nhash *delist) +{ + int err; + unsigned int sz; + const unsigned int deblk_sz = vdir->vd_deblk_sz; + union au_vdir_deblk_p p, *room, deblk_end; + struct au_vdir_dehstr *dehstr; + + p.deblk = last_deblk(vdir); + deblk_end.deblk = p.deblk + deblk_sz; + room = &vdir->vd_last.p; + AuDebugOn(room->deblk < p.deblk || deblk_end.deblk <= room->deblk + || !is_deblk_end(room, &deblk_end)); + + sz = calc_size(nlen); + if (unlikely(sz > deblk_end.deblk - room->deblk)) { + err = append_deblk(vdir); + if (unlikely(err)) + goto out; + + p.deblk = last_deblk(vdir); + deblk_end.deblk = p.deblk + deblk_sz; + /* smp_mb(); */ + AuDebugOn(room->deblk != p.deblk); + } + + err = -ENOMEM; + dehstr = au_cache_alloc_vdir_dehstr(); + if (unlikely(!dehstr)) + goto out; + + dehstr->str = &room->de->de_str; + hlist_add_head(&dehstr->hash, au_name_hash(delist, name, nlen)); + room->de->de_ino = ino; + room->de->de_type = d_type; + room->de->de_str.len = nlen; + memcpy(room->de->de_str.name, name, nlen); + + err = 0; + room->deblk += sz; + if (unlikely(set_deblk_end(room, &deblk_end))) + err = append_deblk(vdir); + /* smp_mb(); */ + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_vdir_free(struct au_vdir *vdir) +{ + unsigned char **deblk; + + deblk = vdir->vd_deblk; + while (vdir->vd_nblk--) + au_kfree_try_rcu(*deblk++); + au_kfree_try_rcu(vdir->vd_deblk); + au_cache_free_vdir(vdir); +} + +static struct au_vdir *alloc_vdir(struct file *file) +{ + struct au_vdir *vdir; + struct super_block *sb; + int err; + + sb = file->f_path.dentry->d_sb; + SiMustAnyLock(sb); + + err = -ENOMEM; + vdir = au_cache_alloc_vdir(); + if (unlikely(!vdir)) + goto out; + + vdir->vd_deblk = kzalloc(sizeof(*vdir->vd_deblk), GFP_NOFS); + if (unlikely(!vdir->vd_deblk)) + goto out_free; + + vdir->vd_deblk_sz = au_sbi(sb)->si_rdblk; + if (!vdir->vd_deblk_sz) { + /* estimate the appropriate size for deblk */ + vdir->vd_deblk_sz = au_dir_size(file, /*dentry*/NULL); + /* pr_info("vd_deblk_sz %u\n", vdir->vd_deblk_sz); */ + } + vdir->vd_nblk = 0; + vdir->vd_version = 0; + vdir->vd_jiffy = 0; + err = append_deblk(vdir); + if (!err) + return vdir; /* success */ + + au_kfree_try_rcu(vdir->vd_deblk); + +out_free: + au_cache_free_vdir(vdir); +out: + vdir = ERR_PTR(err); + return vdir; +} + +static int reinit_vdir(struct au_vdir *vdir) +{ + int err; + union au_vdir_deblk_p p, deblk_end; + + while (vdir->vd_nblk > 1) { + au_kfree_try_rcu(vdir->vd_deblk[vdir->vd_nblk - 1]); + /* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */ + vdir->vd_nblk--; + } + p.deblk = vdir->vd_deblk[0]; + deblk_end.deblk = p.deblk + vdir->vd_deblk_sz; + err = set_deblk_end(&p, &deblk_end); + /* keep vd_dblk_sz */ + vdir->vd_last.ul = 0; + vdir->vd_last.p.deblk = vdir->vd_deblk[0]; + vdir->vd_version = 0; + vdir->vd_jiffy = 0; + /* smp_mb(); */ + return err; +} + +/* ---------------------------------------------------------------------- */ + +#define AuFillVdir_CALLED 1 +#define AuFillVdir_WHABLE (1 << 1) +#define AuFillVdir_SHWH (1 << 2) +#define au_ftest_fillvdir(flags, name) ((flags) & AuFillVdir_##name) +#define au_fset_fillvdir(flags, name) \ + do { (flags) |= AuFillVdir_##name; } while (0) +#define au_fclr_fillvdir(flags, name) \ + do { (flags) &= ~AuFillVdir_##name; } while (0) + +#ifndef CONFIG_AUFS_SHWH +#undef AuFillVdir_SHWH +#define AuFillVdir_SHWH 0 +#endif + +struct fillvdir_arg { + struct dir_context ctx; + struct file *file; + struct au_vdir *vdir; + struct au_nhash delist; + struct au_nhash whlist; + aufs_bindex_t bindex; + unsigned int flags; + int err; +}; + +static int fillvdir(struct dir_context *ctx, const char *__name, int nlen, + loff_t offset __maybe_unused, u64 h_ino, + unsigned int d_type) +{ + struct fillvdir_arg *arg = container_of(ctx, struct fillvdir_arg, ctx); + char *name = (void *)__name; + struct super_block *sb; + ino_t ino; + const unsigned char shwh = !!au_ftest_fillvdir(arg->flags, SHWH); + + arg->err = 0; + sb = arg->file->f_path.dentry->d_sb; + au_fset_fillvdir(arg->flags, CALLED); + /* smp_mb(); */ + if (nlen <= AUFS_WH_PFX_LEN + || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + if (test_known(&arg->delist, name, nlen) + || au_nhash_test_known_wh(&arg->whlist, name, nlen)) + goto out; /* already exists or whiteouted */ + + arg->err = au_ino(sb, arg->bindex, h_ino, d_type, &ino); + if (!arg->err) { + if (unlikely(nlen > AUFS_MAX_NAMELEN)) + d_type = DT_UNKNOWN; + arg->err = append_de(arg->vdir, name, nlen, ino, + d_type, &arg->delist); + } + } else if (au_ftest_fillvdir(arg->flags, WHABLE)) { + name += AUFS_WH_PFX_LEN; + nlen -= AUFS_WH_PFX_LEN; + if (au_nhash_test_known_wh(&arg->whlist, name, nlen)) + goto out; /* already whiteouted */ + + ino = 0; /* just to suppress a warning */ + if (shwh) + arg->err = au_wh_ino(sb, arg->bindex, h_ino, d_type, + &ino); + if (!arg->err) { + if (nlen <= AUFS_MAX_NAMELEN + AUFS_WH_PFX_LEN) + d_type = DT_UNKNOWN; + arg->err = au_nhash_append_wh + (&arg->whlist, name, nlen, ino, d_type, + arg->bindex, shwh); + } + } + +out: + if (!arg->err) + arg->vdir->vd_jiffy = jiffies; + /* smp_mb(); */ + AuTraceErr(arg->err); + return arg->err; +} + +static int au_handle_shwh(struct super_block *sb, struct au_vdir *vdir, + struct au_nhash *whlist, struct au_nhash *delist) +{ +#ifdef CONFIG_AUFS_SHWH + int err; + unsigned int nh, u; + struct hlist_head *head; + struct au_vdir_wh *pos; + struct hlist_node *n; + char *p, *o; + struct au_vdir_destr *destr; + + AuDebugOn(!au_opt_test(au_mntflags(sb), SHWH)); + + err = -ENOMEM; + o = p = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = 0; + nh = whlist->nh_num; + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + p += AUFS_WH_PFX_LEN; + for (u = 0; u < nh; u++) { + head = whlist->nh_head + u; + hlist_for_each_entry_safe(pos, n, head, wh_hash) { + destr = &pos->wh_str; + memcpy(p, destr->name, destr->len); + err = append_de(vdir, o, destr->len + AUFS_WH_PFX_LEN, + pos->wh_ino, pos->wh_type, delist); + if (unlikely(err)) + break; + } + } + + free_page((unsigned long)o); + +out: + AuTraceErr(err); + return err; +#else + return 0; +#endif +} + +static int au_do_read_vdir(struct fillvdir_arg *arg) +{ + int err; + unsigned int rdhash; + loff_t offset; + aufs_bindex_t bbot, bindex, btop; + unsigned char shwh; + struct file *hf, *file; + struct super_block *sb; + + file = arg->file; + sb = file->f_path.dentry->d_sb; + SiMustAnyLock(sb); + + rdhash = au_sbi(sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(file, /*dentry*/NULL)); + err = au_nhash_alloc(&arg->delist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_nhash_alloc(&arg->whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out_delist; + + err = 0; + arg->flags = 0; + shwh = 0; + if (au_opt_test(au_mntflags(sb), SHWH)) { + shwh = 1; + au_fset_fillvdir(arg->flags, SHWH); + } + btop = au_fbtop(file); + bbot = au_fbbot_dir(file); + for (bindex = btop; !err && bindex <= bbot; bindex++) { + hf = au_hf_dir(file, bindex); + if (!hf) + continue; + + offset = vfsub_llseek(hf, 0, SEEK_SET); + err = offset; + if (unlikely(offset)) + break; + + arg->bindex = bindex; + au_fclr_fillvdir(arg->flags, WHABLE); + if (shwh + || (bindex != bbot + && au_br_whable(au_sbr_perm(sb, bindex)))) + au_fset_fillvdir(arg->flags, WHABLE); + do { + arg->err = 0; + au_fclr_fillvdir(arg->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(hf, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err && au_ftest_fillvdir(arg->flags, CALLED)); + + /* + * dir_relax() may be good for concurrency, but aufs should not + * use it since it will cause a lockdep problem. + */ + } + + if (!err && shwh) + err = au_handle_shwh(sb, arg->vdir, &arg->whlist, &arg->delist); + + au_nhash_wh_free(&arg->whlist); + +out_delist: + au_nhash_de_free(&arg->delist); +out: + return err; +} + +static int read_vdir(struct file *file, int may_read) +{ + int err; + unsigned long expire; + unsigned char do_read; + struct fillvdir_arg arg = { + .ctx = { + .actor = fillvdir + } + }; + struct inode *inode; + struct au_vdir *vdir, *allocated; + + err = 0; + inode = file_inode(file); + IMustLock(inode); + IiMustWriteLock(inode); + SiMustAnyLock(inode->i_sb); + + allocated = NULL; + do_read = 0; + expire = au_sbi(inode->i_sb)->si_rdcache; + vdir = au_ivdir(inode); + if (!vdir) { + do_read = 1; + vdir = alloc_vdir(file); + err = PTR_ERR(vdir); + if (IS_ERR(vdir)) + goto out; + err = 0; + allocated = vdir; + } else if (may_read + && (inode->i_version != vdir->vd_version + || time_after(jiffies, vdir->vd_jiffy + expire))) { + do_read = 1; + err = reinit_vdir(vdir); + if (unlikely(err)) + goto out; + } + + if (!do_read) + return 0; /* success */ + + arg.file = file; + arg.vdir = vdir; + err = au_do_read_vdir(&arg); + if (!err) { + /* file->f_pos = 0; */ /* todo: ctx->pos? */ + vdir->vd_version = inode->i_version; + vdir->vd_last.ul = 0; + vdir->vd_last.p.deblk = vdir->vd_deblk[0]; + if (allocated) + au_set_ivdir(inode, allocated); + } else if (allocated) + au_vdir_free(allocated); + +out: + return err; +} + +static int copy_vdir(struct au_vdir *tgt, struct au_vdir *src) +{ + int err, rerr; + unsigned long ul, n; + const unsigned int deblk_sz = src->vd_deblk_sz; + + AuDebugOn(tgt->vd_nblk != 1); + + err = -ENOMEM; + if (tgt->vd_nblk < src->vd_nblk) { + unsigned char **p; + + p = au_krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, + GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) + goto out; + tgt->vd_deblk = p; + } + + if (tgt->vd_deblk_sz != deblk_sz) { + unsigned char *p; + + tgt->vd_deblk_sz = deblk_sz; + p = au_krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS, + /*may_shrink*/1); + if (unlikely(!p)) + goto out; + tgt->vd_deblk[0] = p; + } + memcpy(tgt->vd_deblk[0], src->vd_deblk[0], deblk_sz); + tgt->vd_version = src->vd_version; + tgt->vd_jiffy = src->vd_jiffy; + + n = src->vd_nblk; + for (ul = 1; ul < n; ul++) { + tgt->vd_deblk[ul] = kmemdup(src->vd_deblk[ul], deblk_sz, + GFP_NOFS); + if (unlikely(!tgt->vd_deblk[ul])) + goto out; + tgt->vd_nblk++; + } + tgt->vd_nblk = n; + tgt->vd_last.ul = tgt->vd_last.ul; + tgt->vd_last.p.deblk = tgt->vd_deblk[tgt->vd_last.ul]; + tgt->vd_last.p.deblk += src->vd_last.p.deblk + - src->vd_deblk[src->vd_last.ul]; + /* smp_mb(); */ + return 0; /* success */ + +out: + rerr = reinit_vdir(tgt); + BUG_ON(rerr); + return err; +} + +int au_vdir_init(struct file *file) +{ + int err; + struct inode *inode; + struct au_vdir *vdir_cache, *allocated; + + /* test file->f_pos here instead of ctx->pos */ + err = read_vdir(file, !file->f_pos); + if (unlikely(err)) + goto out; + + allocated = NULL; + vdir_cache = au_fvdir_cache(file); + if (!vdir_cache) { + vdir_cache = alloc_vdir(file); + err = PTR_ERR(vdir_cache); + if (IS_ERR(vdir_cache)) + goto out; + allocated = vdir_cache; + } else if (!file->f_pos && vdir_cache->vd_version != file->f_version) { + /* test file->f_pos here instead of ctx->pos */ + err = reinit_vdir(vdir_cache); + if (unlikely(err)) + goto out; + } else + return 0; /* success */ + + inode = file_inode(file); + err = copy_vdir(vdir_cache, au_ivdir(inode)); + if (!err) { + file->f_version = inode->i_version; + if (allocated) + au_set_fvdir_cache(file, allocated); + } else if (allocated) + au_vdir_free(allocated); + +out: + return err; +} + +static loff_t calc_offset(struct au_vdir *vdir) +{ + loff_t offset; + union au_vdir_deblk_p p; + + p.deblk = vdir->vd_deblk[vdir->vd_last.ul]; + offset = vdir->vd_last.p.deblk - p.deblk; + offset += vdir->vd_deblk_sz * vdir->vd_last.ul; + return offset; +} + +/* returns true or false */ +static int seek_vdir(struct file *file, struct dir_context *ctx) +{ + int valid; + unsigned int deblk_sz; + unsigned long ul, n; + loff_t offset; + union au_vdir_deblk_p p, deblk_end; + struct au_vdir *vdir_cache; + + valid = 1; + vdir_cache = au_fvdir_cache(file); + offset = calc_offset(vdir_cache); + AuDbg("offset %lld\n", offset); + if (ctx->pos == offset) + goto out; + + vdir_cache->vd_last.ul = 0; + vdir_cache->vd_last.p.deblk = vdir_cache->vd_deblk[0]; + if (!ctx->pos) + goto out; + + valid = 0; + deblk_sz = vdir_cache->vd_deblk_sz; + ul = div64_u64(ctx->pos, deblk_sz); + AuDbg("ul %lu\n", ul); + if (ul >= vdir_cache->vd_nblk) + goto out; + + n = vdir_cache->vd_nblk; + for (; ul < n; ul++) { + p.deblk = vdir_cache->vd_deblk[ul]; + deblk_end.deblk = p.deblk + deblk_sz; + offset = ul; + offset *= deblk_sz; + while (!is_deblk_end(&p, &deblk_end) && offset < ctx->pos) { + unsigned int l; + + l = calc_size(p.de->de_str.len); + offset += l; + p.deblk += l; + } + if (!is_deblk_end(&p, &deblk_end)) { + valid = 1; + vdir_cache->vd_last.ul = ul; + vdir_cache->vd_last.p = p; + break; + } + } + +out: + /* smp_mb(); */ + if (!valid) + AuDbg("valid %d\n", !valid); + return valid; +} + +int au_vdir_fill_de(struct file *file, struct dir_context *ctx) +{ + unsigned int l, deblk_sz; + union au_vdir_deblk_p deblk_end; + struct au_vdir *vdir_cache; + struct au_vdir_de *de; + + if (!seek_vdir(file, ctx)) + return 0; + + vdir_cache = au_fvdir_cache(file); + deblk_sz = vdir_cache->vd_deblk_sz; + while (1) { + deblk_end.deblk = vdir_cache->vd_deblk[vdir_cache->vd_last.ul]; + deblk_end.deblk += deblk_sz; + while (!is_deblk_end(&vdir_cache->vd_last.p, &deblk_end)) { + de = vdir_cache->vd_last.p.de; + AuDbg("%.*s, off%lld, i%lu, dt%d\n", + de->de_str.len, de->de_str.name, ctx->pos, + (unsigned long)de->de_ino, de->de_type); + if (unlikely(!dir_emit(ctx, de->de_str.name, + de->de_str.len, de->de_ino, + de->de_type))) { + /* todo: ignore the error caused by udba? */ + /* return err; */ + return 0; + } + + l = calc_size(de->de_str.len); + vdir_cache->vd_last.p.deblk += l; + ctx->pos += l; + } + if (vdir_cache->vd_last.ul < vdir_cache->vd_nblk - 1) { + vdir_cache->vd_last.ul++; + vdir_cache->vd_last.p.deblk + = vdir_cache->vd_deblk[vdir_cache->vd_last.ul]; + ctx->pos = deblk_sz * vdir_cache->vd_last.ul; + continue; + } + break; + } + + /* smp_mb(); */ + return 0; +} diff --git a/fs/aufs/vfsub.c b/fs/aufs/vfsub.c new file mode 100644 index 0000000000000..a9074da429ec7 --- /dev/null +++ b/fs/aufs/vfsub.c @@ -0,0 +1,903 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for VFS + */ + +#include +#include +#include +#include +#include +#include "aufs.h" + +#ifdef CONFIG_AUFS_BR_FUSE +int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb) +{ + if (!au_test_fuse(h_sb) || !au_userns) + return 0; + + return is_current_mnt_ns(mnt) ? 0 : -EACCES; +} +#endif + +int vfsub_sync_filesystem(struct super_block *h_sb, int wait) +{ + int err; + + lockdep_off(); + down_read(&h_sb->s_umount); + err = __sync_filesystem(h_sb, wait); + up_read(&h_sb->s_umount); + lockdep_on(); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_update_h_iattr(struct path *h_path, int *did) +{ + int err; + struct kstat st; + struct super_block *h_sb; + + /* for remote fs, leave work for its getattr or d_revalidate */ + /* for bad i_attr fs, handle them in aufs_getattr() */ + /* still some fs may acquire i_mutex. we need to skip them */ + err = 0; + if (!did) + did = &err; + h_sb = h_path->dentry->d_sb; + *did = (!au_test_fs_remote(h_sb) && au_test_fs_refresh_iattr(h_sb)); + if (*did) + err = vfsub_getattr(h_path, &st); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct file *vfsub_dentry_open(struct path *path, int flags) +{ + struct file *file; + + file = dentry_open(path, flags /* | __FMODE_NONOTIFY */, + current_cred()); + if (!IS_ERR_OR_NULL(file) + && (file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) + i_readcount_inc(d_inode(path->dentry)); + + return file; +} + +struct file *vfsub_filp_open(const char *path, int oflags, int mode) +{ + struct file *file; + + lockdep_off(); + file = filp_open(path, + oflags /* | __FMODE_NONOTIFY */, + mode); + lockdep_on(); + if (IS_ERR(file)) + goto out; + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + +out: + return file; +} + +/* + * Ideally this function should call VFS:do_last() in order to keep all its + * checkings. But it is very hard for aufs to regenerate several VFS internal + * structure such as nameidata. This is a second (or third) best approach. + * cf. linux/fs/namei.c:do_last(), lookup_open() and atomic_open(). + */ +int vfsub_atomic_open(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args) +{ + int err; + struct au_branch *br = args->br; + struct file *file = args->file; + /* copied from linux/fs/namei.c:atomic_open() */ + struct dentry *const DENTRY_NOT_SET = (void *)-1UL; + + IMustLock(dir); + AuDebugOn(!dir->i_op->atomic_open); + + err = au_br_test_oflag(args->open_flag, br); + if (unlikely(err)) + goto out; + + au_lcnt_inc(&br->br_nfiles); + file->f_path.dentry = DENTRY_NOT_SET; + file->f_path.mnt = au_br_mnt(br); + err = dir->i_op->atomic_open(dir, dentry, file, args->open_flag, + args->create_mode, args->opened); + if (unlikely(err < 0)) { + au_lcnt_dec(&br->br_nfiles); + goto out; + } + + /* temporary workaround for nfsv4 branch */ + if (au_test_nfs(dir->i_sb)) + nfs_mark_for_revalidate(dir); + + /* some filesystems don't set FILE_CREATED while succeeded? */ + if (*args->opened & FILE_CREATED) + fsnotify_create(dir, dentry); + if (!(*args->opened & FILE_OPENED)) { + au_lcnt_dec(&br->br_nfiles); + goto out; + } + + /* todo: call VFS:may_open() here */ + err = open_check_o_direct(file); + /* todo: ima_file_check() too? */ + if (!err && (args->open_flag & __FMODE_EXEC)) + err = deny_write_access(file); + if (!err) + fsnotify_open(file); + else + au_lcnt_dec(&br->br_nfiles); + /* note that the file is created and still opened */ + +out: + return err; +} + +int vfsub_kern_path(const char *name, unsigned int flags, struct path *path) +{ + int err; + + err = kern_path(name, flags, path); + if (!err && d_is_positive(path->dentry)) + vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/ + return err; +} + +struct dentry *vfsub_lookup_one_len_unlocked(const char *name, + struct dentry *parent, int len) +{ + struct path path = { + .mnt = NULL + }; + + path.dentry = lookup_one_len_unlocked(name, parent, len); + if (IS_ERR(path.dentry)) + goto out; + if (d_is_positive(path.dentry)) + vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/ + +out: + AuTraceErrPtr(path.dentry); + return path.dentry; +} + +struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent, + int len) +{ + struct path path = { + .mnt = NULL + }; + + /* VFS checks it too, but by WARN_ON_ONCE() */ + IMustLock(d_inode(parent)); + + path.dentry = lookup_one_len(name, parent, len); + if (IS_ERR(path.dentry)) + goto out; + if (d_is_positive(path.dentry)) + vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/ + +out: + AuTraceErrPtr(path.dentry); + return path.dentry; +} + +void vfsub_call_lkup_one(void *args) +{ + struct vfsub_lkup_one_args *a = args; + *a->errp = vfsub_lkup_one(a->name, a->parent); +} + +/* ---------------------------------------------------------------------- */ + +struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2) +{ + struct dentry *d; + + lockdep_off(); + d = lock_rename(d1, d2); + lockdep_on(); + au_hn_suspend(hdir1); + if (hdir1 != hdir2) + au_hn_suspend(hdir2); + + return d; +} + +void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2) +{ + au_hn_resume(hdir1); + if (hdir1 != hdir2) + au_hn_resume(hdir2); + lockdep_off(); + unlock_rename(d1, d2); + lockdep_on(); +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_create(struct inode *dir, struct path *path, int mode, bool want_excl) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mknod(path, d, mode, 0); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_create(dir, path->dentry, mode, want_excl); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_symlink(struct inode *dir, struct path *path, const char *symname) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_symlink(path, d, symname); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_symlink(dir, path->dentry, symname); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mknod(path, d, mode, new_encode_dev(dev)); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_mknod(dir, path->dentry, mode, dev); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +static int au_test_nlink(struct inode *inode) +{ + const unsigned int link_max = UINT_MAX >> 1; /* rough margin */ + + if (!au_test_fs_no_limit_nlink(inode->i_sb) + || inode->i_nlink < link_max) + return 0; + return -EMLINK; +} + +int vfsub_link(struct dentry *src_dentry, struct inode *dir, struct path *path, + struct inode **delegated_inode) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + err = au_test_nlink(d_inode(src_dentry)); + if (unlikely(err)) + return err; + + /* we don't call may_linkat() */ + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_link(src_dentry, path, d); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_link(src_dentry, dir, path->dentry, delegated_inode); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + /* fuse has different memory inode for the same inumber */ + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + tmp.dentry = src_dentry; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_rename(struct inode *src_dir, struct dentry *src_dentry, + struct inode *dir, struct path *path, + struct inode **delegated_inode, unsigned int flags) +{ + int err; + struct path tmp = { + .mnt = path->mnt + }; + struct dentry *d; + + IMustLock(dir); + IMustLock(src_dir); + + d = path->dentry; + path->dentry = d->d_parent; + tmp.dentry = src_dentry->d_parent; + err = security_path_rename(&tmp, src_dentry, path, d, /*flags*/0); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_rename(src_dir, src_dentry, dir, path->dentry, + delegated_inode, flags); + lockdep_on(); + if (!err) { + int did; + + tmp.dentry = d->d_parent; + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = src_dentry; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + tmp.dentry = src_dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_mkdir(struct inode *dir, struct path *path, int mode) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mkdir(path, d, mode); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_mkdir(dir, path->dentry, mode); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_rmdir(struct inode *dir, struct path *path) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_rmdir(path, d); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_rmdir(dir, path->dentry); + lockdep_on(); + if (!err) { + struct path tmp = { + .dentry = path->dentry->d_parent, + .mnt = path->mnt + }; + + vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/ + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* todo: support mmap_sem? */ +ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + + lockdep_off(); + err = vfs_read(file, ubuf, count, ppos); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +/* todo: kernel_read()? */ +ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + char __user *u; + } buf; + + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + err = vfsub_read_u(file, buf.u, count, ppos); + set_fs(oldfs); + return err; +} + +ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + + lockdep_off(); + err = vfs_write(file, ubuf, count, ppos); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count, loff_t *ppos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + const char __user *u; + } buf; + + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + err = vfsub_write_u(file, buf.u, count, ppos); + set_fs(oldfs); + return err; +} + +int vfsub_flush(struct file *file, fl_owner_t id) +{ + int err; + + err = 0; + if (file->f_op->flush) { + if (!au_test_nfs(file->f_path.dentry->d_sb)) + err = file->f_op->flush(file, id); + else { + lockdep_off(); + err = file->f_op->flush(file, id); + lockdep_on(); + } + if (!err) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); + /*ignore*/ + } + return err; +} + +int vfsub_iterate_dir(struct file *file, struct dir_context *ctx) +{ + int err; + + AuDbg("%pD, ctx{%pf, %llu}\n", file, ctx->actor, ctx->pos); + + lockdep_off(); + err = iterate_dir(file, ctx); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + + return err; +} + +long vfsub_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + long err; + + lockdep_off(); + err = do_splice_to(in, ppos, pipe, len, flags); + lockdep_on(); + file_accessed(in); + if (err >= 0) + vfsub_update_h_iattr(&in->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + long err; + + lockdep_off(); + err = do_splice_from(pipe, out, ppos, len, flags); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&out->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +int vfsub_fsync(struct file *file, struct path *path, int datasync) +{ + int err; + + /* file can be NULL */ + lockdep_off(); + err = vfs_fsync(file, datasync); + lockdep_on(); + if (!err) { + if (!path) { + AuDebugOn(!file); + path = &file->f_path; + } + vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/ + } + return err; +} + +/* cf. open.c:do_sys_truncate() and do_sys_ftruncate() */ +int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr, + struct file *h_file) +{ + int err; + struct inode *h_inode; + struct super_block *h_sb; + + if (!h_file) { + err = vfsub_truncate(h_path, length); + goto out; + } + + h_inode = d_inode(h_path->dentry); + h_sb = h_inode->i_sb; + lockdep_off(); + sb_start_write(h_sb); + lockdep_on(); + err = locks_verify_truncate(h_inode, h_file, length); + if (!err) + err = security_path_truncate(h_path); + if (!err) { + lockdep_off(); + err = do_truncate(h_path->dentry, length, attr, h_file); + lockdep_on(); + } + lockdep_off(); + sb_end_write(h_sb); + lockdep_on(); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_vfsub_mkdir_args { + int *errp; + struct inode *dir; + struct path *path; + int mode; +}; + +static void au_call_vfsub_mkdir(void *args) +{ + struct au_vfsub_mkdir_args *a = args; + *a->errp = vfsub_mkdir(a->dir, a->path, a->mode); +} + +int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode) +{ + int err, do_sio, wkq_err; + + do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE); + if (!do_sio) { + lockdep_off(); + err = vfsub_mkdir(dir, path, mode); + lockdep_on(); + } else { + struct au_vfsub_mkdir_args args = { + .errp = &err, + .dir = dir, + .path = path, + .mode = mode + }; + wkq_err = au_wkq_wait(au_call_vfsub_mkdir, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} + +struct au_vfsub_rmdir_args { + int *errp; + struct inode *dir; + struct path *path; +}; + +static void au_call_vfsub_rmdir(void *args) +{ + struct au_vfsub_rmdir_args *a = args; + *a->errp = vfsub_rmdir(a->dir, a->path); +} + +int vfsub_sio_rmdir(struct inode *dir, struct path *path) +{ + int err, do_sio, wkq_err; + + do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE); + if (!do_sio) { + lockdep_off(); + err = vfsub_rmdir(dir, path); + lockdep_on(); + } else { + struct au_vfsub_rmdir_args args = { + .errp = &err, + .dir = dir, + .path = path + }; + wkq_err = au_wkq_wait(au_call_vfsub_rmdir, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct notify_change_args { + int *errp; + struct path *path; + struct iattr *ia; + struct inode **delegated_inode; +}; + +static void call_notify_change(void *args) +{ + struct notify_change_args *a = args; + struct inode *h_inode; + + h_inode = d_inode(a->path->dentry); + IMustLock(h_inode); + + *a->errp = -EPERM; + if (!IS_IMMUTABLE(h_inode) && !IS_APPEND(h_inode)) { + lockdep_off(); + *a->errp = notify_change(a->path->dentry, a->ia, + a->delegated_inode); + lockdep_on(); + if (!*a->errp) + vfsub_update_h_iattr(a->path, /*did*/NULL); /*ignore*/ + } + AuTraceErr(*a->errp); +} + +int vfsub_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode) +{ + int err; + struct notify_change_args args = { + .errp = &err, + .path = path, + .ia = ia, + .delegated_inode = delegated_inode + }; + + call_notify_change(&args); + + return err; +} + +int vfsub_sio_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode) +{ + int err, wkq_err; + struct notify_change_args args = { + .errp = &err, + .path = path, + .ia = ia, + .delegated_inode = delegated_inode + }; + + wkq_err = au_wkq_wait(call_notify_change, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct unlink_args { + int *errp; + struct inode *dir; + struct path *path; + struct inode **delegated_inode; +}; + +static void call_unlink(void *args) +{ + struct unlink_args *a = args; + struct dentry *d = a->path->dentry; + struct inode *h_inode; + const int stop_sillyrename = (au_test_nfs(d->d_sb) + && au_dcount(d) == 1); + + IMustLock(a->dir); + + a->path->dentry = d->d_parent; + *a->errp = security_path_unlink(a->path, d); + a->path->dentry = d; + if (unlikely(*a->errp)) + return; + + if (!stop_sillyrename) + dget(d); + h_inode = NULL; + if (d_is_positive(d)) { + h_inode = d_inode(d); + ihold(h_inode); + } + + lockdep_off(); + *a->errp = vfs_unlink(a->dir, d, a->delegated_inode); + lockdep_on(); + if (!*a->errp) { + struct path tmp = { + .dentry = d->d_parent, + .mnt = a->path->mnt + }; + vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/ + } + + if (!stop_sillyrename) + dput(d); + if (h_inode) + iput(h_inode); + + AuTraceErr(*a->errp); +} + +/* + * @dir: must be locked. + * @dentry: target dentry. + */ +int vfsub_unlink(struct inode *dir, struct path *path, + struct inode **delegated_inode, int force) +{ + int err; + struct unlink_args args = { + .errp = &err, + .dir = dir, + .path = path, + .delegated_inode = delegated_inode + }; + + if (!force) + call_unlink(&args); + else { + int wkq_err; + + wkq_err = au_wkq_wait(call_unlink, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} diff --git a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h new file mode 100644 index 0000000000000..2953aa410d096 --- /dev/null +++ b/fs/aufs/vfsub.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for VFS + */ + +#ifndef __AUFS_VFSUB_H__ +#define __AUFS_VFSUB_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include "debug.h" + +/* copied from linux/fs/internal.h */ +/* todo: BAD approach!! */ +extern void __mnt_drop_write(struct vfsmount *); +extern int open_check_o_direct(struct file *f); + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for lower inode */ +/* default MAX_LOCKDEP_SUBCLASSES(8) is not enough */ +/* reduce? gave up. */ +enum { + AuLsc_I_Begin = I_MUTEX_PARENT2, /* 5 */ + AuLsc_I_PARENT, /* lower inode, parent first */ + AuLsc_I_PARENT2, /* copyup dirs */ + AuLsc_I_PARENT3, /* copyup wh */ + AuLsc_I_CHILD, + AuLsc_I_CHILD2, + AuLsc_I_End +}; + +/* to debug easier, do not make them inlined functions */ +#define MtxMustLock(mtx) AuDebugOn(!mutex_is_locked(mtx)) +#define IMustLock(i) AuDebugOn(!inode_is_locked(i)) + +/* why VFS doesn't define it? */ +static inline +void vfsub_inode_lock_shared_nested(struct inode *inode, unsigned int sc) +{ + down_read_nested(&inode->i_rwsem, sc); +} + +/* ---------------------------------------------------------------------- */ + +static inline void vfsub_drop_nlink(struct inode *inode) +{ + AuDebugOn(!inode->i_nlink); + drop_nlink(inode); +} + +static inline void vfsub_dead_dir(struct inode *inode) +{ + AuDebugOn(!S_ISDIR(inode->i_mode)); + inode->i_flags |= S_DEAD; + clear_nlink(inode); +} + +static inline int vfsub_native_ro(struct inode *inode) +{ + return sb_rdonly(inode->i_sb) + || IS_RDONLY(inode) + /* || IS_APPEND(inode) */ + || IS_IMMUTABLE(inode); +} + +#ifdef CONFIG_AUFS_BR_FUSE +int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb); +#else +AuStubInt0(vfsub_test_mntns, struct vfsmount *mnt, struct super_block *h_sb); +#endif + +int vfsub_sync_filesystem(struct super_block *h_sb, int wait); + +/* ---------------------------------------------------------------------- */ + +int vfsub_update_h_iattr(struct path *h_path, int *did); +struct file *vfsub_dentry_open(struct path *path, int flags); +struct file *vfsub_filp_open(const char *path, int oflags, int mode); +struct au_branch; +struct vfsub_aopen_args { + struct file *file; + unsigned int open_flag; + umode_t create_mode; + struct au_branch *br; + int *opened; +}; +int vfsub_atomic_open(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args); +int vfsub_kern_path(const char *name, unsigned int flags, struct path *path); + +struct dentry *vfsub_lookup_one_len_unlocked(const char *name, + struct dentry *parent, int len); +struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent, + int len); + +struct vfsub_lkup_one_args { + struct dentry **errp; + struct qstr *name; + struct dentry *parent; +}; + +static inline struct dentry *vfsub_lkup_one(struct qstr *name, + struct dentry *parent) +{ + return vfsub_lookup_one_len(name->name, parent, name->len); +} + +void vfsub_call_lkup_one(void *args); + +/* ---------------------------------------------------------------------- */ + +static inline int vfsub_mnt_want_write(struct vfsmount *mnt) +{ + int err; + + lockdep_off(); + err = mnt_want_write(mnt); + lockdep_on(); + return err; +} + +static inline void vfsub_mnt_drop_write(struct vfsmount *mnt) +{ + lockdep_off(); + mnt_drop_write(mnt); + lockdep_on(); +} + +#if 0 /* reserved */ +static inline void vfsub_mnt_drop_write_file(struct file *file) +{ + lockdep_off(); + mnt_drop_write_file(file); + lockdep_on(); +} +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_hinode; +struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2); +void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2); + +int vfsub_create(struct inode *dir, struct path *path, int mode, + bool want_excl); +int vfsub_symlink(struct inode *dir, struct path *path, + const char *symname); +int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev); +int vfsub_link(struct dentry *src_dentry, struct inode *dir, + struct path *path, struct inode **delegated_inode); +int vfsub_rename(struct inode *src_hdir, struct dentry *src_dentry, + struct inode *hdir, struct path *path, + struct inode **delegated_inode, unsigned int flags); +int vfsub_mkdir(struct inode *dir, struct path *path, int mode); +int vfsub_rmdir(struct inode *dir, struct path *path); + +/* ---------------------------------------------------------------------- */ + +ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count, + loff_t *ppos); +ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos); +ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count, + loff_t *ppos); +ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos); +int vfsub_flush(struct file *file, fl_owner_t id); +int vfsub_iterate_dir(struct file *file, struct dir_context *ctx); + +static inline loff_t vfsub_f_size_read(struct file *file) +{ + return i_size_read(file_inode(file)); +} + +static inline unsigned int vfsub_file_flags(struct file *file) +{ + unsigned int flags; + + spin_lock(&file->f_lock); + flags = file->f_flags; + spin_unlock(&file->f_lock); + + return flags; +} + +static inline int vfsub_file_execed(struct file *file) +{ + /* todo: direct access f_flags */ + return !!(vfsub_file_flags(file) & __FMODE_EXEC); +} + +#if 0 /* reserved */ +static inline void vfsub_file_accessed(struct file *h_file) +{ + file_accessed(h_file); + vfsub_update_h_iattr(&h_file->f_path, /*did*/NULL); /*ignore*/ +} +#endif + +#if 0 /* reserved */ +static inline void vfsub_touch_atime(struct vfsmount *h_mnt, + struct dentry *h_dentry) +{ + struct path h_path = { + .dentry = h_dentry, + .mnt = h_mnt + }; + touch_atime(&h_path); + vfsub_update_h_iattr(&h_path, /*did*/NULL); /*ignore*/ +} +#endif + +static inline int vfsub_update_time(struct inode *h_inode, struct timespec *ts, + int flags) +{ + return update_time(h_inode, ts, flags); + /* no vfsub_update_h_iattr() since we don't have struct path */ +} + +#ifdef CONFIG_FS_POSIX_ACL +static inline int vfsub_acl_chmod(struct inode *h_inode, umode_t h_mode) +{ + int err; + + err = posix_acl_chmod(h_inode, h_mode); + if (err == -EOPNOTSUPP) + err = 0; + return err; +} +#else +AuStubInt0(vfsub_acl_chmod, struct inode *h_inode, umode_t h_mode); +#endif + +long vfsub_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); +long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags); + +static inline long vfsub_truncate(struct path *path, loff_t length) +{ + long err; + + lockdep_off(); + err = vfs_truncate(path, length); + lockdep_on(); + return err; +} + +int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr, + struct file *h_file); +int vfsub_fsync(struct file *file, struct path *path, int datasync); + +/* + * re-use branch fs's ioctl(FICLONE) while aufs itself doesn't support such + * ioctl. + */ +static inline int vfsub_clone_file_range(struct file *src, struct file *dst, + u64 len) +{ + int err; + + lockdep_off(); + err = vfs_clone_file_range(src, 0, dst, 0, len); + lockdep_on(); + + return err; +} + +/* copy_file_range(2) is a systemcall */ +static inline ssize_t vfsub_copy_file_range(struct file *src, loff_t src_pos, + struct file *dst, loff_t dst_pos, + size_t len, unsigned int flags) +{ + ssize_t ssz; + + lockdep_off(); + ssz = vfs_copy_file_range(src, src_pos, dst, dst_pos, len, flags); + lockdep_on(); + + return ssz; +} + +/* ---------------------------------------------------------------------- */ + +static inline loff_t vfsub_llseek(struct file *file, loff_t offset, int origin) +{ + loff_t err; + + lockdep_off(); + err = vfs_llseek(file, offset, origin); + lockdep_on(); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode); +int vfsub_sio_rmdir(struct inode *dir, struct path *path); +int vfsub_sio_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode); +int vfsub_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode); +int vfsub_unlink(struct inode *dir, struct path *path, + struct inode **delegated_inode, int force); + +static inline int vfsub_getattr(const struct path *path, struct kstat *st) +{ + return vfs_getattr(path, st, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); +} + +/* ---------------------------------------------------------------------- */ + +static inline int vfsub_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + int err; + + lockdep_off(); + err = vfs_setxattr(dentry, name, value, size, flags); + lockdep_on(); + + return err; +} + +static inline int vfsub_removexattr(struct dentry *dentry, const char *name) +{ + int err; + + lockdep_off(); + err = vfs_removexattr(dentry, name); + lockdep_on(); + + return err; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_VFSUB_H__ */ diff --git a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c new file mode 100644 index 0000000000000..64954145c0ec0 --- /dev/null +++ b/fs/aufs/wbr_policy.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * policies for selecting one among multiple writable branches + */ + +#include +#include "aufs.h" + +/* subset of cpup_attr() */ +static noinline_for_stack +int au_cpdown_attr(struct path *h_path, struct dentry *h_src) +{ + int err, sbits; + struct iattr ia; + struct inode *h_isrc; + + h_isrc = d_inode(h_src); + ia.ia_valid = ATTR_FORCE | ATTR_MODE | ATTR_UID | ATTR_GID; + ia.ia_mode = h_isrc->i_mode; + ia.ia_uid = h_isrc->i_uid; + ia.ia_gid = h_isrc->i_gid; + sbits = !!(ia.ia_mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(d_inode(h_path->dentry), h_isrc->i_flags); + /* no delegation since it is just created */ + err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL); + + /* is this nfs only? */ + if (!err && sbits && au_test_nfs(h_path->dentry->d_sb)) { + ia.ia_valid = ATTR_FORCE | ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL); + } + + return err; +} + +#define AuCpdown_PARENT_OPQ 1 +#define AuCpdown_WHED (1 << 1) +#define AuCpdown_MADE_DIR (1 << 2) +#define AuCpdown_DIROPQ (1 << 3) +#define au_ftest_cpdown(flags, name) ((flags) & AuCpdown_##name) +#define au_fset_cpdown(flags, name) \ + do { (flags) |= AuCpdown_##name; } while (0) +#define au_fclr_cpdown(flags, name) \ + do { (flags) &= ~AuCpdown_##name; } while (0) + +static int au_cpdown_dir_opq(struct dentry *dentry, aufs_bindex_t bdst, + unsigned int *flags) +{ + int err; + struct dentry *opq_dentry; + + opq_dentry = au_diropq_create(dentry, bdst); + err = PTR_ERR(opq_dentry); + if (IS_ERR(opq_dentry)) + goto out; + dput(opq_dentry); + au_fset_cpdown(*flags, DIROPQ); + +out: + return err; +} + +static int au_cpdown_dir_wh(struct dentry *dentry, struct dentry *h_parent, + struct inode *dir, aufs_bindex_t bdst) +{ + int err; + struct path h_path; + struct au_branch *br; + + br = au_sbr(dentry->d_sb, bdst); + h_path.dentry = au_wh_lkup(h_parent, &dentry->d_name, br); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + err = 0; + if (d_is_positive(h_path.dentry)) { + h_path.mnt = au_br_mnt(br); + err = au_wh_unlink_dentry(au_h_iptr(dir, bdst), &h_path, + dentry); + } + dput(h_path.dentry); + +out: + return err; +} + +static int au_cpdown_dir(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg) +{ + int err, rerr; + aufs_bindex_t bopq, btop; + struct path h_path; + struct dentry *parent; + struct inode *h_dir, *h_inode, *inode, *dir; + unsigned int *flags = arg; + + btop = au_dbtop(dentry); + /* dentry is di-locked */ + parent = dget_parent(dentry); + dir = d_inode(parent); + h_dir = d_inode(h_parent); + AuDebugOn(h_dir != au_h_iptr(dir, bdst)); + IMustLock(h_dir); + + err = au_lkup_neg(dentry, bdst, /*wh*/0); + if (unlikely(err < 0)) + goto out; + h_path.dentry = au_h_dptr(dentry, bdst); + h_path.mnt = au_sbr_mnt(dentry->d_sb, bdst); + err = vfsub_sio_mkdir(au_h_iptr(dir, bdst), &h_path, 0755); + if (unlikely(err)) + goto out_put; + au_fset_cpdown(*flags, MADE_DIR); + + bopq = au_dbdiropq(dentry); + au_fclr_cpdown(*flags, WHED); + au_fclr_cpdown(*flags, DIROPQ); + if (au_dbwh(dentry) == bdst) + au_fset_cpdown(*flags, WHED); + if (!au_ftest_cpdown(*flags, PARENT_OPQ) && bopq <= bdst) + au_fset_cpdown(*flags, PARENT_OPQ); + h_inode = d_inode(h_path.dentry); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + if (au_ftest_cpdown(*flags, WHED)) { + err = au_cpdown_dir_opq(dentry, bdst, flags); + if (unlikely(err)) { + inode_unlock(h_inode); + goto out_dir; + } + } + + err = au_cpdown_attr(&h_path, au_h_dptr(dentry, btop)); + inode_unlock(h_inode); + if (unlikely(err)) + goto out_opq; + + if (au_ftest_cpdown(*flags, WHED)) { + err = au_cpdown_dir_wh(dentry, h_parent, dir, bdst); + if (unlikely(err)) + goto out_opq; + } + + inode = d_inode(dentry); + if (au_ibbot(inode) < bdst) + au_set_ibbot(inode, bdst); + au_set_h_iptr(inode, bdst, au_igrab(h_inode), + au_hi_flags(inode, /*isdir*/1)); + au_fhsm_wrote(dentry->d_sb, bdst, /*force*/0); + goto out; /* success */ + + /* revert */ +out_opq: + if (au_ftest_cpdown(*flags, DIROPQ)) { + inode_lock_nested(h_inode, AuLsc_I_CHILD); + rerr = au_diropq_remove(dentry, bdst); + inode_unlock(h_inode); + if (unlikely(rerr)) { + AuIOErr("failed removing diropq for %pd b%d (%d)\n", + dentry, bdst, rerr); + err = -EIO; + goto out; + } + } +out_dir: + if (au_ftest_cpdown(*flags, MADE_DIR)) { + rerr = vfsub_sio_rmdir(au_h_iptr(dir, bdst), &h_path); + if (unlikely(rerr)) { + AuIOErr("failed removing %pd b%d (%d)\n", + dentry, bdst, rerr); + err = -EIO; + } + } +out_put: + au_set_h_dptr(dentry, bdst, NULL); + if (au_dbbot(dentry) == bdst) + au_update_dbbot(dentry); +out: + dput(parent); + return err; +} + +int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + int err; + unsigned int flags; + + flags = 0; + err = au_cp_dirs(dentry, bdst, au_cpdown_dir, &flags); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* policies for create */ + +int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex) +{ + int err, i, j, ndentry; + aufs_bindex_t bopq; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries, *parent, *d; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + parent = dget_parent(dentry); + err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/0); + if (unlikely(err)) + goto out_free; + + err = bindex; + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + d = dentries[j]; + di_read_lock_parent2(d, !AuLock_IR); + bopq = au_dbdiropq(d); + di_read_unlock(d, !AuLock_IR); + if (bopq >= 0 && bopq < err) + err = bopq; + } + } + +out_free: + dput(parent); + au_dpages_free(&dpages); +out: + return err; +} + +static int au_wbr_bu(struct super_block *sb, aufs_bindex_t bindex) +{ + for (; bindex >= 0; bindex--) + if (!au_br_rdonly(au_sbr(sb, bindex))) + return bindex; + return -EROFS; +} + +/* top down parent */ +static int au_wbr_create_tdp(struct dentry *dentry, + unsigned int flags __maybe_unused) +{ + int err; + aufs_bindex_t btop, bindex; + struct super_block *sb; + struct dentry *parent, *h_parent; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + err = btop; + if (!au_br_rdonly(au_sbr(sb, btop))) + goto out; + + err = -EROFS; + parent = dget_parent(dentry); + for (bindex = au_dbtop(parent); bindex < btop; bindex++) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = bindex; + break; + } + } + dput(parent); + + /* bottom up here */ + if (unlikely(err < 0)) { + err = au_wbr_bu(sb, btop - 1); + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + } + +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* an exception for the policy other than tdp */ +static int au_wbr_create_exp(struct dentry *dentry) +{ + int err; + aufs_bindex_t bwh, bdiropq; + struct dentry *parent; + + err = -1; + bwh = au_dbwh(dentry); + parent = dget_parent(dentry); + bdiropq = au_dbdiropq(parent); + if (bwh >= 0) { + if (bdiropq >= 0) + err = min(bdiropq, bwh); + else + err = bwh; + AuDbg("%d\n", err); + } else if (bdiropq >= 0) { + err = bdiropq; + AuDbg("%d\n", err); + } + dput(parent); + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + + if (err >= 0 && au_br_rdonly(au_sbr(dentry->d_sb, err))) + err = -1; + + AuDbg("%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* round robin */ +static int au_wbr_create_init_rr(struct super_block *sb) +{ + int err; + + err = au_wbr_bu(sb, au_sbbot(sb)); + atomic_set(&au_sbi(sb)->si_wbr_rr_next, -err); /* less important */ + /* smp_mb(); */ + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_rr(struct dentry *dentry, unsigned int flags) +{ + int err, nbr; + unsigned int u; + aufs_bindex_t bindex, bbot; + struct super_block *sb; + atomic_t *next; + + err = au_wbr_create_exp(dentry); + if (err >= 0) + goto out; + + sb = dentry->d_sb; + next = &au_sbi(sb)->si_wbr_rr_next; + bbot = au_sbbot(sb); + nbr = bbot + 1; + for (bindex = 0; bindex <= bbot; bindex++) { + if (!au_ftest_wbr(flags, DIR)) { + err = atomic_dec_return(next) + 1; + /* modulo for 0 is meaningless */ + if (unlikely(!err)) + err = atomic_dec_return(next) + 1; + } else + err = atomic_read(next); + AuDbg("%d\n", err); + u = err; + err = u % nbr; + AuDbg("%d\n", err); + if (!au_br_rdonly(au_sbr(sb, err))) + break; + err = -EROFS; + } + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out: + AuDbg("%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* most free space */ +static void au_mfs(struct dentry *dentry, struct dentry *parent) +{ + struct super_block *sb; + struct au_branch *br; + struct au_wbr_mfs *mfs; + struct dentry *h_parent; + aufs_bindex_t bindex, bbot; + int err; + unsigned long long b, bavail; + struct path h_path; + /* reduce the stack usage */ + struct kstatfs *st; + + st = kmalloc(sizeof(*st), GFP_NOFS); + if (unlikely(!st)) { + AuWarn1("failed updating mfs(%d), ignored\n", -ENOMEM); + return; + } + + bavail = 0; + sb = dentry->d_sb; + mfs = &au_sbi(sb)->si_wbr_mfs; + MtxMustLock(&mfs->mfs_lock); + mfs->mfs_bindex = -EROFS; + mfs->mfsrr_bytes = 0; + if (!parent) { + bindex = 0; + bbot = au_sbbot(sb); + } else { + bindex = au_dbtop(parent); + bbot = au_dbtaildir(parent); + } + + for (; bindex <= bbot; bindex++) { + if (parent) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + } + br = au_sbr(sb, bindex); + if (au_br_rdonly(br)) + continue; + + /* sb->s_root for NFS is unreliable */ + h_path.mnt = au_br_mnt(br); + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, st); + if (unlikely(err)) { + AuWarn1("failed statfs, b%d, %d\n", bindex, err); + continue; + } + + /* when the available size is equal, select the lower one */ + BUILD_BUG_ON(sizeof(b) < sizeof(st->f_bavail) + || sizeof(b) < sizeof(st->f_bsize)); + b = st->f_bavail * st->f_bsize; + br->br_wbr->wbr_bytes = b; + if (b >= bavail) { + bavail = b; + mfs->mfs_bindex = bindex; + mfs->mfs_jiffy = jiffies; + } + } + + mfs->mfsrr_bytes = bavail; + AuDbg("b%d\n", mfs->mfs_bindex); + au_kfree_rcu(st); +} + +static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags) +{ + int err; + struct dentry *parent; + struct super_block *sb; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_exp(dentry); + if (err >= 0) + goto out; + + sb = dentry->d_sb; + parent = NULL; + if (au_ftest_wbr(flags, PARENT)) + parent = dget_parent(dentry); + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire) + || mfs->mfs_bindex < 0 + || au_br_rdonly(au_sbr(sb, mfs->mfs_bindex))) + au_mfs(dentry, parent); + mutex_unlock(&mfs->mfs_lock); + err = mfs->mfs_bindex; + dput(parent); + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out: + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_init_mfs(struct super_block *sb) +{ + struct au_wbr_mfs *mfs; + + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_init(&mfs->mfs_lock); + mfs->mfs_jiffy = 0; + mfs->mfs_bindex = -EROFS; + + return 0; +} + +static int au_wbr_create_fin_mfs(struct super_block *sb __maybe_unused) +{ + mutex_destroy(&au_sbi(sb)->si_wbr_mfs.mfs_lock); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +/* top down regardless parent, and then mfs */ +static int au_wbr_create_tdmfs(struct dentry *dentry, + unsigned int flags __maybe_unused) +{ + int err; + aufs_bindex_t bwh, btail, bindex, bfound, bmfs; + unsigned long long watermark; + struct super_block *sb; + struct au_wbr_mfs *mfs; + struct au_branch *br; + struct dentry *parent; + + sb = dentry->d_sb; + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire) + || mfs->mfs_bindex < 0) + au_mfs(dentry, /*parent*/NULL); + watermark = mfs->mfsrr_watermark; + bmfs = mfs->mfs_bindex; + mutex_unlock(&mfs->mfs_lock); + + /* another style of au_wbr_create_exp() */ + bwh = au_dbwh(dentry); + parent = dget_parent(dentry); + btail = au_dbtaildir(parent); + if (bwh >= 0 && bwh < btail) + btail = bwh; + + err = au_wbr_nonopq(dentry, btail); + if (unlikely(err < 0)) + goto out; + btail = err; + bfound = -1; + for (bindex = 0; bindex <= btail; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_rdonly(br)) + continue; + if (br->br_wbr->wbr_bytes > watermark) { + bfound = bindex; + break; + } + } + err = bfound; + if (err < 0) + err = bmfs; + +out: + dput(parent); + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* most free space and then round robin */ +static int au_wbr_create_mfsrr(struct dentry *dentry, unsigned int flags) +{ + int err; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_mfs(dentry, flags); + if (err >= 0) { + mfs = &au_sbi(dentry->d_sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (mfs->mfsrr_bytes < mfs->mfsrr_watermark) + err = au_wbr_create_rr(dentry, flags); + mutex_unlock(&mfs->mfs_lock); + } + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_init_mfsrr(struct super_block *sb) +{ + int err; + + au_wbr_create_init_mfs(sb); /* ignore */ + err = au_wbr_create_init_rr(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* top down parent and most free space */ +static int au_wbr_create_pmfs(struct dentry *dentry, unsigned int flags) +{ + int err, e2; + unsigned long long b; + aufs_bindex_t bindex, btop, bbot; + struct super_block *sb; + struct dentry *parent, *h_parent; + struct au_branch *br; + + err = au_wbr_create_tdp(dentry, flags); + if (unlikely(err < 0)) + goto out; + parent = dget_parent(dentry); + btop = au_dbtop(parent); + bbot = au_dbtaildir(parent); + if (btop == bbot) + goto out_parent; /* success */ + + e2 = au_wbr_create_mfs(dentry, flags); + if (e2 < 0) + goto out_parent; /* success */ + + /* when the available size is equal, select upper one */ + sb = dentry->d_sb; + br = au_sbr(sb, err); + b = br->br_wbr->wbr_bytes; + AuDbg("b%d, %llu\n", err, b); + + for (bindex = btop; bindex <= bbot; bindex++) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + br = au_sbr(sb, bindex); + if (!au_br_rdonly(br) && br->br_wbr->wbr_bytes > b) { + b = br->br_wbr->wbr_bytes; + err = bindex; + AuDbg("b%d, %llu\n", err, b); + } + } + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out_parent: + dput(parent); +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * - top down parent + * - most free space with parent + * - most free space round-robin regardless parent + */ +static int au_wbr_create_pmfsrr(struct dentry *dentry, unsigned int flags) +{ + int err; + unsigned long long watermark; + struct super_block *sb; + struct au_branch *br; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_pmfs(dentry, flags | AuWbr_PARENT); + if (unlikely(err < 0)) + goto out; + + sb = dentry->d_sb; + br = au_sbr(sb, err); + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + watermark = mfs->mfsrr_watermark; + mutex_unlock(&mfs->mfs_lock); + if (br->br_wbr->wbr_bytes < watermark) + /* regardless the parent dir */ + err = au_wbr_create_mfsrr(dentry, flags); + +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* policies for copyup */ + +/* top down parent */ +static int au_wbr_copyup_tdp(struct dentry *dentry) +{ + return au_wbr_create_tdp(dentry, /*flags, anything is ok*/0); +} + +/* bottom up parent */ +static int au_wbr_copyup_bup(struct dentry *dentry) +{ + int err; + aufs_bindex_t bindex, btop; + struct dentry *parent, *h_parent; + struct super_block *sb; + + err = -EROFS; + sb = dentry->d_sb; + parent = dget_parent(dentry); + btop = au_dbtop(parent); + for (bindex = au_dbtop(dentry); bindex >= btop; bindex--) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = bindex; + break; + } + } + dput(parent); + + /* bottom up here */ + if (unlikely(err < 0)) + err = au_wbr_bu(sb, btop - 1); + + AuDbg("b%d\n", err); + return err; +} + +/* bottom up */ +int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop) +{ + int err; + + err = au_wbr_bu(dentry->d_sb, btop); + AuDbg("b%d\n", err); + if (err > btop) + err = au_wbr_nonopq(dentry, err); + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_copyup_bu(struct dentry *dentry) +{ + int err; + aufs_bindex_t btop; + + btop = au_dbtop(dentry); + err = au_wbr_do_copyup_bu(dentry, btop); + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_wbr_copyup_operations au_wbr_copyup_ops[] = { + [AuWbrCopyup_TDP] = { + .copyup = au_wbr_copyup_tdp + }, + [AuWbrCopyup_BUP] = { + .copyup = au_wbr_copyup_bup + }, + [AuWbrCopyup_BU] = { + .copyup = au_wbr_copyup_bu + } +}; + +struct au_wbr_create_operations au_wbr_create_ops[] = { + [AuWbrCreate_TDP] = { + .create = au_wbr_create_tdp + }, + [AuWbrCreate_RR] = { + .create = au_wbr_create_rr, + .init = au_wbr_create_init_rr + }, + [AuWbrCreate_MFS] = { + .create = au_wbr_create_mfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSV] = { + .create = au_wbr_create_mfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSRR] = { + .create = au_wbr_create_mfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSRRV] = { + .create = au_wbr_create_mfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_TDMFS] = { + .create = au_wbr_create_tdmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_TDMFSV] = { + .create = au_wbr_create_tdmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFS] = { + .create = au_wbr_create_pmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSV] = { + .create = au_wbr_create_pmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSRR] = { + .create = au_wbr_create_pmfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSRRV] = { + .create = au_wbr_create_pmfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + } +}; diff --git a/fs/aufs/whout.c b/fs/aufs/whout.c new file mode 100644 index 0000000000000..6c9c8501562ae --- /dev/null +++ b/fs/aufs/whout.c @@ -0,0 +1,1062 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * whiteout for logical deletion and opaque directory + */ + +#include "aufs.h" + +#define WH_MASK 0444 + +/* + * If a directory contains this file, then it is opaque. We start with the + * .wh. flag so that it is blocked by lookup. + */ +static struct qstr diropq_name = QSTR_INIT(AUFS_WH_DIROPQ, + sizeof(AUFS_WH_DIROPQ) - 1); + +/* + * generate whiteout name, which is NOT terminated by NULL. + * @name: original d_name.name + * @len: original d_name.len + * @wh: whiteout qstr + * returns zero when succeeds, otherwise error. + * succeeded value as wh->name should be freed by kfree(). + */ +int au_wh_name_alloc(struct qstr *wh, const struct qstr *name) +{ + char *p; + + if (unlikely(name->len > PATH_MAX - AUFS_WH_PFX_LEN)) + return -ENAMETOOLONG; + + wh->len = name->len + AUFS_WH_PFX_LEN; + p = kmalloc(wh->len, GFP_NOFS); + wh->name = p; + if (p) { + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + memcpy(p + AUFS_WH_PFX_LEN, name->name, name->len); + /* smp_mb(); */ + return 0; + } + return -ENOMEM; +} + +/* ---------------------------------------------------------------------- */ + +/* + * test if the @wh_name exists under @h_parent. + * @try_sio specifies the necessary of super-io. + */ +int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio) +{ + int err; + struct dentry *wh_dentry; + + if (!try_sio) + wh_dentry = vfsub_lkup_one(wh_name, h_parent); + else + wh_dentry = au_sio_lkup_one(wh_name, h_parent); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) { + if (err == -ENAMETOOLONG) + err = 0; + goto out; + } + + err = 0; + if (d_is_negative(wh_dentry)) + goto out_wh; /* success */ + + err = 1; + if (d_is_reg(wh_dentry)) + goto out_wh; /* success */ + + err = -EIO; + AuIOErr("%pd Invalid whiteout entry type 0%o.\n", + wh_dentry, d_inode(wh_dentry)->i_mode); + +out_wh: + dput(wh_dentry); +out: + return err; +} + +/* + * test if the @h_dentry sets opaque or not. + */ +int au_diropq_test(struct dentry *h_dentry) +{ + int err; + struct inode *h_dir; + + h_dir = d_inode(h_dentry); + err = au_wh_test(h_dentry, &diropq_name, + au_test_h_perm_sio(h_dir, MAY_EXEC)); + return err; +} + +/* + * returns a negative dentry whose name is unique and temporary. + */ +struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br, + struct qstr *prefix) +{ + struct dentry *dentry; + int i; + char defname[NAME_MAX - AUFS_MAX_NAMELEN + DNAME_INLINE_LEN + 1], + *name, *p; + /* strict atomic_t is unnecessary here */ + static unsigned short cnt; + struct qstr qs; + + BUILD_BUG_ON(sizeof(cnt) * 2 > AUFS_WH_TMP_LEN); + + name = defname; + qs.len = sizeof(defname) - DNAME_INLINE_LEN + prefix->len - 1; + if (unlikely(prefix->len > DNAME_INLINE_LEN)) { + dentry = ERR_PTR(-ENAMETOOLONG); + if (unlikely(qs.len > NAME_MAX)) + goto out; + dentry = ERR_PTR(-ENOMEM); + name = kmalloc(qs.len + 1, GFP_NOFS); + if (unlikely(!name)) + goto out; + } + + /* doubly whiteout-ed */ + memcpy(name, AUFS_WH_PFX AUFS_WH_PFX, AUFS_WH_PFX_LEN * 2); + p = name + AUFS_WH_PFX_LEN * 2; + memcpy(p, prefix->name, prefix->len); + p += prefix->len; + *p++ = '.'; + AuDebugOn(name + qs.len + 1 - p <= AUFS_WH_TMP_LEN); + + qs.name = name; + for (i = 0; i < 3; i++) { + sprintf(p, "%.*x", AUFS_WH_TMP_LEN, cnt++); + dentry = au_sio_lkup_one(&qs, h_parent); + if (IS_ERR(dentry) || d_is_negative(dentry)) + goto out_name; + dput(dentry); + } + /* pr_warn("could not get random name\n"); */ + dentry = ERR_PTR(-EEXIST); + AuDbg("%.*s\n", AuLNPair(&qs)); + BUG(); + +out_name: + if (name != defname) + au_kfree_try_rcu(name); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* + * rename the @h_dentry on @br to the whiteouted temporary name. + */ +int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + struct inode *h_dir, *delegated; + struct dentry *h_parent; + + h_parent = h_dentry->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + h_path.dentry = au_whtmp_lkup(h_parent, br, &h_dentry->d_name); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + /* under the same dir, no need to lock_rename() */ + delegated = NULL; + err = vfsub_rename(h_dir, h_dentry, h_dir, &h_path, &delegated, + /*flags*/0); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + dput(h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * functions for removing a whiteout + */ + +static int do_unlink_wh(struct inode *h_dir, struct path *h_path) +{ + int err, force; + struct inode *delegated; + + /* + * forces superio when the dir has a sticky bit. + * this may be a violation of unix fs semantics. + */ + force = (h_dir->i_mode & S_ISVTX) + && !uid_eq(current_fsuid(), d_inode(h_path->dentry)->i_uid); + delegated = NULL; + err = vfsub_unlink(h_dir, h_path, &delegated, force); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + return err; +} + +int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path, + struct dentry *dentry) +{ + int err; + + err = do_unlink_wh(h_dir, h_path); + if (!err && dentry) + au_set_dbwh(dentry, -1); + + return err; +} + +static int unlink_wh_name(struct dentry *h_parent, struct qstr *wh, + struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + + err = 0; + h_path.dentry = vfsub_lkup_one(wh, h_parent); + if (IS_ERR(h_path.dentry)) + err = PTR_ERR(h_path.dentry); + else { + if (d_is_reg(h_path.dentry)) + err = do_unlink_wh(d_inode(h_parent), &h_path); + dput(h_path.dentry); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * initialize/clean whiteout for a branch + */ + +static void au_wh_clean(struct inode *h_dir, struct path *whpath, + const int isdir) +{ + int err; + struct inode *delegated; + + if (d_is_negative(whpath->dentry)) + return; + + if (isdir) + err = vfsub_rmdir(h_dir, whpath); + else { + delegated = NULL; + err = vfsub_unlink(h_dir, whpath, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } + if (unlikely(err)) + pr_warn("failed removing %pd (%d), ignored.\n", + whpath->dentry, err); +} + +static int test_linkable(struct dentry *h_root) +{ + struct inode *h_dir = d_inode(h_root); + + if (h_dir->i_op->link) + return 0; + + pr_err("%pd (%s) doesn't support link(2), use noplink and rw+nolwh\n", + h_root, au_sbtype(h_root->d_sb)); + return -ENOSYS; +} + +/* todo: should this mkdir be done in /sbin/mount.aufs helper? */ +static int au_whdir(struct inode *h_dir, struct path *path) +{ + int err; + + err = -EEXIST; + if (d_is_negative(path->dentry)) { + int mode = 0700; + + if (au_test_nfs(path->dentry->d_sb)) + mode |= 0111; + err = vfsub_mkdir(h_dir, path, mode); + } else if (d_is_dir(path->dentry)) + err = 0; + else + pr_err("unknown %pd exists\n", path->dentry); + + return err; +} + +struct au_wh_base { + const struct qstr *name; + struct dentry *dentry; +}; + +static void au_wh_init_ro(struct inode *h_dir, struct au_wh_base base[], + struct path *h_path) +{ + h_path->dentry = base[AuBrWh_BASE].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/0); + h_path->dentry = base[AuBrWh_PLINK].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/1); + h_path->dentry = base[AuBrWh_ORPH].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/1); +} + +/* + * returns tri-state, + * minus: error, caller should print the message + * zero: success + * plus: error, caller should NOT print the message + */ +static int au_wh_init_rw_nolink(struct dentry *h_root, struct au_wbr *wbr, + int do_plink, struct au_wh_base base[], + struct path *h_path) +{ + int err; + struct inode *h_dir; + + h_dir = d_inode(h_root); + h_path->dentry = base[AuBrWh_BASE].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/0); + h_path->dentry = base[AuBrWh_PLINK].dentry; + if (do_plink) { + err = test_linkable(h_root); + if (unlikely(err)) { + err = 1; + goto out; + } + + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry); + } else + au_wh_clean(h_dir, h_path, /*isdir*/1); + h_path->dentry = base[AuBrWh_ORPH].dentry; + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry); + +out: + return err; +} + +/* + * for the moment, aufs supports the branch filesystem which does not support + * link(2). testing on FAT which does not support i_op->setattr() fully either, + * copyup failed. finally, such filesystem will not be used as the writable + * branch. + * + * returns tri-state, see above. + */ +static int au_wh_init_rw(struct dentry *h_root, struct au_wbr *wbr, + int do_plink, struct au_wh_base base[], + struct path *h_path) +{ + int err; + struct inode *h_dir; + + WbrWhMustWriteLock(wbr); + + err = test_linkable(h_root); + if (unlikely(err)) { + err = 1; + goto out; + } + + /* + * todo: should this create be done in /sbin/mount.aufs helper? + */ + err = -EEXIST; + h_dir = d_inode(h_root); + if (d_is_negative(base[AuBrWh_BASE].dentry)) { + h_path->dentry = base[AuBrWh_BASE].dentry; + err = vfsub_create(h_dir, h_path, WH_MASK, /*want_excl*/true); + } else if (d_is_reg(base[AuBrWh_BASE].dentry)) + err = 0; + else + pr_err("unknown %pd2 exists\n", base[AuBrWh_BASE].dentry); + if (unlikely(err)) + goto out; + + h_path->dentry = base[AuBrWh_PLINK].dentry; + if (do_plink) { + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry); + } else + au_wh_clean(h_dir, h_path, /*isdir*/1); + wbr->wbr_whbase = dget(base[AuBrWh_BASE].dentry); + + h_path->dentry = base[AuBrWh_ORPH].dentry; + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry); + +out: + return err; +} + +/* + * initialize the whiteout base file/dir for @br. + */ +int au_wh_init(struct au_branch *br, struct super_block *sb) +{ + int err, i; + const unsigned char do_plink + = !!au_opt_test(au_mntflags(sb), PLINK); + struct inode *h_dir; + struct path path = br->br_path; + struct dentry *h_root = path.dentry; + struct au_wbr *wbr = br->br_wbr; + static const struct qstr base_name[] = { + [AuBrWh_BASE] = QSTR_INIT(AUFS_BASE_NAME, + sizeof(AUFS_BASE_NAME) - 1), + [AuBrWh_PLINK] = QSTR_INIT(AUFS_PLINKDIR_NAME, + sizeof(AUFS_PLINKDIR_NAME) - 1), + [AuBrWh_ORPH] = QSTR_INIT(AUFS_ORPHDIR_NAME, + sizeof(AUFS_ORPHDIR_NAME) - 1) + }; + struct au_wh_base base[] = { + [AuBrWh_BASE] = { + .name = base_name + AuBrWh_BASE, + .dentry = NULL + }, + [AuBrWh_PLINK] = { + .name = base_name + AuBrWh_PLINK, + .dentry = NULL + }, + [AuBrWh_ORPH] = { + .name = base_name + AuBrWh_ORPH, + .dentry = NULL + } + }; + + if (wbr) + WbrWhMustWriteLock(wbr); + + for (i = 0; i < AuBrWh_Last; i++) { + /* doubly whiteouted */ + struct dentry *d; + + d = au_wh_lkup(h_root, (void *)base[i].name, br); + err = PTR_ERR(d); + if (IS_ERR(d)) + goto out; + + base[i].dentry = d; + AuDebugOn(wbr + && wbr->wbr_wh[i] + && wbr->wbr_wh[i] != base[i].dentry); + } + + if (wbr) + for (i = 0; i < AuBrWh_Last; i++) { + dput(wbr->wbr_wh[i]); + wbr->wbr_wh[i] = NULL; + } + + err = 0; + if (!au_br_writable(br->br_perm)) { + h_dir = d_inode(h_root); + au_wh_init_ro(h_dir, base, &path); + } else if (!au_br_wh_linkable(br->br_perm)) { + err = au_wh_init_rw_nolink(h_root, wbr, do_plink, base, &path); + if (err > 0) + goto out; + else if (err) + goto out_err; + } else { + err = au_wh_init_rw(h_root, wbr, do_plink, base, &path); + if (err > 0) + goto out; + else if (err) + goto out_err; + } + goto out; /* success */ + +out_err: + pr_err("an error(%d) on the writable branch %pd(%s)\n", + err, h_root, au_sbtype(h_root->d_sb)); +out: + for (i = 0; i < AuBrWh_Last; i++) + dput(base[i].dentry); + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * whiteouts are all hard-linked usually. + * when its link count reaches a ceiling, we create a new whiteout base + * asynchronously. + */ + +struct reinit_br_wh { + struct super_block *sb; + struct au_branch *br; +}; + +static void reinit_br_wh(void *arg) +{ + int err; + aufs_bindex_t bindex; + struct path h_path; + struct reinit_br_wh *a = arg; + struct au_wbr *wbr; + struct inode *dir, *delegated; + struct dentry *h_root; + struct au_hinode *hdir; + + err = 0; + wbr = a->br->br_wbr; + /* big aufs lock */ + si_noflush_write_lock(a->sb); + if (!au_br_writable(a->br->br_perm)) + goto out; + bindex = au_br_index(a->sb, a->br->br_id); + if (unlikely(bindex < 0)) + goto out; + + di_read_lock_parent(a->sb->s_root, AuLock_IR); + dir = d_inode(a->sb->s_root); + hdir = au_hi(dir, bindex); + h_root = au_h_dptr(a->sb->s_root, bindex); + AuDebugOn(h_root != au_br_dentry(a->br)); + + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + wbr_wh_write_lock(wbr); + err = au_h_verify(wbr->wbr_whbase, au_opt_udba(a->sb), hdir->hi_inode, + h_root, a->br); + if (!err) { + h_path.dentry = wbr->wbr_whbase; + h_path.mnt = au_br_mnt(a->br); + delegated = NULL; + err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated, + /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } else { + pr_warn("%pd is moved, ignored\n", wbr->wbr_whbase); + err = 0; + } + dput(wbr->wbr_whbase); + wbr->wbr_whbase = NULL; + if (!err) + err = au_wh_init(a->br, a->sb); + wbr_wh_write_unlock(wbr); + au_hn_inode_unlock(hdir); + di_read_unlock(a->sb->s_root, AuLock_IR); + if (!err) + au_fhsm_wrote(a->sb, bindex, /*force*/0); + +out: + if (wbr) + atomic_dec(&wbr->wbr_wh_running); + au_lcnt_dec(&a->br->br_count); + si_write_unlock(a->sb); + au_nwt_done(&au_sbi(a->sb)->si_nowait); + au_kfree_rcu(a); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} + +static void kick_reinit_br_wh(struct super_block *sb, struct au_branch *br) +{ + int do_dec, wkq_err; + struct reinit_br_wh *arg; + + do_dec = 1; + if (atomic_inc_return(&br->br_wbr->wbr_wh_running) != 1) + goto out; + + /* ignore ENOMEM */ + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (arg) { + /* + * dec(wh_running), kfree(arg) and dec(br_count) + * in reinit function + */ + arg->sb = sb; + arg->br = br; + au_lcnt_inc(&br->br_count); + wkq_err = au_wkq_nowait(reinit_br_wh, arg, sb, /*flags*/0); + if (unlikely(wkq_err)) { + atomic_dec(&br->br_wbr->wbr_wh_running); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(arg); + } + do_dec = 0; + } + +out: + if (do_dec) + atomic_dec(&br->br_wbr->wbr_wh_running); +} + +/* ---------------------------------------------------------------------- */ + +/* + * create the whiteout @wh. + */ +static int link_or_create_wh(struct super_block *sb, aufs_bindex_t bindex, + struct dentry *wh) +{ + int err; + struct path h_path = { + .dentry = wh + }; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *h_parent; + struct inode *h_dir, *delegated; + + h_parent = wh->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + br = au_sbr(sb, bindex); + h_path.mnt = au_br_mnt(br); + wbr = br->br_wbr; + wbr_wh_read_lock(wbr); + if (wbr->wbr_whbase) { + delegated = NULL; + err = vfsub_link(wbr->wbr_whbase, h_dir, &h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + if (!err || err != -EMLINK) + goto out; + + /* link count full. re-initialize br_whbase. */ + kick_reinit_br_wh(sb, br); + } + + /* return this error in this context */ + err = vfsub_create(h_dir, &h_path, WH_MASK, /*want_excl*/true); + if (!err) + au_fhsm_wrote(sb, bindex, /*force*/0); + +out: + wbr_wh_read_unlock(wbr); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * create or remove the diropq. + */ +static struct dentry *do_diropq(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags) +{ + struct dentry *opq_dentry, *h_dentry; + struct super_block *sb; + struct au_branch *br; + int err; + + sb = dentry->d_sb; + br = au_sbr(sb, bindex); + h_dentry = au_h_dptr(dentry, bindex); + opq_dentry = vfsub_lkup_one(&diropq_name, h_dentry); + if (IS_ERR(opq_dentry)) + goto out; + + if (au_ftest_diropq(flags, CREATE)) { + err = link_or_create_wh(sb, bindex, opq_dentry); + if (!err) { + au_set_dbdiropq(dentry, bindex); + goto out; /* success */ + } + } else { + struct path tmp = { + .dentry = opq_dentry, + .mnt = au_br_mnt(br) + }; + err = do_unlink_wh(au_h_iptr(d_inode(dentry), bindex), &tmp); + if (!err) + au_set_dbdiropq(dentry, -1); + } + dput(opq_dentry); + opq_dentry = ERR_PTR(err); + +out: + return opq_dentry; +} + +struct do_diropq_args { + struct dentry **errp; + struct dentry *dentry; + aufs_bindex_t bindex; + unsigned int flags; +}; + +static void call_do_diropq(void *args) +{ + struct do_diropq_args *a = args; + *a->errp = do_diropq(a->dentry, a->bindex, a->flags); +} + +struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags) +{ + struct dentry *diropq, *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (!au_test_h_perm_sio(d_inode(h_dentry), MAY_EXEC | MAY_WRITE)) + diropq = do_diropq(dentry, bindex, flags); + else { + int wkq_err; + struct do_diropq_args args = { + .errp = &diropq, + .dentry = dentry, + .bindex = bindex, + .flags = flags + }; + + wkq_err = au_wkq_wait(call_do_diropq, &args); + if (unlikely(wkq_err)) + diropq = ERR_PTR(wkq_err); + } + + return diropq; +} + +/* ---------------------------------------------------------------------- */ + +/* + * lookup whiteout dentry. + * @h_parent: lower parent dentry which must exist and be locked + * @base_name: name of dentry which will be whiteouted + * returns dentry for whiteout. + */ +struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name, + struct au_branch *br) +{ + int err; + struct qstr wh_name; + struct dentry *wh_dentry; + + err = au_wh_name_alloc(&wh_name, base_name); + wh_dentry = ERR_PTR(err); + if (!err) { + wh_dentry = vfsub_lkup_one(&wh_name, h_parent); + au_kfree_try_rcu(wh_name.name); + } + return wh_dentry; +} + +/* + * link/create a whiteout for @dentry on @bindex. + */ +struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent) +{ + struct dentry *wh_dentry; + struct super_block *sb; + int err; + + sb = dentry->d_sb; + wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, au_sbr(sb, bindex)); + if (!IS_ERR(wh_dentry) && d_is_negative(wh_dentry)) { + err = link_or_create_wh(sb, bindex, wh_dentry); + if (!err) { + au_set_dbwh(dentry, bindex); + au_fhsm_wrote(sb, bindex, /*force*/0); + } else { + dput(wh_dentry); + wh_dentry = ERR_PTR(err); + } + } + + return wh_dentry; +} + +/* ---------------------------------------------------------------------- */ + +/* Delete all whiteouts in this directory on branch bindex. */ +static int del_wh_children(struct dentry *h_dentry, struct au_nhash *whlist, + aufs_bindex_t bindex, struct au_branch *br) +{ + int err; + unsigned long ul, n; + struct qstr wh_name; + char *p; + struct hlist_head *head; + struct au_vdir_wh *pos; + struct au_vdir_destr *str; + + err = -ENOMEM; + p = (void *)__get_free_page(GFP_NOFS); + wh_name.name = p; + if (unlikely(!wh_name.name)) + goto out; + + err = 0; + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + p += AUFS_WH_PFX_LEN; + n = whlist->nh_num; + head = whlist->nh_head; + for (ul = 0; !err && ul < n; ul++, head++) { + hlist_for_each_entry(pos, head, wh_hash) { + if (pos->wh_bindex != bindex) + continue; + + str = &pos->wh_str; + if (str->len + AUFS_WH_PFX_LEN <= PATH_MAX) { + memcpy(p, str->name, str->len); + wh_name.len = AUFS_WH_PFX_LEN + str->len; + err = unlink_wh_name(h_dentry, &wh_name, br); + if (!err) + continue; + break; + } + AuIOErr("whiteout name too long %.*s\n", + str->len, str->name); + err = -EIO; + break; + } + } + free_page((unsigned long)wh_name.name); + +out: + return err; +} + +struct del_wh_children_args { + int *errp; + struct dentry *h_dentry; + struct au_nhash *whlist; + aufs_bindex_t bindex; + struct au_branch *br; +}; + +static void call_del_wh_children(void *args) +{ + struct del_wh_children_args *a = args; + *a->errp = del_wh_children(a->h_dentry, a->whlist, a->bindex, a->br); +} + +/* ---------------------------------------------------------------------- */ + +struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp) +{ + struct au_whtmp_rmdir *whtmp; + int err; + unsigned int rdhash; + + SiMustAnyLock(sb); + + whtmp = kzalloc(sizeof(*whtmp), gfp); + if (unlikely(!whtmp)) { + whtmp = ERR_PTR(-ENOMEM); + goto out; + } + + /* no estimation for dir size */ + rdhash = au_sbi(sb)->si_rdhash; + if (!rdhash) + rdhash = AUFS_RDHASH_DEF; + err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp); + if (unlikely(err)) { + au_kfree_rcu(whtmp); + whtmp = ERR_PTR(err); + } + +out: + return whtmp; +} + +void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp) +{ + if (whtmp->br) + au_lcnt_dec(&whtmp->br->br_count); + dput(whtmp->wh_dentry); + iput(whtmp->dir); + au_nhash_wh_free(&whtmp->whlist); + au_kfree_rcu(whtmp); +} + +/* + * rmdir the whiteouted temporary named dir @h_dentry. + * @whlist: whiteouted children. + */ +int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_nhash *whlist) +{ + int err; + unsigned int h_nlink; + struct path h_tmp; + struct inode *wh_inode, *h_dir; + struct au_branch *br; + + h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */ + IMustLock(h_dir); + + br = au_sbr(dir->i_sb, bindex); + wh_inode = d_inode(wh_dentry); + inode_lock_nested(wh_inode, AuLsc_I_CHILD); + + /* + * someone else might change some whiteouts while we were sleeping. + * it means this whlist may have an obsoleted entry. + */ + if (!au_test_h_perm_sio(wh_inode, MAY_EXEC | MAY_WRITE)) + err = del_wh_children(wh_dentry, whlist, bindex, br); + else { + int wkq_err; + struct del_wh_children_args args = { + .errp = &err, + .h_dentry = wh_dentry, + .whlist = whlist, + .bindex = bindex, + .br = br + }; + + wkq_err = au_wkq_wait(call_del_wh_children, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + inode_unlock(wh_inode); + + if (!err) { + h_tmp.dentry = wh_dentry; + h_tmp.mnt = au_br_mnt(br); + h_nlink = h_dir->i_nlink; + err = vfsub_rmdir(h_dir, &h_tmp); + /* some fs doesn't change the parent nlink in some cases */ + h_nlink -= h_dir->i_nlink; + } + + if (!err) { + if (au_ibtop(dir) == bindex) { + /* todo: dir->i_mutex is necessary */ + au_cpup_attr_timesizes(dir); + if (h_nlink) + vfsub_drop_nlink(dir); + } + return 0; /* success */ + } + + pr_warn("failed removing %pd(%d), ignored\n", wh_dentry, err); + return err; +} + +static void call_rmdir_whtmp(void *args) +{ + int err; + aufs_bindex_t bindex; + struct au_whtmp_rmdir *a = args; + struct super_block *sb; + struct dentry *h_parent; + struct inode *h_dir; + struct au_hinode *hdir; + + /* rmdir by nfsd may cause deadlock with this i_mutex */ + /* inode_lock(a->dir); */ + err = -EROFS; + sb = a->dir->i_sb; + si_read_lock(sb, !AuLock_FLUSH); + if (!au_br_writable(a->br->br_perm)) + goto out; + bindex = au_br_index(sb, a->br->br_id); + if (unlikely(bindex < 0)) + goto out; + + err = -EIO; + ii_write_lock_parent(a->dir); + h_parent = dget_parent(a->wh_dentry); + h_dir = d_inode(h_parent); + hdir = au_hi(a->dir, bindex); + err = vfsub_mnt_want_write(au_br_mnt(a->br)); + if (unlikely(err)) + goto out_mnt; + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + err = au_h_verify(a->wh_dentry, au_opt_udba(sb), h_dir, h_parent, + a->br); + if (!err) + err = au_whtmp_rmdir(a->dir, bindex, a->wh_dentry, &a->whlist); + au_hn_inode_unlock(hdir); + vfsub_mnt_drop_write(au_br_mnt(a->br)); + +out_mnt: + dput(h_parent); + ii_write_unlock(a->dir); +out: + /* inode_unlock(a->dir); */ + au_whtmp_rmdir_free(a); + si_read_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} + +void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_whtmp_rmdir *args) +{ + int wkq_err; + struct super_block *sb; + + IMustLock(dir); + + /* all post-process will be done in do_rmdir_whtmp(). */ + sb = dir->i_sb; + args->dir = au_igrab(dir); + args->br = au_sbr(sb, bindex); + au_lcnt_inc(&args->br->br_count); + args->wh_dentry = dget(wh_dentry); + wkq_err = au_wkq_nowait(call_rmdir_whtmp, args, sb, /*flags*/0); + if (unlikely(wkq_err)) { + pr_warn("rmdir error %pd (%d), ignored\n", wh_dentry, wkq_err); + au_whtmp_rmdir_free(args); + } +} diff --git a/fs/aufs/whout.h b/fs/aufs/whout.h new file mode 100644 index 0000000000000..06f69b24e5c14 --- /dev/null +++ b/fs/aufs/whout.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * whiteout for logical deletion and opaque directory + */ + +#ifndef __AUFS_WHOUT_H__ +#define __AUFS_WHOUT_H__ + +#ifdef __KERNEL__ + +#include "dir.h" + +/* whout.c */ +int au_wh_name_alloc(struct qstr *wh, const struct qstr *name); +int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio); +int au_diropq_test(struct dentry *h_dentry); +struct au_branch; +struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br, + struct qstr *prefix); +int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br); +int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path, + struct dentry *dentry); +int au_wh_init(struct au_branch *br, struct super_block *sb); + +/* diropq flags */ +#define AuDiropq_CREATE 1 +#define au_ftest_diropq(flags, name) ((flags) & AuDiropq_##name) +#define au_fset_diropq(flags, name) \ + do { (flags) |= AuDiropq_##name; } while (0) +#define au_fclr_diropq(flags, name) \ + do { (flags) &= ~AuDiropq_##name; } while (0) + +struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags); +struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name, + struct au_branch *br); +struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent); + +/* real rmdir for the whiteout-ed dir */ +struct au_whtmp_rmdir { + struct inode *dir; + struct au_branch *br; + struct dentry *wh_dentry; + struct au_nhash whlist; +}; + +struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp); +void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp); +int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_nhash *whlist); +void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_whtmp_rmdir *args); + +/* ---------------------------------------------------------------------- */ + +static inline struct dentry *au_diropq_create(struct dentry *dentry, + aufs_bindex_t bindex) +{ + return au_diropq_sio(dentry, bindex, AuDiropq_CREATE); +} + +static inline int au_diropq_remove(struct dentry *dentry, aufs_bindex_t bindex) +{ + return PTR_ERR(au_diropq_sio(dentry, bindex, !AuDiropq_CREATE)); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_WHOUT_H__ */ diff --git a/fs/aufs/wkq.c b/fs/aufs/wkq.c new file mode 100644 index 0000000000000..b0dd8df4e4147 --- /dev/null +++ b/fs/aufs/wkq.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * workqueue for asynchronous/super-io operations + * todo: try new credential scheme + */ + +#include +#include "aufs.h" + +/* internal workqueue named AUFS_WKQ_NAME */ + +static struct workqueue_struct *au_wkq; + +struct au_wkinfo { + struct work_struct wk; + struct kobject *kobj; + + unsigned int flags; /* see wkq.h */ + + au_wkq_func_t func; + void *args; + +#ifdef CONFIG_LOCKDEP + int dont_check; + struct held_lock **hlock; +#endif + + struct completion *comp; +}; + +/* ---------------------------------------------------------------------- */ +/* + * Aufs passes some operations to the workqueue such as the internal copyup. + * This scheme looks rather unnatural for LOCKDEP debugging feature, since the + * job run by workqueue depends upon the locks acquired in the other task. + * Delegating a small operation to the workqueue, aufs passes its lockdep + * information too. And the job in the workqueue restores the info in order to + * pretend as if it acquired those locks. This is just to make LOCKDEP work + * correctly and expectedly. + */ + +#ifndef CONFIG_LOCKDEP +AuStubInt0(au_wkq_lockdep_alloc, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_free, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_pre, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_post, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_init, struct au_wkinfo *wkinfo); +#else +static void au_wkq_lockdep_init(struct au_wkinfo *wkinfo) +{ + wkinfo->hlock = NULL; + wkinfo->dont_check = 0; +} + +/* + * 1: matched + * 0: unmatched + */ +static int au_wkq_lockdep_test(struct lock_class_key *key, const char *name) +{ + static DEFINE_SPINLOCK(spin); + static struct { + char *name; + struct lock_class_key *key; + } a[] = { + { .name = "&sbinfo->si_rwsem" }, + { .name = "&finfo->fi_rwsem" }, + { .name = "&dinfo->di_rwsem" }, + { .name = "&iinfo->ii_rwsem" } + }; + static int set; + int i; + + /* lockless read from 'set.' see below */ + if (set == ARRAY_SIZE(a)) { + for (i = 0; i < ARRAY_SIZE(a); i++) + if (a[i].key == key) + goto match; + goto unmatch; + } + + spin_lock(&spin); + if (set) + for (i = 0; i < ARRAY_SIZE(a); i++) + if (a[i].key == key) { + spin_unlock(&spin); + goto match; + } + for (i = 0; i < ARRAY_SIZE(a); i++) { + if (a[i].key) { + if (unlikely(a[i].key == key)) { /* rare but possible */ + spin_unlock(&spin); + goto match; + } else + continue; + } + if (strstr(a[i].name, name)) { + /* + * the order of these three lines is important for the + * lockless read above. + */ + a[i].key = key; + spin_unlock(&spin); + set++; + /* AuDbg("%d, %s\n", set, name); */ + goto match; + } + } + spin_unlock(&spin); + goto unmatch; + +match: + return 1; +unmatch: + return 0; +} + +static int au_wkq_lockdep_alloc(struct au_wkinfo *wkinfo) +{ + int err, n; + struct task_struct *curr; + struct held_lock **hl, *held_locks, *p; + + err = 0; + curr = current; + wkinfo->dont_check = lockdep_recursing(curr); + if (wkinfo->dont_check) + goto out; + n = curr->lockdep_depth; + if (!n) + goto out; + + err = -ENOMEM; + wkinfo->hlock = kmalloc_array(n + 1, sizeof(*wkinfo->hlock), GFP_NOFS); + if (unlikely(!wkinfo->hlock)) + goto out; + + err = 0; +#if 0 + if (0 && au_debug_test()) /* left for debugging */ + lockdep_print_held_locks(curr); +#endif + held_locks = curr->held_locks; + hl = wkinfo->hlock; + while (n--) { + p = held_locks++; + if (au_wkq_lockdep_test(p->instance->key, p->instance->name)) + *hl++ = p; + } + *hl = NULL; + +out: + return err; +} + +static void au_wkq_lockdep_free(struct au_wkinfo *wkinfo) +{ + au_kfree_try_rcu(wkinfo->hlock); +} + +static void au_wkq_lockdep_pre(struct au_wkinfo *wkinfo) +{ + struct held_lock *p, **hl = wkinfo->hlock; + int subclass; + + if (wkinfo->dont_check) + lockdep_off(); + if (!hl) + return; + while ((p = *hl++)) { /* assignment */ + subclass = lockdep_hlock_class(p)->subclass; + /* AuDbg("%s, %d\n", p->instance->name, subclass); */ + if (p->read) + rwsem_acquire_read(p->instance, subclass, 0, + /*p->acquire_ip*/_RET_IP_); + else + rwsem_acquire(p->instance, subclass, 0, + /*p->acquire_ip*/_RET_IP_); + } +} + +static void au_wkq_lockdep_post(struct au_wkinfo *wkinfo) +{ + struct held_lock *p, **hl = wkinfo->hlock; + + if (wkinfo->dont_check) + lockdep_on(); + if (!hl) + return; + while ((p = *hl++)) /* assignment */ + rwsem_release(p->instance, 0, /*p->acquire_ip*/_RET_IP_); +} +#endif + +static void wkq_func(struct work_struct *wk) +{ + struct au_wkinfo *wkinfo = container_of(wk, struct au_wkinfo, wk); + + AuDebugOn(!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)); + AuDebugOn(rlimit(RLIMIT_FSIZE) != RLIM_INFINITY); + + au_wkq_lockdep_pre(wkinfo); + wkinfo->func(wkinfo->args); + au_wkq_lockdep_post(wkinfo); + if (au_ftest_wkq(wkinfo->flags, WAIT)) + complete(wkinfo->comp); + else { + kobject_put(wkinfo->kobj); + module_put(THIS_MODULE); /* todo: ?? */ + au_kfree_rcu(wkinfo); + } +} + +/* + * Since struct completion is large, try allocating it dynamically. + */ +#if 1 /* defined(CONFIG_4KSTACKS) || defined(AuTest4KSTACKS) */ +#define AuWkqCompDeclare(name) struct completion *comp = NULL + +static int au_wkq_comp_alloc(struct au_wkinfo *wkinfo, struct completion **comp) +{ + *comp = kmalloc(sizeof(**comp), GFP_NOFS); + if (*comp) { + init_completion(*comp); + wkinfo->comp = *comp; + return 0; + } + return -ENOMEM; +} + +static void au_wkq_comp_free(struct completion *comp) +{ + au_kfree_rcu(comp); +} + +#else + +/* no braces */ +#define AuWkqCompDeclare(name) \ + DECLARE_COMPLETION_ONSTACK(_ ## name); \ + struct completion *comp = &_ ## name + +static int au_wkq_comp_alloc(struct au_wkinfo *wkinfo, struct completion **comp) +{ + wkinfo->comp = *comp; + return 0; +} + +static void au_wkq_comp_free(struct completion *comp __maybe_unused) +{ + /* empty */ +} +#endif /* 4KSTACKS */ + +static void au_wkq_run(struct au_wkinfo *wkinfo) +{ + if (au_ftest_wkq(wkinfo->flags, NEST)) { + if (au_wkq_test()) { + AuWarn1("wkq from wkq, unless silly-rename on NFS," + " due to a dead dir by UDBA," + " or async xino write?\n"); + AuDebugOn(au_ftest_wkq(wkinfo->flags, WAIT)); + } + } else + au_dbg_verify_kthread(); + + if (au_ftest_wkq(wkinfo->flags, WAIT)) { + INIT_WORK_ONSTACK(&wkinfo->wk, wkq_func); + queue_work(au_wkq, &wkinfo->wk); + } else { + INIT_WORK(&wkinfo->wk, wkq_func); + schedule_work(&wkinfo->wk); + } +} + +/* + * Be careful. It is easy to make deadlock happen. + * processA: lock, wkq and wait + * processB: wkq and wait, lock in wkq + * --> deadlock + */ +int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args) +{ + int err; + AuWkqCompDeclare(comp); + struct au_wkinfo wkinfo = { + .flags = flags, + .func = func, + .args = args + }; + + err = au_wkq_comp_alloc(&wkinfo, &comp); + if (unlikely(err)) + goto out; + err = au_wkq_lockdep_alloc(&wkinfo); + if (unlikely(err)) + goto out_comp; + if (!err) { + au_wkq_run(&wkinfo); + /* no timeout, no interrupt */ + wait_for_completion(wkinfo.comp); + } + au_wkq_lockdep_free(&wkinfo); + +out_comp: + au_wkq_comp_free(comp); +out: + destroy_work_on_stack(&wkinfo.wk); + return err; +} + +/* + * Note: dget/dput() in func for aufs dentries are not supported. It will be a + * problem in a concurrent umounting. + */ +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags) +{ + int err; + struct au_wkinfo *wkinfo; + + atomic_inc(&au_sbi(sb)->si_nowait.nw_len); + + /* + * wkq_func() must free this wkinfo. + * it highly depends upon the implementation of workqueue. + */ + err = 0; + wkinfo = kmalloc(sizeof(*wkinfo), GFP_NOFS); + if (wkinfo) { + wkinfo->kobj = &au_sbi(sb)->si_kobj; + wkinfo->flags = flags & ~AuWkq_WAIT; + wkinfo->func = func; + wkinfo->args = args; + wkinfo->comp = NULL; + au_wkq_lockdep_init(wkinfo); + kobject_get(wkinfo->kobj); + __module_get(THIS_MODULE); /* todo: ?? */ + + au_wkq_run(wkinfo); + } else { + err = -ENOMEM; + au_nwt_done(&au_sbi(sb)->si_nowait); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_nwt_init(struct au_nowait_tasks *nwt) +{ + atomic_set(&nwt->nw_len, 0); + /* smp_mb(); */ /* atomic_set */ + init_waitqueue_head(&nwt->nw_wq); +} + +void au_wkq_fin(void) +{ + destroy_workqueue(au_wkq); +} + +int __init au_wkq_init(void) +{ + int err; + + err = 0; + au_wkq = alloc_workqueue(AUFS_WKQ_NAME, 0, WQ_DFL_ACTIVE); + if (IS_ERR(au_wkq)) + err = PTR_ERR(au_wkq); + else if (!au_wkq) + err = -ENOMEM; + + return err; +} diff --git a/fs/aufs/wkq.h b/fs/aufs/wkq.h new file mode 100644 index 0000000000000..235370d91693d --- /dev/null +++ b/fs/aufs/wkq.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * workqueue for asynchronous/super-io operations + * todo: try new credentials management scheme + */ + +#ifndef __AUFS_WKQ_H__ +#define __AUFS_WKQ_H__ + +#ifdef __KERNEL__ + +#include + +struct super_block; + +/* ---------------------------------------------------------------------- */ + +/* + * in the next operation, wait for the 'nowait' tasks in system-wide workqueue + */ +struct au_nowait_tasks { + atomic_t nw_len; + wait_queue_head_t nw_wq; +}; + +/* ---------------------------------------------------------------------- */ + +typedef void (*au_wkq_func_t)(void *args); + +/* wkq flags */ +#define AuWkq_WAIT 1 +#define AuWkq_NEST (1 << 1) +#define au_ftest_wkq(flags, name) ((flags) & AuWkq_##name) +#define au_fset_wkq(flags, name) \ + do { (flags) |= AuWkq_##name; } while (0) +#define au_fclr_wkq(flags, name) \ + do { (flags) &= ~AuWkq_##name; } while (0) + +/* wkq.c */ +int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args); +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags); +void au_nwt_init(struct au_nowait_tasks *nwt); +int __init au_wkq_init(void); +void au_wkq_fin(void); + +/* ---------------------------------------------------------------------- */ + +static inline int au_wkq_test(void) +{ + return current->flags & PF_WQ_WORKER; +} + +static inline int au_wkq_wait(au_wkq_func_t func, void *args) +{ + return au_wkq_do_wait(AuWkq_WAIT, func, args); +} + +static inline void au_nwt_done(struct au_nowait_tasks *nwt) +{ + if (atomic_dec_and_test(&nwt->nw_len)) + wake_up_all(&nwt->nw_wq); +} + +static inline int au_nwt_flush(struct au_nowait_tasks *nwt) +{ + wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len)); + return 0; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_WKQ_H__ */ diff --git a/fs/aufs/xattr.c b/fs/aufs/xattr.c new file mode 100644 index 0000000000000..4b25f2f14d4fc --- /dev/null +++ b/fs/aufs/xattr.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * handling xattr functions + */ + +#include +#include +#include +#include "aufs.h" + +static int au_xattr_ignore(int err, char *name, unsigned int ignore_flags) +{ + if (!ignore_flags) + goto out; + switch (err) { + case -ENOMEM: + case -EDQUOT: + goto out; + } + + if ((ignore_flags & AuBrAttr_ICEX) == AuBrAttr_ICEX) { + err = 0; + goto out; + } + +#define cmp(brattr, prefix) do { \ + if (!strncmp(name, XATTR_##prefix##_PREFIX, \ + XATTR_##prefix##_PREFIX_LEN)) { \ + if (ignore_flags & AuBrAttr_ICEX_##brattr) \ + err = 0; \ + goto out; \ + } \ + } while (0) + + cmp(SEC, SECURITY); + cmp(SYS, SYSTEM); + cmp(TR, TRUSTED); + cmp(USR, USER); +#undef cmp + + if (ignore_flags & AuBrAttr_ICEX_OTH) + err = 0; + +out: + return err; +} + +static const int au_xattr_out_of_list = AuBrAttr_ICEX_OTH << 1; + +static int au_do_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, + char *name, char **buf, unsigned int ignore_flags, + unsigned int verbose) +{ + int err; + ssize_t ssz; + struct inode *h_idst; + + ssz = vfs_getxattr_alloc(h_src, name, buf, 0, GFP_NOFS); + err = ssz; + if (unlikely(err <= 0)) { + if (err == -ENODATA + || (err == -EOPNOTSUPP + && ((ignore_flags & au_xattr_out_of_list) + || (au_test_nfs_noacl(d_inode(h_src)) + && (!strcmp(name, XATTR_NAME_POSIX_ACL_ACCESS) + || !strcmp(name, + XATTR_NAME_POSIX_ACL_DEFAULT)))) + )) + err = 0; + if (err && (verbose || au_debug_test())) + pr_err("%s, err %d\n", name, err); + goto out; + } + + /* unlock it temporary */ + h_idst = d_inode(h_dst); + inode_unlock(h_idst); + err = vfsub_setxattr(h_dst, name, *buf, ssz, /*flags*/0); + inode_lock_nested(h_idst, AuLsc_I_CHILD2); + if (unlikely(err)) { + if (verbose || au_debug_test()) + pr_err("%s, err %d\n", name, err); + err = au_xattr_ignore(err, name, ignore_flags); + } + +out: + return err; +} + +int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, + unsigned int verbose) +{ + int err, unlocked, acl_access, acl_default; + ssize_t ssz; + struct inode *h_isrc, *h_idst; + char *value, *p, *o, *e; + + /* try stopping to update the source inode while we are referencing */ + /* there should not be the parent-child relationship between them */ + h_isrc = d_inode(h_src); + h_idst = d_inode(h_dst); + inode_unlock(h_idst); + vfsub_inode_lock_shared_nested(h_isrc, AuLsc_I_CHILD); + inode_lock_nested(h_idst, AuLsc_I_CHILD2); + unlocked = 0; + + /* some filesystems don't list POSIX ACL, for example tmpfs */ + ssz = vfs_listxattr(h_src, NULL, 0); + err = ssz; + if (unlikely(err < 0)) { + AuTraceErr(err); + if (err == -ENODATA + || err == -EOPNOTSUPP) + err = 0; /* ignore */ + goto out; + } + + err = 0; + p = NULL; + o = NULL; + if (ssz) { + err = -ENOMEM; + p = kmalloc(ssz, GFP_NOFS); + o = p; + if (unlikely(!p)) + goto out; + err = vfs_listxattr(h_src, p, ssz); + } + inode_unlock_shared(h_isrc); + unlocked = 1; + AuDbg("err %d, ssz %zd\n", err, ssz); + if (unlikely(err < 0)) + goto out_free; + + err = 0; + e = p + ssz; + value = NULL; + acl_access = 0; + acl_default = 0; + while (!err && p < e) { + acl_access |= !strncmp(p, XATTR_NAME_POSIX_ACL_ACCESS, + sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1); + acl_default |= !strncmp(p, XATTR_NAME_POSIX_ACL_DEFAULT, + sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) + - 1); + err = au_do_cpup_xattr(h_dst, h_src, p, &value, ignore_flags, + verbose); + p += strlen(p) + 1; + } + AuTraceErr(err); + ignore_flags |= au_xattr_out_of_list; + if (!err && !acl_access) { + err = au_do_cpup_xattr(h_dst, h_src, + XATTR_NAME_POSIX_ACL_ACCESS, &value, + ignore_flags, verbose); + AuTraceErr(err); + } + if (!err && !acl_default) { + err = au_do_cpup_xattr(h_dst, h_src, + XATTR_NAME_POSIX_ACL_DEFAULT, &value, + ignore_flags, verbose); + AuTraceErr(err); + } + + au_kfree_try_rcu(value); + +out_free: + au_kfree_try_rcu(o); +out: + if (!unlocked) + inode_unlock_shared(h_isrc); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_smack_reentering(struct super_block *sb) +{ +#if IS_ENABLED(CONFIG_SECURITY_SMACK) + /* + * as a part of lookup, smack_d_instantiate() is called, and it calls + * i_op->getxattr(). ouch. + */ + return si_pid_test(sb); +#else + return 0; +#endif +} + +enum { + AU_XATTR_LIST, + AU_XATTR_GET +}; + +struct au_lgxattr { + int type; + union { + struct { + char *list; + size_t size; + } list; + struct { + const char *name; + void *value; + size_t size; + } get; + } u; +}; + +static ssize_t au_lgxattr(struct dentry *dentry, struct au_lgxattr *arg) +{ + ssize_t err; + int reenter; + struct path h_path; + struct super_block *sb; + + sb = dentry->d_sb; + reenter = au_smack_reentering(sb); + if (!reenter) { + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + } + err = au_h_path_getattr(dentry, /*force*/1, &h_path, reenter); + if (unlikely(err)) + goto out_si; + if (unlikely(!h_path.dentry)) + /* illegally overlapped or something */ + goto out_di; /* pretending success */ + + /* always topmost entry only */ + switch (arg->type) { + case AU_XATTR_LIST: + err = vfs_listxattr(h_path.dentry, + arg->u.list.list, arg->u.list.size); + break; + case AU_XATTR_GET: + AuDebugOn(d_is_negative(h_path.dentry)); + err = vfs_getxattr(h_path.dentry, + arg->u.get.name, arg->u.get.value, + arg->u.get.size); + break; + } + +out_di: + if (!reenter) + di_read_unlock(dentry, AuLock_IR); +out_si: + if (!reenter) + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size) +{ + struct au_lgxattr arg = { + .type = AU_XATTR_LIST, + .u.list = { + .list = list, + .size = size + }, + }; + + return au_lgxattr(dentry, &arg); +} + +static ssize_t au_getxattr(struct dentry *dentry, + struct inode *inode __maybe_unused, + const char *name, void *value, size_t size) +{ + struct au_lgxattr arg = { + .type = AU_XATTR_GET, + .u.get = { + .name = name, + .value = value, + .size = size + }, + }; + + return au_lgxattr(dentry, &arg); +} + +static int au_setxattr(struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + struct au_sxattr arg = { + .type = AU_XATTR_SET, + .u.set = { + .name = name, + .value = value, + .size = size, + .flags = flags + }, + }; + + return au_sxattr(dentry, inode, &arg); +} + +/* ---------------------------------------------------------------------- */ + +static int au_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return au_getxattr(dentry, inode, name, buffer, size); +} + +static int au_xattr_set(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + return au_setxattr(dentry, inode, name, value, size, flags); +} + +static const struct xattr_handler au_xattr_handler = { + .name = "", + .prefix = "", + .get = au_xattr_get, + .set = au_xattr_set +}; + +static const struct xattr_handler *au_xattr_handlers[] = { +#ifdef CONFIG_FS_POSIX_ACL + &posix_acl_access_xattr_handler, + &posix_acl_default_xattr_handler, +#endif + &au_xattr_handler, /* must be last */ + NULL +}; + +void au_xattr_init(struct super_block *sb) +{ + sb->s_xattr = au_xattr_handlers; +} diff --git a/fs/aufs/xino.c b/fs/aufs/xino.c new file mode 100644 index 0000000000000..35ea33469d363 --- /dev/null +++ b/fs/aufs/xino.c @@ -0,0 +1,1965 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * external inode number translation table and bitmap + * + * things to consider + * - the lifetime + * + au_xino object + * + XINO files (xino, xib, xigen) + * + dynamic debugfs entries (xiN) + * + static debugfs entries (xib, xigen) + * + static sysfs entry (xi_path) + * - several entry points to handle them. + * + mount(2) without xino option (default) + * + mount(2) with xino option + * + mount(2) with noxino option + * + umount(2) + * + remount with add/del branches + * + remount with xino/noxino options + */ + +#include +#include +#include "aufs.h" + +static aufs_bindex_t sbr_find_shared(struct super_block *sb, aufs_bindex_t btop, + aufs_bindex_t bbot, + struct super_block *h_sb) +{ + /* todo: try binary-search if the branches are many */ + for (; btop <= bbot; btop++) + if (h_sb == au_sbr_sb(sb, btop)) + return btop; + return -1; +} + +/* + * find another branch who is on the same filesystem of the specified + * branch{@btgt}. search until @bbot. + */ +static aufs_bindex_t is_sb_shared(struct super_block *sb, aufs_bindex_t btgt, + aufs_bindex_t bbot) +{ + aufs_bindex_t bindex; + struct super_block *tgt_sb; + + tgt_sb = au_sbr_sb(sb, btgt); + bindex = sbr_find_shared(sb, /*btop*/0, btgt - 1, tgt_sb); + if (bindex < 0) + bindex = sbr_find_shared(sb, btgt + 1, bbot, tgt_sb); + + return bindex; +} + +/* ---------------------------------------------------------------------- */ + +/* + * stop unnecessary notify events at creating xino files + */ + +aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry) +{ + aufs_bindex_t bfound, bindex, bbot; + struct dentry *parent; + struct au_branch *br; + + bfound = -1; + parent = dentry->d_parent; /* safe d_parent access */ + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_dentry(br) == parent) { + bfound = bindex; + break; + } + } + + AuDbg("bfound b%d\n", bfound); + return bfound; +} + +struct au_xino_lock_dir { + struct au_hinode *hdir; + struct dentry *parent; + struct inode *dir; +}; + +static struct dentry *au_dget_parent_lock(struct dentry *dentry, + unsigned int lsc) +{ + struct dentry *parent; + struct inode *dir; + + parent = dget_parent(dentry); + dir = d_inode(parent); + inode_lock_nested(dir, lsc); +#if 0 /* it should not happen */ + spin_lock(&dentry->d_lock); + if (unlikely(dentry->d_parent != parent)) { + spin_unlock(&dentry->d_lock); + inode_unlock(dir); + dput(parent); + parent = NULL; + goto out; + } + spin_unlock(&dentry->d_lock); + +out: +#endif + return parent; +} + +static void au_xino_lock_dir(struct super_block *sb, struct path *xipath, + struct au_xino_lock_dir *ldir) +{ + aufs_bindex_t bindex; + + ldir->hdir = NULL; + bindex = au_xi_root(sb, xipath->dentry); + if (bindex >= 0) { + /* rw branch root */ + ldir->hdir = au_hi(d_inode(sb->s_root), bindex); + au_hn_inode_lock_nested(ldir->hdir, AuLsc_I_PARENT); + } else { + /* other */ + ldir->parent = au_dget_parent_lock(xipath->dentry, + AuLsc_I_PARENT); + ldir->dir = d_inode(ldir->parent); + } +} + +static void au_xino_unlock_dir(struct au_xino_lock_dir *ldir) +{ + if (ldir->hdir) + au_hn_inode_unlock(ldir->hdir); + else { + inode_unlock(ldir->dir); + dput(ldir->parent); + } +} + +/* ---------------------------------------------------------------------- */ + +/* + * create and set a new xino file + */ +struct file *au_xino_create(struct super_block *sb, char *fpath, int silent, + int wbrtop) +{ + struct file *file; + struct dentry *h_parent, *d; + struct inode *h_dir, *inode; + int err; + static DEFINE_MUTEX(mtx); + + /* + * at mount-time, and the xino file is the default path, + * hnotify is disabled so we have no notify events to ignore. + * when a user specified the xino, we cannot get au_hdir to be ignored. + */ + if (!wbrtop) + mutex_lock(&mtx); + file = vfsub_filp_open(fpath, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE + /* | __FMODE_NONOTIFY */, + 0666); + if (IS_ERR(file)) { + if (!wbrtop) + mutex_unlock(&mtx); + if (!silent) + pr_err("open %s(%ld)\n", fpath, PTR_ERR(file)); + return file; + } + + /* keep file count */ + err = 0; + d = file->f_path.dentry; + h_parent = au_dget_parent_lock(d, AuLsc_I_PARENT); + if (!wbrtop) + mutex_unlock(&mtx); + /* mnt_want_write() is unnecessary here */ + h_dir = d_inode(h_parent); + inode = file_inode(file); + /* no delegation since it is just created */ + if (inode->i_nlink) + err = vfsub_unlink(h_dir, &file->f_path, /*delegated*/NULL, + /*force*/0); + inode_unlock(h_dir); + dput(h_parent); + if (unlikely(err)) { + if (!silent) + pr_err("unlink %s(%d)\n", fpath, err); + goto out; + } + + err = -EINVAL; + if (unlikely(sb == d->d_sb)) { + if (!silent) + pr_err("%s must be outside\n", fpath); + goto out; + } + if (unlikely(au_test_fs_bad_xino(d->d_sb))) { + if (!silent) + pr_err("xino doesn't support %s(%s)\n", + fpath, au_sbtype(d->d_sb)); + goto out; + } + return file; /* success */ + +out: + fput(file); + file = ERR_PTR(err); + return file; +} + +/* + * create a new xinofile at the same place/path as @base. + */ +struct file *au_xino_create2(struct super_block *sb, struct path *base, + struct file *copy_src) +{ + struct file *file; + struct dentry *dentry, *parent; + struct inode *dir, *delegated; + struct qstr *name; + struct path path; + int err, do_unlock; + struct au_xino_lock_dir ldir; + + do_unlock = 1; + au_xino_lock_dir(sb, base, &ldir); + dentry = base->dentry; + parent = dentry->d_parent; /* dir inode is locked */ + dir = d_inode(parent); + IMustLock(dir); + + name = &dentry->d_name; + path.dentry = vfsub_lookup_one_len(name->name, parent, name->len); + if (IS_ERR(path.dentry)) { + file = (void *)path.dentry; + pr_err("%pd lookup err %ld\n", dentry, PTR_ERR(path.dentry)); + goto out; + } + + /* no need to mnt_want_write() since we call dentry_open() later */ + err = vfs_create(dir, path.dentry, 0666, NULL); + if (unlikely(err)) { + file = ERR_PTR(err); + pr_err("%pd create err %d\n", dentry, err); + goto out_dput; + } + + path.mnt = base->mnt; + file = vfsub_dentry_open(&path, + O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE + /* | __FMODE_NONOTIFY */); + if (IS_ERR(file)) { + pr_err("%pd open err %ld\n", dentry, PTR_ERR(file)); + goto out_dput; + } + + delegated = NULL; + err = vfsub_unlink(dir, &file->f_path, &delegated, /*force*/0); + au_xino_unlock_dir(&ldir); + do_unlock = 0; + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) { + pr_err("%pd unlink err %d\n", dentry, err); + goto out_fput; + } + + if (copy_src) { + /* no one can touch copy_src xino */ + err = au_copy_file(file, copy_src, vfsub_f_size_read(copy_src)); + if (unlikely(err)) { + pr_err("%pd copy err %d\n", dentry, err); + goto out_fput; + } + } + goto out_dput; /* success */ + +out_fput: + fput(file); + file = ERR_PTR(err); +out_dput: + dput(path.dentry); +out: + if (do_unlock) + au_xino_unlock_dir(&ldir); + return file; +} + +struct file *au_xino_file1(struct au_xino *xi) +{ + struct file *file; + unsigned int u, nfile; + + file = NULL; + nfile = xi->xi_nfile; + for (u = 0; u < nfile; u++) { + file = xi->xi_file[u]; + if (file) + break; + } + + return file; +} + +static int au_xino_file_set(struct au_xino *xi, int idx, struct file *file) +{ + int err; + struct file *f; + void *p; + + if (file) + get_file(file); + + err = 0; + f = NULL; + if (idx < xi->xi_nfile) { + f = xi->xi_file[idx]; + if (f) + fput(f); + } else { + p = au_kzrealloc(xi->xi_file, + sizeof(*xi->xi_file) * xi->xi_nfile, + sizeof(*xi->xi_file) * (idx + 1), + GFP_NOFS, /*may_shrink*/0); + if (p) { + MtxMustLock(&xi->xi_mtx); + xi->xi_file = p; + xi->xi_nfile = idx + 1; + } else { + err = -ENOMEM; + if (file) + fput(file); + goto out; + } + } + xi->xi_file[idx] = file; + +out: + return err; +} + +/* + * if @xinew->xi is not set, then create new xigen file. + */ +struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew) +{ + struct file *file; + int err; + + SiMustAnyLock(sb); + + file = au_xino_create2(sb, xinew->base, xinew->copy_src); + if (IS_ERR(file)) { + err = PTR_ERR(file); + pr_err("%s[%d], err %d\n", + xinew->xi ? "xino" : "xigen", + xinew->idx, err); + goto out; + } + + if (xinew->xi) + err = au_xino_file_set(xinew->xi, xinew->idx, file); + else { + BUG(); + /* todo: make xigen file an array */ + /* err = au_xigen_file_set(sb, xinew->idx, file); */ + } + fput(file); + if (unlikely(err)) + file = ERR_PTR(err); + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* + * truncate xino files + */ +static int au_xino_do_trunc(struct super_block *sb, aufs_bindex_t bindex, + int idx, struct kstatfs *st) +{ + int err; + blkcnt_t blocks; + struct file *file, *new_xino; + struct au_xi_new xinew = { + .idx = idx + }; + + err = 0; + xinew.xi = au_sbr(sb, bindex)->br_xino; + file = au_xino_file(xinew.xi, idx); + if (!file) + goto out; + + xinew.base = &file->f_path; + err = vfs_statfs(xinew.base, st); + if (unlikely(err)) { + AuErr1("statfs err %d, ignored\n", err); + err = 0; + goto out; + } + + blocks = file_inode(file)->i_blocks; + pr_info("begin truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n", + bindex, idx, (u64)blocks, st->f_bfree, st->f_blocks); + + xinew.copy_src = file; + new_xino = au_xi_new(sb, &xinew); + if (IS_ERR(new_xino)) { + err = PTR_ERR(new_xino); + pr_err("xino(b%d-%d), err %d, ignored\n", bindex, idx, err); + goto out; + } + + err = vfs_statfs(&new_xino->f_path, st); + if (!err) + pr_info("end truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n", + bindex, idx, (u64)file_inode(new_xino)->i_blocks, + st->f_bfree, st->f_blocks); + else { + AuErr1("statfs err %d, ignored\n", err); + err = 0; + } + +out: + return err; +} + +int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin) +{ + int err, i; + unsigned long jiffy; + aufs_bindex_t bbot; + struct kstatfs *st; + struct au_branch *br; + struct au_xino *xi; + + err = -ENOMEM; + st = kmalloc(sizeof(*st), GFP_NOFS); + if (unlikely(!st)) + goto out; + + err = -EINVAL; + bbot = au_sbbot(sb); + if (unlikely(bindex < 0 || bbot < bindex)) + goto out_st; + + err = 0; + jiffy = jiffies; + br = au_sbr(sb, bindex); + xi = br->br_xino; + for (i = idx_begin; !err && i < xi->xi_nfile; i++) + err = au_xino_do_trunc(sb, bindex, i, st); + if (!err) + au_sbi(sb)->si_xino_jiffy = jiffy; + +out_st: + au_kfree_rcu(st); +out: + return err; +} + +struct xino_do_trunc_args { + struct super_block *sb; + struct au_branch *br; + int idx; +}; + +static void xino_do_trunc(void *_args) +{ + struct xino_do_trunc_args *args = _args; + struct super_block *sb; + struct au_branch *br; + struct inode *dir; + int err, idx; + aufs_bindex_t bindex; + + err = 0; + sb = args->sb; + dir = d_inode(sb->s_root); + br = args->br; + idx = args->idx; + + si_noflush_write_lock(sb); + ii_read_lock_parent(dir); + bindex = au_br_index(sb, br->br_id); + err = au_xino_trunc(sb, bindex, idx); + ii_read_unlock(dir); + if (unlikely(err)) + pr_warn("err b%d, (%d)\n", bindex, err); + atomic_dec(&br->br_xino->xi_truncating); + au_lcnt_dec(&br->br_count); + si_write_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); + au_kfree_rcu(args); +} + +/* + * returns the index in the xi_file array whose corresponding file is necessary + * to truncate, or -1 which means no need to truncate. + */ +static int xino_trunc_test(struct super_block *sb, struct au_branch *br) +{ + int err; + unsigned int u; + struct kstatfs st; + struct au_sbinfo *sbinfo; + struct au_xino *xi; + struct file *file; + + /* todo: si_xino_expire and the ratio should be customizable */ + sbinfo = au_sbi(sb); + if (time_before(jiffies, + sbinfo->si_xino_jiffy + sbinfo->si_xino_expire)) + return -1; + + /* truncation border */ + xi = br->br_xino; + for (u = 0; u < xi->xi_nfile; u++) { + file = au_xino_file(xi, u); + if (!file) + continue; + + err = vfs_statfs(&file->f_path, &st); + if (unlikely(err)) { + AuErr1("statfs err %d, ignored\n", err); + return -1; + } + if (div64_u64(st.f_bfree * 100, st.f_blocks) + >= AUFS_XINO_DEF_TRUNC) + return u; + } + + return -1; +} + +static void xino_try_trunc(struct super_block *sb, struct au_branch *br) +{ + int idx; + struct xino_do_trunc_args *args; + int wkq_err; + + idx = xino_trunc_test(sb, br); + if (idx < 0) + return; + + if (atomic_inc_return(&br->br_xino->xi_truncating) > 1) + goto out; + + /* lock and kfree() will be called in trunc_xino() */ + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); + goto out; + } + + au_lcnt_inc(&br->br_count); + args->sb = sb; + args->br = br; + args->idx = idx; + wkq_err = au_wkq_nowait(xino_do_trunc, args, sb, /*flags*/0); + if (!wkq_err) + return; /* success */ + + pr_err("wkq %d\n", wkq_err); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(args); + +out: + atomic_dec(&br->br_xino->xi_truncating); +} + +/* ---------------------------------------------------------------------- */ + +struct au_xi_calc { + int idx; + loff_t pos; +}; + +static void au_xi_calc(struct super_block *sb, ino_t h_ino, + struct au_xi_calc *calc) +{ + loff_t maxent; + + maxent = au_xi_maxent(sb); + calc->idx = div64_u64_rem(h_ino, maxent, &calc->pos); + calc->pos *= sizeof(ino_t); +} + +static int au_xino_do_new_async(struct super_block *sb, struct au_branch *br, + struct au_xi_calc *calc) +{ + int err; + struct file *file; + struct au_xino *xi = br->br_xino; + struct au_xi_new xinew = { + .xi = xi + }; + + SiMustAnyLock(sb); + + err = 0; + if (!xi) + goto out; + + mutex_lock(&xi->xi_mtx); + file = au_xino_file(xi, calc->idx); + if (file) + goto out_mtx; + + file = au_xino_file(xi, /*idx*/-1); + AuDebugOn(!file); + xinew.idx = calc->idx; + xinew.base = &file->f_path; + /* xinew.copy_src = NULL; */ + file = au_xi_new(sb, &xinew); + if (IS_ERR(file)) + err = PTR_ERR(file); + +out_mtx: + mutex_unlock(&xi->xi_mtx); +out: + return err; +} + +struct au_xino_do_new_async_args { + struct super_block *sb; + struct au_branch *br; + struct au_xi_calc calc; + ino_t ino; +}; + +struct au_xi_writing { + struct hlist_bl_node node; + ino_t h_ino, ino; +}; + +static int au_xino_do_write(vfs_writef_t write, struct file *file, + struct au_xi_calc *calc, ino_t ino); + +static void au_xino_call_do_new_async(void *args) +{ + struct au_xino_do_new_async_args *a = args; + struct au_branch *br; + struct super_block *sb; + struct au_sbinfo *sbi; + struct inode *root; + struct file *file; + struct au_xi_writing *del, *p; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + int err; + + br = a->br; + sb = a->sb; + sbi = au_sbi(sb); + si_noflush_read_lock(sb); + root = d_inode(sb->s_root); + ii_read_lock_child(root); + err = au_xino_do_new_async(sb, br, &a->calc); + if (unlikely(err)) { + AuIOErr("err %d\n", err); + goto out; + } + + file = au_xino_file(br->br_xino, a->calc.idx); + AuDebugOn(!file); + err = au_xino_do_write(sbi->si_xwrite, file, &a->calc, a->ino); + if (unlikely(err)) { + AuIOErr("err %d\n", err); + goto out; + } + + del = NULL; + hbl = &br->br_xino->xi_writing; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) { + p = container_of(pos, struct au_xi_writing, node); + if (p->ino == a->ino) { + del = p; + hlist_bl_del(&p->node); + break; + } + } + hlist_bl_unlock(hbl); + au_kfree_rcu(del); + +out: + au_lcnt_dec(&br->br_count); + ii_read_unlock(root); + si_read_unlock(sb); + au_nwt_done(&sbi->si_nowait); + au_kfree_rcu(a); +} + +/* + * create a new xino file asynchronously + */ +static int au_xino_new_async(struct super_block *sb, struct au_branch *br, + struct au_xi_calc *calc, ino_t ino) +{ + int err; + struct au_xino_do_new_async_args *arg; + + err = -ENOMEM; + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (unlikely(!arg)) + goto out; + + arg->sb = sb; + arg->br = br; + arg->calc = *calc; + arg->ino = ino; + au_lcnt_inc(&br->br_count); + err = au_wkq_nowait(au_xino_call_do_new_async, arg, sb, AuWkq_NEST); + if (unlikely(err)) { + pr_err("wkq %d\n", err); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(arg); + } + +out: + return err; +} + +/* + * read @ino from xinofile for the specified branch{@sb, @bindex} + * at the position of @h_ino. + */ +int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t *ino) +{ + int err; + ssize_t sz; + struct au_xi_calc calc; + struct au_sbinfo *sbinfo; + struct file *file; + struct au_xino *xi; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_xi_writing *p; + + *ino = 0; + if (!au_opt_test(au_mntflags(sb), XINO)) + return 0; /* no xino */ + + err = 0; + au_xi_calc(sb, h_ino, &calc); + xi = au_sbr(sb, bindex)->br_xino; + file = au_xino_file(xi, calc.idx); + if (!file) { + hbl = &xi->xi_writing; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) { + p = container_of(pos, struct au_xi_writing, node); + if (p->h_ino == h_ino) { + AuDbg("hi%llu, i%llu, found\n", + (u64)p->h_ino, (u64)p->ino); + *ino = p->ino; + break; + } + } + hlist_bl_unlock(hbl); + return 0; + } else if (vfsub_f_size_read(file) < calc.pos + sizeof(*ino)) + return 0; /* no xino */ + + sbinfo = au_sbi(sb); + sz = xino_fread(sbinfo->si_xread, file, ino, sizeof(*ino), &calc.pos); + if (sz == sizeof(*ino)) + return 0; /* success */ + + err = sz; + if (unlikely(sz >= 0)) { + err = -EIO; + AuIOErr("xino read error (%zd)\n", sz); + } + return err; +} + +static int au_xino_do_write(vfs_writef_t write, struct file *file, + struct au_xi_calc *calc, ino_t ino) +{ + ssize_t sz; + + sz = xino_fwrite(write, file, &ino, sizeof(ino), &calc->pos); + if (sz == sizeof(ino)) + return 0; /* success */ + + AuIOErr("write failed (%zd)\n", sz); + return -EIO; +} + +/* + * write @ino to the xinofile for the specified branch{@sb, @bindex} + * at the position of @h_ino. + * even if @ino is zero, it is written to the xinofile and means no entry. + * if the size of the xino file on a specific filesystem exceeds the watermark, + * try truncating it. + */ +int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t ino) +{ + int err; + unsigned int mnt_flags; + struct au_xi_calc calc; + struct file *file; + struct au_branch *br; + struct au_xino *xi; + struct au_xi_writing *p; + + SiMustAnyLock(sb); + + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, XINO)) + return 0; + + au_xi_calc(sb, h_ino, &calc); + br = au_sbr(sb, bindex); + xi = br->br_xino; + file = au_xino_file(xi, calc.idx); + if (!file) { + /* store the inum pair into the list */ + p = kmalloc(sizeof(*p), GFP_NOFS | __GFP_NOFAIL); + p->h_ino = h_ino; + p->ino = ino; + au_hbl_add(&p->node, &xi->xi_writing); + + /* create and write a new xino file asynchronously */ + err = au_xino_new_async(sb, br, &calc, ino); + if (!err) + return 0; /* success */ + goto out; + } + + err = au_xino_do_write(au_sbi(sb)->si_xwrite, file, &calc, ino); + if (!err) { + br = au_sbr(sb, bindex); + if (au_opt_test(mnt_flags, TRUNC_XINO) + && au_test_fs_trunc_xino(au_br_sb(br))) + xino_try_trunc(sb, br); + return 0; /* success */ + } + +out: + AuIOErr("write failed (%d)\n", err); + return -EIO; +} + +static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +/* todo: unnecessary to support mmap_sem since kernel-space? */ +ssize_t xino_fread(vfs_readf_t func, struct file *file, void *kbuf, size_t size, + loff_t *pos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + char __user *u; + } buf; + int i; + const int prevent_endless = 10; + + i = 0; + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + do { + err = func(file, buf.u, size, pos); + if (err == -EINTR + && !au_wkq_test() + && fatal_signal_pending(current)) { + set_fs(oldfs); + err = xino_fread_wkq(func, file, kbuf, size, pos); + BUG_ON(err == -EINTR); + oldfs = get_fs(); + set_fs(KERNEL_DS); + } + } while (i++ < prevent_endless + && (err == -EAGAIN || err == -EINTR)); + set_fs(oldfs); + +#if 0 /* reserved for future use */ + if (err > 0) + fsnotify_access(file->f_path.dentry); +#endif + + return err; +} + +struct xino_fread_args { + ssize_t *errp; + vfs_readf_t func; + struct file *file; + void *buf; + size_t size; + loff_t *pos; +}; + +static void call_xino_fread(void *args) +{ + struct xino_fread_args *a = args; + *a->errp = xino_fread(a->func, a->file, a->buf, a->size, a->pos); +} + +static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + int wkq_err; + struct xino_fread_args args = { + .errp = &err, + .func = func, + .file = file, + .buf = buf, + .size = size, + .pos = pos + }; + + wkq_err = au_wkq_wait(call_xino_fread, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +static ssize_t do_xino_fwrite(vfs_writef_t func, struct file *file, void *kbuf, + size_t size, loff_t *pos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + const char __user *u; + } buf; + int i; + const int prevent_endless = 10; + + i = 0; + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + do { + err = func(file, buf.u, size, pos); + if (err == -EINTR + && !au_wkq_test() + && fatal_signal_pending(current)) { + set_fs(oldfs); + err = xino_fwrite_wkq(func, file, kbuf, size, pos); + BUG_ON(err == -EINTR); + oldfs = get_fs(); + set_fs(KERNEL_DS); + } + } while (i++ < prevent_endless + && (err == -EAGAIN || err == -EINTR)); + set_fs(oldfs); + +#if 0 /* reserved for future use */ + if (err > 0) + fsnotify_modify(file->f_path.dentry); +#endif + + return err; +} + +struct do_xino_fwrite_args { + ssize_t *errp; + vfs_writef_t func; + struct file *file; + void *buf; + size_t size; + loff_t *pos; +}; + +static void call_do_xino_fwrite(void *args) +{ + struct do_xino_fwrite_args *a = args; + *a->errp = do_xino_fwrite(a->func, a->file, a->buf, a->size, a->pos); +} + +static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + int wkq_err; + struct do_xino_fwrite_args args = { + .errp = &err, + .func = func, + .file = file, + .buf = buf, + .size = size, + .pos = pos + }; + + /* + * it breaks RLIMIT_FSIZE and normal user's limit, + * users should care about quota and real 'filesystem full.' + */ + wkq_err = au_wkq_wait(call_do_xino_fwrite, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + + if (rlimit(RLIMIT_FSIZE) == RLIM_INFINITY) { + lockdep_off(); + err = do_xino_fwrite(func, file, buf, size, pos); + lockdep_on(); + } else { + lockdep_off(); + err = xino_fwrite_wkq(func, file, buf, size, pos); + lockdep_on(); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * inode number bitmap + */ +static const int page_bits = (int)PAGE_SIZE * BITS_PER_BYTE; +static ino_t xib_calc_ino(unsigned long pindex, int bit) +{ + ino_t ino; + + AuDebugOn(bit < 0 || page_bits <= bit); + ino = AUFS_FIRST_INO + pindex * page_bits + bit; + return ino; +} + +static void xib_calc_bit(ino_t ino, unsigned long *pindex, int *bit) +{ + AuDebugOn(ino < AUFS_FIRST_INO); + ino -= AUFS_FIRST_INO; + *pindex = ino / page_bits; + *bit = ino % page_bits; +} + +static int xib_pindex(struct super_block *sb, unsigned long pindex) +{ + int err; + loff_t pos; + ssize_t sz; + struct au_sbinfo *sbinfo; + struct file *xib; + unsigned long *p; + + sbinfo = au_sbi(sb); + MtxMustLock(&sbinfo->si_xib_mtx); + AuDebugOn(pindex > ULONG_MAX / PAGE_SIZE + || !au_opt_test(sbinfo->si_mntflags, XINO)); + + if (pindex == sbinfo->si_xib_last_pindex) + return 0; + + xib = sbinfo->si_xib; + p = sbinfo->si_xib_buf; + pos = sbinfo->si_xib_last_pindex; + pos *= PAGE_SIZE; + sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos); + if (unlikely(sz != PAGE_SIZE)) + goto out; + + pos = pindex; + pos *= PAGE_SIZE; + if (vfsub_f_size_read(xib) >= pos + PAGE_SIZE) + sz = xino_fread(sbinfo->si_xread, xib, p, PAGE_SIZE, &pos); + else { + memset(p, 0, PAGE_SIZE); + sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos); + } + if (sz == PAGE_SIZE) { + sbinfo->si_xib_last_pindex = pindex; + return 0; /* success */ + } + +out: + AuIOErr1("write failed (%zd)\n", sz); + err = sz; + if (sz >= 0) + err = -EIO; + return err; +} + +static void au_xib_clear_bit(struct inode *inode) +{ + int err, bit; + unsigned long pindex; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + AuDebugOn(inode->i_nlink); + + sb = inode->i_sb; + xib_calc_bit(inode->i_ino, &pindex, &bit); + AuDebugOn(page_bits <= bit); + sbinfo = au_sbi(sb); + mutex_lock(&sbinfo->si_xib_mtx); + err = xib_pindex(sb, pindex); + if (!err) { + clear_bit(bit, sbinfo->si_xib_buf); + sbinfo->si_xib_next_bit = bit; + } + mutex_unlock(&sbinfo->si_xib_mtx); +} + +/* ---------------------------------------------------------------------- */ + +/* + * truncate a xino bitmap file + */ + +/* todo: slow */ +static int do_xib_restore(struct super_block *sb, struct file *file, void *page) +{ + int err, bit; + ssize_t sz; + unsigned long pindex; + loff_t pos, pend; + struct au_sbinfo *sbinfo; + vfs_readf_t func; + ino_t *ino; + unsigned long *p; + + err = 0; + sbinfo = au_sbi(sb); + MtxMustLock(&sbinfo->si_xib_mtx); + p = sbinfo->si_xib_buf; + func = sbinfo->si_xread; + pend = vfsub_f_size_read(file); + pos = 0; + while (pos < pend) { + sz = xino_fread(func, file, page, PAGE_SIZE, &pos); + err = sz; + if (unlikely(sz <= 0)) + goto out; + + err = 0; + for (ino = page; sz > 0; ino++, sz -= sizeof(ino)) { + if (unlikely(*ino < AUFS_FIRST_INO)) + continue; + + xib_calc_bit(*ino, &pindex, &bit); + AuDebugOn(page_bits <= bit); + err = xib_pindex(sb, pindex); + if (!err) + set_bit(bit, p); + else + goto out; + } + } + +out: + return err; +} + +static int xib_restore(struct super_block *sb) +{ + int err, i; + unsigned int nfile; + aufs_bindex_t bindex, bbot; + void *page; + struct au_branch *br; + struct au_xino *xi; + struct file *file; + + err = -ENOMEM; + page = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!page)) + goto out; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) + if (!bindex || is_sb_shared(sb, bindex, bindex - 1) < 0) { + br = au_sbr(sb, bindex); + xi = br->br_xino; + nfile = xi->xi_nfile; + for (i = 0; i < nfile; i++) { + file = au_xino_file(xi, i); + if (file) + err = do_xib_restore(sb, file, page); + } + } else + AuDbg("skip shared b%d\n", bindex); + free_page((unsigned long)page); + +out: + return err; +} + +int au_xib_trunc(struct super_block *sb) +{ + int err; + ssize_t sz; + loff_t pos; + struct au_sbinfo *sbinfo; + unsigned long *p; + struct file *file; + + SiMustWriteLock(sb); + + err = 0; + sbinfo = au_sbi(sb); + if (!au_opt_test(sbinfo->si_mntflags, XINO)) + goto out; + + file = sbinfo->si_xib; + if (vfsub_f_size_read(file) <= PAGE_SIZE) + goto out; + + file = au_xino_create2(sb, &sbinfo->si_xib->f_path, NULL); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + fput(sbinfo->si_xib); + sbinfo->si_xib = file; + + p = sbinfo->si_xib_buf; + memset(p, 0, PAGE_SIZE); + pos = 0; + sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xib, p, PAGE_SIZE, &pos); + if (unlikely(sz != PAGE_SIZE)) { + err = sz; + AuIOErr("err %d\n", err); + if (sz >= 0) + err = -EIO; + goto out; + } + + mutex_lock(&sbinfo->si_xib_mtx); + /* mnt_want_write() is unnecessary here */ + err = xib_restore(sb); + mutex_unlock(&sbinfo->si_xib_mtx); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_xino *au_xino_alloc(unsigned int nfile) +{ + struct au_xino *xi; + + xi = kzalloc(sizeof(*xi), GFP_NOFS); + if (unlikely(!xi)) + goto out; + xi->xi_nfile = nfile; + xi->xi_file = kcalloc(nfile, sizeof(*xi->xi_file), GFP_NOFS); + if (unlikely(!xi->xi_file)) + goto out_free; + + xi->xi_nondir.total = 8; /* initial size */ + xi->xi_nondir.array = kcalloc(xi->xi_nondir.total, sizeof(ino_t), + GFP_NOFS); + if (unlikely(!xi->xi_nondir.array)) + goto out_file; + + spin_lock_init(&xi->xi_nondir.spin); + init_waitqueue_head(&xi->xi_nondir.wqh); + mutex_init(&xi->xi_mtx); + INIT_HLIST_BL_HEAD(&xi->xi_writing); + atomic_set(&xi->xi_truncating, 0); + kref_init(&xi->xi_kref); + goto out; /* success */ + +out_file: + au_kfree_try_rcu(xi->xi_file); +out_free: + au_kfree_rcu(xi); + xi = NULL; +out: + return xi; +} + +static int au_xino_init(struct au_branch *br, int idx, struct file *file) +{ + int err; + struct au_xino *xi; + + err = 0; + xi = au_xino_alloc(idx + 1); + if (unlikely(!xi)) { + err = -ENOMEM; + goto out; + } + + if (file) + get_file(file); + xi->xi_file[idx] = file; + AuDebugOn(br->br_xino); + br->br_xino = xi; + +out: + return err; +} + +static void au_xino_release(struct kref *kref) +{ + struct au_xino *xi; + int i; + unsigned long ul; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *n; + struct au_xi_writing *p; + + xi = container_of(kref, struct au_xino, xi_kref); + for (i = 0; i < xi->xi_nfile; i++) + if (xi->xi_file[i]) + fput(xi->xi_file[i]); + for (i = xi->xi_nondir.total - 1; i >= 0; i--) + AuDebugOn(xi->xi_nondir.array[i]); + mutex_destroy(&xi->xi_mtx); + hbl = &xi->xi_writing; + ul = au_hbl_count(hbl); + if (unlikely(ul)) { + pr_warn("xi_writing %lu\n", ul); + hlist_bl_lock(hbl); + hlist_bl_for_each_entry_safe (p, pos, n, hbl, node) { + hlist_bl_del(&p->node); + au_kfree_rcu(p); + } + hlist_bl_unlock(hbl); + } + au_kfree_try_rcu(xi->xi_file); + au_kfree_try_rcu(xi->xi_nondir.array); + au_kfree_rcu(xi); +} + +int au_xino_put(struct au_branch *br) +{ + int ret; + struct au_xino *xi; + + ret = 0; + xi = br->br_xino; + if (xi) { + br->br_xino = NULL; + ret = kref_put(&xi->xi_kref, au_xino_release); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* + * xino mount option handlers + */ + +/* xino bitmap */ +static void xino_clear_xib(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + /* unnecessary to clear sbinfo->si_xread and ->si_xwrite */ + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; + if (sbinfo->si_xib_buf) + free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; +} + +static int au_xino_set_xib(struct super_block *sb, struct path *path) +{ + int err; + loff_t pos; + struct au_sbinfo *sbinfo; + struct file *file; + struct super_block *xi_sb; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + file = au_xino_create2(sb, path, sbinfo->si_xib); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = file; + sbinfo->si_xread = vfs_readf(file); + sbinfo->si_xwrite = vfs_writef(file); + xi_sb = file_inode(file)->i_sb; + sbinfo->si_ximaxent = xi_sb->s_maxbytes; + if (unlikely(sbinfo->si_ximaxent < PAGE_SIZE)) { + err = -EIO; + pr_err("s_maxbytes(%llu) on %s is too small\n", + (u64)sbinfo->si_ximaxent, au_sbtype(xi_sb)); + goto out_unset; + } + sbinfo->si_ximaxent /= sizeof(ino_t); + + err = -ENOMEM; + if (!sbinfo->si_xib_buf) + sbinfo->si_xib_buf = (void *)get_zeroed_page(GFP_NOFS); + if (unlikely(!sbinfo->si_xib_buf)) + goto out_unset; + + sbinfo->si_xib_last_pindex = 0; + sbinfo->si_xib_next_bit = 0; + if (vfsub_f_size_read(file) < PAGE_SIZE) { + pos = 0; + err = xino_fwrite(sbinfo->si_xwrite, file, sbinfo->si_xib_buf, + PAGE_SIZE, &pos); + if (unlikely(err != PAGE_SIZE)) + goto out_free; + } + err = 0; + goto out; /* success */ + +out_free: + if (sbinfo->si_xib_buf) + free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; + if (err >= 0) + err = -EIO; +out_unset: + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; +out: + AuTraceErr(err); + return err; +} + +/* xino for each branch */ +static void xino_clear_br(struct super_block *sb) +{ + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + AuDebugOn(!br); + au_xino_put(br); + } +} + +static void au_xino_set_br_shared(struct super_block *sb, struct au_branch *br, + aufs_bindex_t bshared) +{ + struct au_branch *brshared; + + brshared = au_sbr(sb, bshared); + AuDebugOn(!brshared->br_xino); + AuDebugOn(!brshared->br_xino->xi_file); + if (br->br_xino != brshared->br_xino) { + au_xino_get(brshared); + au_xino_put(br); + br->br_xino = brshared->br_xino; + } +} + +struct au_xino_do_set_br { + vfs_writef_t writef; + struct au_branch *br; + ino_t h_ino; + aufs_bindex_t bshared; +}; + +static int au_xino_do_set_br(struct super_block *sb, struct path *path, + struct au_xino_do_set_br *args) +{ + int err; + struct au_xi_calc calc; + struct file *file; + struct au_branch *br; + struct au_xi_new xinew = { + .base = path + }; + + br = args->br; + xinew.xi = br->br_xino; + au_xi_calc(sb, args->h_ino, &calc); + xinew.copy_src = au_xino_file(xinew.xi, calc.idx); + if (args->bshared >= 0) + /* shared xino */ + au_xino_set_br_shared(sb, br, args->bshared); + else if (!xinew.xi) { + /* new xino */ + err = au_xino_init(br, calc.idx, xinew.copy_src); + if (unlikely(err)) + goto out; + } + + /* force re-creating */ + xinew.xi = br->br_xino; + xinew.idx = calc.idx; + mutex_lock(&xinew.xi->xi_mtx); + file = au_xi_new(sb, &xinew); + mutex_unlock(&xinew.xi->xi_mtx); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + AuDebugOn(!file); + + err = au_xino_do_write(args->writef, file, &calc, AUFS_ROOT_INO); + if (unlikely(err)) + au_xino_put(br); + +out: + AuTraceErr(err); + return err; +} + +static int au_xino_set_br(struct super_block *sb, struct path *path) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_xino_do_set_br args; + struct inode *inode; + + SiMustWriteLock(sb); + + bbot = au_sbbot(sb); + inode = d_inode(sb->s_root); + args.writef = au_sbi(sb)->si_xwrite; + for (bindex = 0; bindex <= bbot; bindex++) { + args.h_ino = au_h_iptr(inode, bindex)->i_ino; + args.br = au_sbr(sb, bindex); + args.bshared = is_sb_shared(sb, bindex, bindex - 1); + err = au_xino_do_set_br(sb, path, &args); + if (unlikely(err)) + break; + } + + AuTraceErr(err); + return err; +} + +void au_xino_clr(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + au_xigen_clr(sb); + xino_clear_xib(sb); + xino_clear_br(sb); + dbgaufs_brs_del(sb, 0); + sbinfo = au_sbi(sb); + /* lvalue, do not call au_mntflags() */ + au_opt_clr(sbinfo->si_mntflags, XINO); +} + +int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount) +{ + int err, skip; + struct dentry *dentry, *parent, *cur_dentry, *cur_parent; + struct qstr *dname, *cur_name; + struct file *cur_xino; + struct au_sbinfo *sbinfo; + struct path *path, *cur_path; + + SiMustWriteLock(sb); + + err = 0; + sbinfo = au_sbi(sb); + path = &xiopt->file->f_path; + dentry = path->dentry; + parent = dget_parent(dentry); + if (remount) { + skip = 0; + cur_xino = sbinfo->si_xib; + if (cur_xino) { + cur_path = &cur_xino->f_path; + cur_dentry = cur_path->dentry; + cur_parent = dget_parent(cur_dentry); + cur_name = &cur_dentry->d_name; + dname = &dentry->d_name; + skip = (cur_parent == parent + && au_qstreq(dname, cur_name)); + dput(cur_parent); + } + if (skip) + goto out; + } + + au_opt_set(sbinfo->si_mntflags, XINO); + err = au_xino_set_xib(sb, path); + /* si_x{read,write} are set */ + if (!err) + err = au_xigen_set(sb, path); + if (!err) + err = au_xino_set_br(sb, path); + if (!err) { + dbgaufs_brs_add(sb, 0, /*topdown*/1); + goto out; /* success */ + } + + /* reset all */ + AuIOErr("failed setting xino(%d).\n", err); + au_xino_clr(sb); + +out: + dput(parent); + return err; +} + +/* + * create a xinofile at the default place/path. + */ +struct file *au_xino_def(struct super_block *sb) +{ + struct file *file; + char *page, *p; + struct au_branch *br; + struct super_block *h_sb; + struct path path; + aufs_bindex_t bbot, bindex, bwr; + + br = NULL; + bbot = au_sbbot(sb); + bwr = -1; + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm) + && !au_test_fs_bad_xino(au_br_sb(br))) { + bwr = bindex; + break; + } + } + + if (bwr >= 0) { + file = ERR_PTR(-ENOMEM); + page = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!page)) + goto out; + path.mnt = au_br_mnt(br); + path.dentry = au_h_dptr(sb->s_root, bwr); + p = d_path(&path, page, PATH_MAX - sizeof(AUFS_XINO_FNAME)); + file = (void *)p; + if (!IS_ERR(p)) { + strcat(p, "/" AUFS_XINO_FNAME); + AuDbg("%s\n", p); + file = au_xino_create(sb, p, /*silent*/0, /*wbrtop*/1); + } + free_page((unsigned long)page); + } else { + file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0, + /*wbrtop*/0); + if (IS_ERR(file)) + goto out; + h_sb = file->f_path.dentry->d_sb; + if (unlikely(au_test_fs_bad_xino(h_sb))) { + pr_err("xino doesn't support %s(%s)\n", + AUFS_XINO_DEFPATH, au_sbtype(h_sb)); + fput(file); + file = ERR_PTR(-EINVAL); + } + } + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* + * initialize the xinofile for the specified branch @br + * at the place/path where @base_file indicates. + * test whether another branch is on the same filesystem or not, + * if found then share the xinofile with another branch. + */ +int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t h_ino, + struct path *base) +{ + int err; + struct au_xino_do_set_br args = { + .h_ino = h_ino, + .br = br + }; + + args.writef = au_sbi(sb)->si_xwrite; + args.bshared = sbr_find_shared(sb, /*btop*/0, au_sbbot(sb), + au_br_sb(br)); + err = au_xino_do_set_br(sb, base, &args); + if (unlikely(err)) + au_xino_put(br); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * get an unused inode number from bitmap + */ +ino_t au_xino_new_ino(struct super_block *sb) +{ + ino_t ino; + unsigned long *p, pindex, ul, pend; + struct au_sbinfo *sbinfo; + struct file *file; + int free_bit, err; + + if (!au_opt_test(au_mntflags(sb), XINO)) + return iunique(sb, AUFS_FIRST_INO); + + sbinfo = au_sbi(sb); + mutex_lock(&sbinfo->si_xib_mtx); + p = sbinfo->si_xib_buf; + free_bit = sbinfo->si_xib_next_bit; + if (free_bit < page_bits && !test_bit(free_bit, p)) + goto out; /* success */ + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + + pindex = sbinfo->si_xib_last_pindex; + for (ul = pindex - 1; ul < ULONG_MAX; ul--) { + err = xib_pindex(sb, ul); + if (unlikely(err)) + goto out_err; + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + } + + file = sbinfo->si_xib; + pend = vfsub_f_size_read(file) / PAGE_SIZE; + for (ul = pindex + 1; ul <= pend; ul++) { + err = xib_pindex(sb, ul); + if (unlikely(err)) + goto out_err; + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + } + BUG(); + +out: + set_bit(free_bit, p); + sbinfo->si_xib_next_bit = free_bit + 1; + pindex = sbinfo->si_xib_last_pindex; + mutex_unlock(&sbinfo->si_xib_mtx); + ino = xib_calc_ino(pindex, free_bit); + AuDbg("i%lu\n", (unsigned long)ino); + return ino; +out_err: + mutex_unlock(&sbinfo->si_xib_mtx); + AuDbg("i0\n"); + return 0; +} + +/* for s_op->delete_inode() */ +void au_xino_delete_inode(struct inode *inode, const int unlinked) +{ + int err; + unsigned int mnt_flags; + aufs_bindex_t bindex, bbot, bi; + unsigned char try_trunc; + struct au_iinfo *iinfo; + struct super_block *sb; + struct au_hinode *hi; + struct inode *h_inode; + struct au_branch *br; + vfs_writef_t xwrite; + struct au_xi_calc calc; + struct file *file; + + AuDebugOn(au_is_bad_inode(inode)); + + sb = inode->i_sb; + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, XINO) + || inode->i_ino == AUFS_ROOT_INO) + return; + + if (unlinked) { + au_xigen_inc(inode); + au_xib_clear_bit(inode); + } + + iinfo = au_ii(inode); + bindex = iinfo->ii_btop; + if (bindex < 0) + return; + + xwrite = au_sbi(sb)->si_xwrite; + try_trunc = !!au_opt_test(mnt_flags, TRUNC_XINO); + hi = au_hinode(iinfo, bindex); + bbot = iinfo->ii_bbot; + for (; bindex <= bbot; bindex++, hi++) { + h_inode = hi->hi_inode; + if (!h_inode + || (!unlinked && h_inode->i_nlink)) + continue; + + /* inode may not be revalidated */ + bi = au_br_index(sb, hi->hi_id); + if (bi < 0) + continue; + + br = au_sbr(sb, bi); + au_xi_calc(sb, h_inode->i_ino, &calc); + file = au_xino_file(br->br_xino, calc.idx); + if (IS_ERR_OR_NULL(file)) + continue; + + err = au_xino_do_write(xwrite, file, &calc, /*ino*/0); + if (!err && try_trunc + && au_test_fs_trunc_xino(au_br_sb(br))) + xino_try_trunc(sb, br); + } +} + +/* ---------------------------------------------------------------------- */ + +static int au_xinondir_find(struct au_xino *xi, ino_t h_ino) +{ + int found, total, i; + + found = -1; + total = xi->xi_nondir.total; + for (i = 0; i < total; i++) { + if (xi->xi_nondir.array[i] != h_ino) + continue; + found = i; + break; + } + + return found; +} + +static int au_xinondir_expand(struct au_xino *xi) +{ + int err, sz; + ino_t *p; + + BUILD_BUG_ON(KMALLOC_MAX_SIZE > INT_MAX); + + err = -ENOMEM; + sz = xi->xi_nondir.total * sizeof(ino_t); + if (unlikely(sz > KMALLOC_MAX_SIZE / 2)) + goto out; + p = au_kzrealloc(xi->xi_nondir.array, sz, sz << 1, GFP_ATOMIC, + /*may_shrink*/0); + if (p) { + xi->xi_nondir.array = p; + xi->xi_nondir.total <<= 1; + AuDbg("xi_nondir.total %d\n", xi->xi_nondir.total); + err = 0; + } + +out: + return err; +} + +void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, int idx) +{ + struct au_xino *xi; + + AuDebugOn(!au_opt_test(au_mntflags(sb), XINO)); + xi = au_sbr(sb, bindex)->br_xino; + AuDebugOn(idx < 0 || xi->xi_nondir.total <= idx); + + spin_lock(&xi->xi_nondir.spin); + AuDebugOn(xi->xi_nondir.array[idx] != h_ino); + xi->xi_nondir.array[idx] = 0; + spin_unlock(&xi->xi_nondir.spin); + wake_up_all(&xi->xi_nondir.wqh); +} + +int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + int *idx) +{ + int err, found, empty; + struct au_xino *xi; + + err = 0; + *idx = -1; + if (!au_opt_test(au_mntflags(sb), XINO)) + goto out; /* no xino */ + + xi = au_sbr(sb, bindex)->br_xino; + +again: + spin_lock(&xi->xi_nondir.spin); + found = au_xinondir_find(xi, h_ino); + if (found == -1) { + empty = au_xinondir_find(xi, /*h_ino*/0); + if (empty == -1) { + empty = xi->xi_nondir.total; + err = au_xinondir_expand(xi); + if (unlikely(err)) + goto out_unlock; + } + xi->xi_nondir.array[empty] = h_ino; + *idx = empty; + } else { + spin_unlock(&xi->xi_nondir.spin); + wait_event(xi->xi_nondir.wqh, + xi->xi_nondir.array[found] != h_ino); + goto again; + } + +out_unlock: + spin_unlock(&xi->xi_nondir.spin); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_xino_path(struct seq_file *seq, struct file *file) +{ + int err; + + err = au_seq_path(seq, &file->f_path); + if (unlikely(err)) + goto out; + +#define Deleted "\\040(deleted)" + seq->count -= sizeof(Deleted) - 1; + AuDebugOn(memcmp(seq->buf + seq->count, Deleted, + sizeof(Deleted) - 1)); +#undef Deleted + +out: + return err; +} diff --git a/fs/dcache.c b/fs/dcache.c index 9ac1290ae44f9..c4fe6f8db0021 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1217,7 +1217,7 @@ enum d_walk_ret { * * The @enter() and @finish() callbacks are called with d_lock held. */ -static void d_walk(struct dentry *parent, void *data, +void d_walk(struct dentry *parent, void *data, enum d_walk_ret (*enter)(void *, struct dentry *), void (*finish)(void *)) { @@ -1325,6 +1325,7 @@ static void d_walk(struct dentry *parent, void *data, seq = 1; goto again; } +EXPORT_SYMBOL_GPL(d_walk); struct check_mount { struct vfsmount *mnt; @@ -2950,6 +2951,7 @@ void d_exchange(struct dentry *dentry1, struct dentry *dentry2) write_sequnlock(&rename_lock); } +EXPORT_SYMBOL_GPL(d_exchange); /** * d_ancestor - search for an ancestor diff --git a/fs/exec.c b/fs/exec.c index 0936b5a8199ac..8a286075e6d6a 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -109,6 +109,7 @@ bool path_noexec(const struct path *path) return (path->mnt->mnt_flags & MNT_NOEXEC) || (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +EXPORT_SYMBOL_GPL(path_noexec); #ifdef CONFIG_USELIB /* diff --git a/fs/fcntl.c b/fs/fcntl.c index 0345a46b88565..e8c9dc38c2e15 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -32,7 +32,7 @@ #define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME) -static int setfl(int fd, struct file * filp, unsigned long arg) +int setfl(int fd, struct file * filp, unsigned long arg) { struct inode * inode = file_inode(filp); int error = 0; @@ -63,6 +63,8 @@ static int setfl(int fd, struct file * filp, unsigned long arg) if (filp->f_op->check_flags) error = filp->f_op->check_flags(arg); + if (!error && filp->f_op->setfl) + error = filp->f_op->setfl(filp, arg); if (error) return error; @@ -83,6 +85,7 @@ static int setfl(int fd, struct file * filp, unsigned long arg) out: return error; } +EXPORT_SYMBOL_GPL(setfl); static void f_modown(struct file *filp, struct pid *pid, enum pid_type type, int force) diff --git a/fs/file_table.c b/fs/file_table.c index 61517f57f8ef7..c6bab39c1813e 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -148,6 +148,7 @@ struct file *get_empty_filp(void) } return ERR_PTR(-ENFILE); } +EXPORT_SYMBOL_GPL(get_empty_filp); /** * alloc_file - allocate and initialize a 'struct file' @@ -258,6 +259,7 @@ void flush_delayed_fput(void) { delayed_fput(NULL); } +EXPORT_SYMBOL_GPL(flush_delayed_fput); static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput); @@ -300,6 +302,7 @@ void __fput_sync(struct file *file) } EXPORT_SYMBOL(fput); +EXPORT_SYMBOL_GPL(__fput_sync); void put_filp(struct file *file) { @@ -308,6 +311,7 @@ void put_filp(struct file *file) file_free(file); } } +EXPORT_SYMBOL_GPL(put_filp); void __init files_init(void) { diff --git a/fs/inode.c b/fs/inode.c index cfc36d11bcb37..8ca328694bc51 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1656,7 +1656,7 @@ EXPORT_SYMBOL(generic_update_time); * This does the actual work of updating an inodes time or version. Must have * had called mnt_want_write() before calling this. */ -static int update_time(struct inode *inode, struct timespec *time, int flags) +int update_time(struct inode *inode, struct timespec *time, int flags) { int (*update_time)(struct inode *, struct timespec *, int); @@ -1665,6 +1665,7 @@ static int update_time(struct inode *inode, struct timespec *time, int flags) return update_time(inode, time, flags); } +EXPORT_SYMBOL_GPL(update_time); /** * touch_atime - update the access time diff --git a/fs/namespace.c b/fs/namespace.c index e9c13eedd7393..3e03d3633e8cf 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -517,6 +517,7 @@ void __mnt_drop_write(struct vfsmount *mnt) mnt_dec_writers(real_mount(mnt)); preempt_enable(); } +EXPORT_SYMBOL_GPL(__mnt_drop_write); /** * mnt_drop_write - give up write access to a mount @@ -855,6 +856,13 @@ static inline int check_mnt(struct mount *mnt) return mnt->mnt_ns == current->nsproxy->mnt_ns; } +/* for aufs, CONFIG_AUFS_BR_FUSE */ +int is_current_mnt_ns(struct vfsmount *mnt) +{ + return check_mnt(real_mount(mnt)); +} +EXPORT_SYMBOL_GPL(is_current_mnt_ns); + /* * vfsmount lock must be held for write */ @@ -1918,6 +1926,7 @@ int iterate_mounts(int (*f)(struct vfsmount *, void *), void *arg, } return 0; } +EXPORT_SYMBOL_GPL(iterate_mounts); static void cleanup_group_ids(struct mount *mnt, struct mount *end) { diff --git a/fs/notify/group.c b/fs/notify/group.c index 32357534de18d..9526908730539 100644 --- a/fs/notify/group.c +++ b/fs/notify/group.c @@ -109,6 +109,7 @@ void fsnotify_get_group(struct fsnotify_group *group) { atomic_inc(&group->refcnt); } +EXPORT_SYMBOL_GPL(fsnotify_get_group); /* * Drop a reference to a group. Free it if it's through. @@ -118,6 +119,7 @@ void fsnotify_put_group(struct fsnotify_group *group) if (atomic_dec_and_test(&group->refcnt)) fsnotify_final_destroy_group(group); } +EXPORT_SYMBOL_GPL(fsnotify_put_group); /* * Create a new fsnotify_group and hold a reference for the group returned. @@ -147,6 +149,7 @@ struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops) return group; } +EXPORT_SYMBOL_GPL(fsnotify_alloc_group); int fsnotify_fasync(int fd, struct file *file, int on) { diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 258d99087183d..cd0dd398f60d1 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -245,6 +245,7 @@ void fsnotify_put_mark(struct fsnotify_mark *mark) queue_delayed_work(system_unbound_wq, &reaper_work, FSNOTIFY_REAPER_DELAY); } +EXPORT_SYMBOL_GPL(fsnotify_put_mark); /* * Get mark reference when we found the mark via lockless traversal of object @@ -392,6 +393,7 @@ void fsnotify_destroy_mark(struct fsnotify_mark *mark, mutex_unlock(&group->mark_mutex); fsnotify_free_mark(mark); } +EXPORT_SYMBOL_GPL(fsnotify_destroy_mark); /* * Sorting function for lists of fsnotify marks. @@ -616,6 +618,7 @@ int fsnotify_add_mark(struct fsnotify_mark *mark, struct inode *inode, mutex_unlock(&group->mark_mutex); return ret; } +EXPORT_SYMBOL_GPL(fsnotify_add_mark); /* * Given a list of marks, find the mark associated with given group. If found @@ -739,6 +742,7 @@ void fsnotify_init_mark(struct fsnotify_mark *mark, fsnotify_get_group(group); mark->group = group; } +EXPORT_SYMBOL_GPL(fsnotify_init_mark); /* * Destroy all marks in destroy_list, waits for SRCU period to finish before diff --git a/fs/open.c b/fs/open.c index 7ea118471dce5..6e2e2418a840e 100644 --- a/fs/open.c +++ b/fs/open.c @@ -64,6 +64,7 @@ int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, inode_unlock(dentry->d_inode); return ret; } +EXPORT_SYMBOL_GPL(do_truncate); long vfs_truncate(const struct path *path, loff_t length) { @@ -691,6 +692,7 @@ int open_check_o_direct(struct file *f) } return 0; } +EXPORT_SYMBOL_GPL(open_check_o_direct); static int do_dentry_open(struct file *f, struct inode *inode, diff --git a/fs/proc/base.c b/fs/proc/base.c index 64695dcf89f3b..eeaad0f45b6ef 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2031,7 +2031,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) down_read(&mm->mmap_sem); vma = find_exact_vma(mm, vm_start, vm_end); if (vma && vma->vm_file) { - *path = vma->vm_file->f_path; + *path = vma_pr_or_file(vma)->f_path; path_get(path); rc = 0; } diff --git a/fs/proc/nommu.c b/fs/proc/nommu.c index 75634379f82ee..7c0dc0ff48828 100644 --- a/fs/proc/nommu.c +++ b/fs/proc/nommu.c @@ -45,7 +45,10 @@ static int nommu_region_show(struct seq_file *m, struct vm_region *region) file = region->vm_file; if (file) { - struct inode *inode = file_inode(region->vm_file); + struct inode *inode; + + file = vmr_pr_or_file(region); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5e63c459dc612..6b5c53e10fef0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -310,7 +310,10 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid) const char *name = NULL; if (file) { - struct inode *inode = file_inode(vma->vm_file); + struct inode *inode; + + file = vma_pr_or_file(vma); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; @@ -1757,7 +1760,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid) struct proc_maps_private *proc_priv = &numa_priv->proc_maps; struct vm_area_struct *vma = v; struct numa_maps *md = &numa_priv->md; - struct file *file = vma->vm_file; + struct file *file = vma_pr_or_file(vma); struct mm_struct *mm = vma->vm_mm; struct mm_walk walk = { .hugetlb_entry = gather_hugetlb_stats, diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c index 5b62f57bd9bce..dfb4a3bd0fac2 100644 --- a/fs/proc/task_nommu.c +++ b/fs/proc/task_nommu.c @@ -156,7 +156,10 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma, file = vma->vm_file; if (file) { - struct inode *inode = file_inode(vma->vm_file); + struct inode *inode; + + file = vma_pr_or_file(vma); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT; diff --git a/fs/read_write.c b/fs/read_write.c index 57a00ef895b29..1221dda338d57 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -454,6 +454,7 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) return ret; } +EXPORT_SYMBOL_GPL(vfs_read); static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) { @@ -484,6 +485,30 @@ ssize_t __vfs_write(struct file *file, const char __user *p, size_t count, return -EINVAL; } +vfs_readf_t vfs_readf(struct file *file) +{ + const struct file_operations *fop = file->f_op; + + if (fop->read) + return fop->read; + if (fop->read_iter) + return new_sync_read; + return ERR_PTR(-ENOSYS); +} +EXPORT_SYMBOL_GPL(vfs_readf); + +vfs_writef_t vfs_writef(struct file *file) +{ + const struct file_operations *fop = file->f_op; + + if (fop->write) + return fop->write; + if (fop->write_iter) + return new_sync_write; + return ERR_PTR(-ENOSYS); +} +EXPORT_SYMBOL_GPL(vfs_writef); + ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) { mm_segment_t old_fs; @@ -552,6 +577,7 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ return ret; } +EXPORT_SYMBOL_GPL(vfs_write); static inline loff_t file_pos_read(struct file *file) { diff --git a/fs/splice.c b/fs/splice.c index 00d2f142dcf98..8d94e2333f57e 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -837,8 +837,8 @@ EXPORT_SYMBOL(generic_splice_sendpage); /* * Attempt to initiate a splice from pipe to file. */ -static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, - loff_t *ppos, size_t len, unsigned int flags) +long do_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) { ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); @@ -850,13 +850,14 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, return splice_write(pipe, out, ppos, len, flags); } +EXPORT_SYMBOL_GPL(do_splice_from); /* * Attempt to initiate a splice from a file to a pipe. */ -static long do_splice_to(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) +long do_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) { ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); @@ -879,6 +880,7 @@ static long do_splice_to(struct file *in, loff_t *ppos, return splice_read(in, ppos, pipe, len, flags); } +EXPORT_SYMBOL_GPL(do_splice_to); /** * splice_direct_to_actor - splices data directly between two non-pipes diff --git a/fs/sync.c b/fs/sync.c index 83ac79a960dd1..e3386eabba814 100644 --- a/fs/sync.c +++ b/fs/sync.c @@ -28,7 +28,7 @@ * wait == 1 case since in that case write_inode() functions do * sync_dirty_buffer() and thus effectively write one block at a time. */ -static int __sync_filesystem(struct super_block *sb, int wait) +int __sync_filesystem(struct super_block *sb, int wait) { if (wait) sync_inodes_sb(sb); @@ -39,6 +39,7 @@ static int __sync_filesystem(struct super_block *sb, int wait) sb->s_op->sync_fs(sb, wait); return __sync_blockdev(sb->s_bdev, wait); } +EXPORT_SYMBOL_GPL(__sync_filesystem); /* * Write out and wait upon all dirty data associated with this diff --git a/fs/xattr.c b/fs/xattr.c index 50029811fbe38..ee34b72567d10 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -297,6 +297,7 @@ vfs_getxattr_alloc(struct dentry *dentry, const char *name, char **xattr_value, *xattr_value = value; return error; } +EXPORT_SYMBOL_GPL(vfs_getxattr_alloc); ssize_t __vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name, diff --git a/include/dt-bindings/board/am335x-bbw-bbb-base.h b/include/dt-bindings/board/am335x-bbw-bbb-base.h new file mode 100644 index 0000000000000..35f6d57ef4923 --- /dev/null +++ b/include/dt-bindings/board/am335x-bbw-bbb-base.h @@ -0,0 +1,103 @@ +/* + * This header provides constants for bbw/bbb pinctrl bindings. + * + * Copyright (C) 2014 Robert Nelson + * + * Numbers Based on: https://github.com/derekmolloy/boneDeviceTree/tree/master/docs + */ + +#ifndef _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H +#define _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H + +#define BONE_P8_03 0x018 +#define BONE_P8_04 0x01C + +#define BONE_P8_05 0x008 +#define BONE_P8_06 0x00C +#define BONE_P8_07 0x090 +#define BONE_P8_08 0x094 + +#define BONE_P8_09 0x09C +#define BONE_P8_10 0x098 +#define BONE_P8_11 0x034 +#define BONE_P8_12 0x030 + +#define BONE_P8_13 0x024 +#define BONE_P8_14 0x028 +#define BONE_P8_15 0x03C +#define BONE_P8_16 0x038 + +#define BONE_P8_17 0x02C +#define BONE_P8_18 0x08C +#define BONE_P8_19 0x020 +#define BONE_P8_20 0x084 + +#define BONE_P8_21 0x080 +#define BONE_P8_22 0x014 +#define BONE_P8_23 0x010 +#define BONE_P8_24 0x004 + +#define BONE_P8_25 0x000 +#define BONE_P8_26 0x07C +#define BONE_P8_27 0x0E0 +#define BONE_P8_28 0x0E8 + +#define BONE_P8_29 0x0E4 +#define BONE_P8_30 0x0EC +#define BONE_P8_31 0x0D8 +#define BONE_P8_32 0x0DC + +#define BONE_P8_33 0x0D4 +#define BONE_P8_34 0x0CC +#define BONE_P8_35 0x0D0 +#define BONE_P8_36 0x0C8 + +#define BONE_P8_37 0x0C0 +#define BONE_P8_38 0x0C4 +#define BONE_P8_39 0x0B8 +#define BONE_P8_40 0x0BC + +#define BONE_P8_41 0x0B0 +#define BONE_P8_42 0x0B4 +#define BONE_P8_43 0x0A8 +#define BONE_P8_44 0x0AC + +#define BONE_P8_45 0x0A0 +#define BONE_P8_46 0x0A4 + +#define BONE_P9_11 0x070 +#define BONE_P9_12 0x078 + +#define BONE_P9_13 0x074 +#define BONE_P9_14 0x048 +#define BONE_P9_15 0x040 +#define BONE_P9_16 0x04C + +#define BONE_P9_17 0x15C +#define BONE_P9_18 0x158 +#define BONE_P9_19 0x17C +#define BONE_P9_20 0x178 + +#define BONE_P9_21 0x154 +#define BONE_P9_22 0x150 +#define BONE_P9_23 0x044 +#define BONE_P9_24 0x184 + +#define BONE_P9_25 0x1AC +#define BONE_P9_26 0x180 +#define BONE_P9_27 0x1A4 +#define BONE_P9_28 0x19C + +#define BONE_P9_29 0x194 +#define BONE_P9_30 0x198 +#define BONE_P9_31 0x190 + +/* Shared P21 of P11 */ +#define BONE_P9_41A 0x1B4 +#define BONE_P9_41B 0x1A8 + +/* Shared P22 of P11 */ +#define BONE_P9_42A 0x164 +#define BONE_P9_42B 0x1A0 + +#endif diff --git a/include/dt-bindings/interrupt-controller/arm-gic.h b/include/dt-bindings/interrupt-controller/arm-gic.h index 0c85f65c81c7d..35b6f69b7db6a 100644 --- a/include/dt-bindings/interrupt-controller/arm-gic.h +++ b/include/dt-bindings/interrupt-controller/arm-gic.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /* * This header provides constants for the ARM GIC. */ diff --git a/include/dt-bindings/interrupt-controller/irq.h b/include/dt-bindings/interrupt-controller/irq.h index a8b310555f14a..9e3d183e13819 100644 --- a/include/dt-bindings/interrupt-controller/irq.h +++ b/include/dt-bindings/interrupt-controller/irq.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /* * This header provides constants for most IRQ bindings. * diff --git a/include/dt-bindings/media/omap3-isp.h b/include/dt-bindings/media/omap3-isp.h index b18c60e468c78..436c71210e656 100644 --- a/include/dt-bindings/media/omap3-isp.h +++ b/include/dt-bindings/media/omap3-isp.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * include/dt-bindings/media/omap3-isp.h * * Copyright (C) 2015 Sakari Ailus - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #ifndef __DT_BINDINGS_OMAP3_ISP_H__ diff --git a/include/dt-bindings/pinctrl/am33xx.h b/include/dt-bindings/pinctrl/am33xx.h index 7d947a5972209..17877e85980b2 100644 --- a/include/dt-bindings/pinctrl/am33xx.h +++ b/include/dt-bindings/pinctrl/am33xx.h @@ -40,5 +40,133 @@ #undef PIN_OFF_INPUT_PULLDOWN #undef PIN_OFF_WAKEUPENABLE -#endif +#define AM335X_PIN_OFFSET_MIN 0x0800U + +#define AM335X_PIN_GPMC_AD0 0x800 +#define AM335X_PIN_GPMC_AD1 0x804 +#define AM335X_PIN_GPMC_AD2 0x808 +#define AM335X_PIN_GPMC_AD3 0x80c +#define AM335X_PIN_GPMC_AD4 0x810 +#define AM335X_PIN_GPMC_AD5 0x814 +#define AM335X_PIN_GPMC_AD6 0x818 +#define AM335X_PIN_GPMC_AD7 0x81c +#define AM335X_PIN_GPMC_AD8 0x820 +#define AM335X_PIN_GPMC_AD9 0x824 +#define AM335X_PIN_GPMC_AD10 0x828 +#define AM335X_PIN_GPMC_AD11 0x82c +#define AM335X_PIN_GPMC_AD12 0x830 +#define AM335X_PIN_GPMC_AD13 0x834 +#define AM335X_PIN_GPMC_AD14 0x838 +#define AM335X_PIN_GPMC_AD15 0x83c +#define AM335X_PIN_GPMC_A0 0x840 +#define AM335X_PIN_GPMC_A1 0x844 +#define AM335X_PIN_GPMC_A2 0x848 +#define AM335X_PIN_GPMC_A3 0x84c +#define AM335X_PIN_GPMC_A4 0x850 +#define AM335X_PIN_GPMC_A5 0x854 +#define AM335X_PIN_GPMC_A6 0x858 +#define AM335X_PIN_GPMC_A7 0x85c +#define AM335X_PIN_GPMC_A8 0x860 +#define AM335X_PIN_GPMC_A9 0x864 +#define AM335X_PIN_GPMC_A10 0x868 +#define AM335X_PIN_GPMC_A11 0x86c +#define AM335X_PIN_GPMC_WAIT0 0x870 +#define AM335X_PIN_GPMC_WPN 0x874 +#define AM335X_PIN_GPMC_BEN1 0x878 +#define AM335X_PIN_GPMC_CSN0 0x87c +#define AM335X_PIN_GPMC_CSN1 0x880 +#define AM335X_PIN_GPMC_CSN2 0x884 +#define AM335X_PIN_GPMC_CSN3 0x888 +#define AM335X_PIN_GPMC_CLK 0x88c +#define AM335X_PIN_GPMC_ADVN_ALE 0x890 +#define AM335X_PIN_GPMC_OEN_REN 0x894 +#define AM335X_PIN_GPMC_WEN 0x898 +#define AM335X_PIN_GPMC_BEN0_CLE 0x89c +#define AM335X_PIN_LCD_DATA0 0x8a0 +#define AM335X_PIN_LCD_DATA1 0x8a4 +#define AM335X_PIN_LCD_DATA2 0x8a8 +#define AM335X_PIN_LCD_DATA3 0x8ac +#define AM335X_PIN_LCD_DATA4 0x8b0 +#define AM335X_PIN_LCD_DATA5 0x8b4 +#define AM335X_PIN_LCD_DATA6 0x8b8 +#define AM335X_PIN_LCD_DATA7 0x8bc +#define AM335X_PIN_LCD_DATA8 0x8c0 +#define AM335X_PIN_LCD_DATA9 0x8c4 +#define AM335X_PIN_LCD_DATA10 0x8c8 +#define AM335X_PIN_LCD_DATA11 0x8cc +#define AM335X_PIN_LCD_DATA12 0x8d0 +#define AM335X_PIN_LCD_DATA13 0x8d4 +#define AM335X_PIN_LCD_DATA14 0x8d8 +#define AM335X_PIN_LCD_DATA15 0x8dc +#define AM335X_PIN_LCD_VSYNC 0x8e0 +#define AM335X_PIN_LCD_HSYNC 0x8e4 +#define AM335X_PIN_LCD_PCLK 0x8e8 +#define AM335X_PIN_LCD_AC_BIAS_EN 0x8ec +#define AM335X_PIN_MMC0_DAT3 0x8f0 +#define AM335X_PIN_MMC0_DAT2 0x8f4 +#define AM335X_PIN_MMC0_DAT1 0x8f8 +#define AM335X_PIN_MMC0_DAT0 0x8fc +#define AM335X_PIN_MMC0_CLK 0x900 +#define AM335X_PIN_MMC0_CMD 0x904 +#define AM335X_PIN_MII1_COL 0x908 +#define AM335X_PIN_MII1_CRS 0x90c +#define AM335X_PIN_MII1_RX_ER 0x910 +#define AM335X_PIN_MII1_TX_EN 0x914 +#define AM335X_PIN_MII1_RX_DV 0x918 +#define AM335X_PIN_MII1_TXD3 0x91c +#define AM335X_PIN_MII1_TXD2 0x920 +#define AM335X_PIN_MII1_TXD1 0x924 +#define AM335X_PIN_MII1_TXD0 0x928 +#define AM335X_PIN_MII1_TX_CLK 0x92c +#define AM335X_PIN_MII1_RX_CLK 0x930 +#define AM335X_PIN_MII1_RXD3 0x934 +#define AM335X_PIN_MII1_RXD2 0x938 +#define AM335X_PIN_MII1_RXD1 0x93c +#define AM335X_PIN_MII1_RXD0 0x940 +#define AM335X_PIN_RMII1_REF_CLK 0x944 +#define AM335X_PIN_MDIO 0x948 +#define AM335X_PIN_MDC 0x94c +#define AM335X_PIN_SPI0_SCLK 0x950 +#define AM335X_PIN_SPI0_D0 0x954 +#define AM335X_PIN_SPI0_D1 0x958 +#define AM335X_PIN_SPI0_CS0 0x95c +#define AM335X_PIN_SPI0_CS1 0x960 +#define AM335X_PIN_ECAP0_IN_PWM0_OUT 0x964 +#define AM335X_PIN_UART0_CTSN 0x968 +#define AM335X_PIN_UART0_RTSN 0x96c +#define AM335X_PIN_UART0_RXD 0x970 +#define AM335X_PIN_UART0_TXD 0x974 +#define AM335X_PIN_UART1_CTSN 0x978 +#define AM335X_PIN_UART1_RTSN 0x97c +#define AM335X_PIN_UART1_RXD 0x980 +#define AM335X_PIN_UART1_TXD 0x984 +#define AM335X_PIN_I2C0_SDA 0x988 +#define AM335X_PIN_I2C0_SCL 0x98c +#define AM335X_PIN_MCASP0_ACLKX 0x990 +#define AM335X_PIN_MCASP0_FSX 0x994 +#define AM335X_PIN_MCASP0_AXR0 0x998 +#define AM335X_PIN_MCASP0_AHCLKR 0x99c +#define AM335X_PIN_MCASP0_ACLKR 0x9a0 +#define AM335X_PIN_MCASP0_FSR 0x9a4 +#define AM335X_PIN_MCASP0_AXR1 0x9a8 +#define AM335X_PIN_MCASP0_AHCLKX 0x9ac +#define AM335X_PIN_XDMA_EVENT_INTR0 0x9b0 +#define AM335X_PIN_XDMA_EVENT_INTR1 0x9b4 +#define AM335X_PIN_WARMRSTN 0x9b8 +#define AM335X_PIN_NNMI 0x9c0 +#define AM335X_PIN_TMS 0x9d0 +#define AM335X_PIN_TDI 0x9d4 +#define AM335X_PIN_TDO 0x9d8 +#define AM335X_PIN_TCK 0x9dc +#define AM335X_PIN_TRSTN 0x9e0 +#define AM335X_PIN_EMU0 0x9e4 +#define AM335X_PIN_EMU1 0x9e8 +#define AM335X_PIN_RTC_PWRONRSTN 0x9f8 +#define AM335X_PIN_PMIC_POWER_EN 0x9fc +#define AM335X_PIN_EXT_WAKEUP 0xa00 +#define AM335X_PIN_USB0_DRVVBUS 0xa1c +#define AM335X_PIN_USB1_DRVVBUS 0xa34 +#define AM335X_PIN_OFFSET_MAX 0x0a34U + +#endif diff --git a/include/dt-bindings/pinctrl/dra.h b/include/dt-bindings/pinctrl/dra.h index 29423ea8fcc76..e6a5e53ba363c 100644 --- a/include/dt-bindings/pinctrl/dra.h +++ b/include/dt-bindings/pinctrl/dra.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for DRA pinctrl bindings. * * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/ * Author: Rajendra Nayak - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _DT_BINDINGS_PINCTRL_DRA_H diff --git a/include/dt-bindings/pinctrl/omap.h b/include/dt-bindings/pinctrl/omap.h index 49b5dea2b3880..6257180424132 100644 --- a/include/dt-bindings/pinctrl/omap.h +++ b/include/dt-bindings/pinctrl/omap.h @@ -65,6 +65,7 @@ #define DM814X_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) #define DM816X_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) #define AM33XX_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) +#define AM33XX_PADCONF(pa, dir, mux) OMAP_IOPAD_OFFSET((pa), 0x0800) ((dir) | (mux)) /* * Macros to allow using the offset from the padconf physical address diff --git a/include/dt-bindings/thermal/thermal.h b/include/dt-bindings/thermal/thermal.h index b5e6b0069ac77..bc7babb1a67cd 100644 --- a/include/dt-bindings/thermal/thermal.h +++ b/include/dt-bindings/thermal/thermal.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for most thermal bindings. * * Copyright (C) 2013 Texas Instruments * Eduardo Valentin - * - * GPLv2 only */ #ifndef _DT_BINDINGS_THERMAL_THERMAL_H diff --git a/include/linux/file.h b/include/linux/file.h index 279720db984af..76e38eade2255 100644 --- a/include/linux/file.h +++ b/include/linux/file.h @@ -20,6 +20,7 @@ struct dentry; struct path; extern struct file *alloc_file(const struct path *, fmode_t mode, const struct file_operations *fop); +extern struct file *get_empty_filp(void); static inline void fput_light(struct file *file, int fput_needed) { diff --git a/include/linux/fs.h b/include/linux/fs.h index f6a577edec67f..f2d7cf8c9a850 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1267,6 +1267,7 @@ extern void fasync_free(struct fasync_struct *); /* can be called from interrupts */ extern void kill_fasync(struct fasync_struct **, int, int); +extern int setfl(int fd, struct file * filp, unsigned long arg); extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force); extern int f_setown(struct file *filp, unsigned long arg, int force); extern void f_delown(struct file *filp); @@ -1714,6 +1715,7 @@ struct file_operations { ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); + int (*setfl)(struct file *, unsigned long); int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); @@ -1784,6 +1786,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, struct iovec *fast_pointer, struct iovec **ret_pointer); +typedef ssize_t (*vfs_readf_t)(struct file *, char __user *, size_t, loff_t *); +typedef ssize_t (*vfs_writef_t)(struct file *, const char __user *, size_t, + loff_t *); +vfs_readf_t vfs_readf(struct file *file); +vfs_writef_t vfs_writef(struct file *file); + extern ssize_t __vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *); @@ -2188,6 +2196,7 @@ extern int current_umask(void); extern void ihold(struct inode * inode); extern void iput(struct inode *); extern int generic_update_time(struct inode *, struct timespec *, int); +extern int update_time(struct inode *, struct timespec *, int); /* /sys/fs */ extern struct kobject *fs_kobj; @@ -2468,6 +2477,7 @@ static inline bool sb_is_blkdev_sb(struct super_block *sb) return false; } #endif +extern int __sync_filesystem(struct super_block *, int); extern int sync_filesystem(struct super_block *); extern const struct file_operations def_blk_fops; extern const struct file_operations def_chr_fops; diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 55a604ad459f6..83b3446a9e87e 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2112,6 +2112,7 @@ enum ieee80211_key_len { #define FILS_ERP_MAX_RRK_LEN 64 #define PMK_MAX_LEN 48 +#define SAE_PASSWORD_MAX_LEN 128 /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */ enum ieee80211_pub_actioncode { diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index f301d31b473c7..c26f5b4204642 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -406,6 +406,8 @@ static inline int lockdep_match_key(struct lockdep_map *lock, return lock->key == key; } +struct lock_class *lockdep_hlock_class(struct held_lock *hlock); + /* * Acquire a lock. * @@ -530,6 +532,7 @@ struct lock_class_key { }; #define lockdep_depth(tsk) (0) +#define lockdep_is_held(lock) (1) #define lockdep_is_held_type(l, r) (1) #define lockdep_assert_held(l) do { (void)(l); } while (0) diff --git a/include/linux/mfd/stmpe.h b/include/linux/mfd/stmpe.h index 4a827af17e598..87e29d561e223 100644 --- a/include/linux/mfd/stmpe.h +++ b/include/linux/mfd/stmpe.h @@ -1,7 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) ST-Ericsson SA 2010 * - * License Terms: GNU General Public License, version 2 * Author: Rabin Vincent for ST-Ericsson */ @@ -10,6 +10,20 @@ #include +#define STMPE_SAMPLE_TIME(x) ((x & 0xf) << 4) +#define STMPE_MOD_12B(x) ((x & 0x1) << 3) +#define STMPE_REF_SEL(x) ((x & 0x1) << 1) +#define STMPE_ADC_FREQ(x) (x & 0x3) +#define STMPE_AVE_CTRL(x) ((x & 0x3) << 6) +#define STMPE_DET_DELAY(x) ((x & 0x7) << 3) +#define STMPE_SETTLING(x) (x & 0x7) +#define STMPE_FRACTION_Z(x) (x & 0x7) +#define STMPE_I_DRIVE(x) (x & 0x1) +#define STMPE_OP_MODE(x) ((x & 0x7) << 1) + +#define STMPE811_REG_ADC_CTRL1 0x20 +#define STMPE811_REG_ADC_CTRL2 0x21 + struct device; struct regulator; @@ -123,6 +137,12 @@ struct stmpe { u8 ier[2]; u8 oldier[2]; struct stmpe_platform_data *pdata; + + /* For devices that use an ADC */ + u8 sample_time; + u8 mod_12b; + u8 ref_sel; + u8 adc_freq; }; extern int stmpe_reg_write(struct stmpe *stmpe, u8 reg, u8 data); @@ -136,6 +156,7 @@ extern int stmpe_set_altfunc(struct stmpe *stmpe, u32 pins, enum stmpe_block block); extern int stmpe_enable(struct stmpe *stmpe, unsigned int blocks); extern int stmpe_disable(struct stmpe *stmpe, unsigned int blocks); +extern int stmpe811_adc_common_init(struct stmpe *stmpe); #define STMPE_GPIO_NOREQ_811_TOUCH (0xf0) diff --git a/include/linux/mm.h b/include/linux/mm.h index 58f2263de4de5..e8b2d404c974b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1350,6 +1350,28 @@ static inline int fixup_user_fault(struct task_struct *tsk, } #endif +extern void vma_do_file_update_time(struct vm_area_struct *, const char[], int); +extern struct file *vma_do_pr_or_file(struct vm_area_struct *, const char[], + int); +extern void vma_do_get_file(struct vm_area_struct *, const char[], int); +extern void vma_do_fput(struct vm_area_struct *, const char[], int); + +#define vma_file_update_time(vma) vma_do_file_update_time(vma, __func__, \ + __LINE__) +#define vma_pr_or_file(vma) vma_do_pr_or_file(vma, __func__, \ + __LINE__) +#define vma_get_file(vma) vma_do_get_file(vma, __func__, __LINE__) +#define vma_fput(vma) vma_do_fput(vma, __func__, __LINE__) + +#ifndef CONFIG_MMU +extern struct file *vmr_do_pr_or_file(struct vm_region *, const char[], int); +extern void vmr_do_fput(struct vm_region *, const char[], int); + +#define vmr_pr_or_file(region) vmr_do_pr_or_file(region, __func__, \ + __LINE__) +#define vmr_fput(region) vmr_do_fput(region, __func__, __LINE__) +#endif /* !CONFIG_MMU */ + extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags); extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index e41ef532c4ce0..bf258a4ed32b8 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -253,6 +253,7 @@ struct vm_region { unsigned long vm_top; /* region allocated to here */ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ struct file *vm_file; /* the backing file or NULL */ + struct file *vm_prfile; /* the virtual backing file or NULL */ int vm_usage; /* region usage count (access under nommu_region_sem) */ bool vm_icache_flushed : 1; /* true if the icache has been flushed for @@ -327,6 +328,7 @@ struct vm_area_struct { unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ struct file * vm_file; /* File we map to (can be NULL). */ + struct file *vm_prfile; /* shadow of vm_file */ void * vm_private_data; /* was vm_pte (shared mem) */ atomic_long_t swap_readahead_info; diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 0a7abe8a407ff..a447df5bffba5 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -41,7 +41,10 @@ #define SDIO_DEVICE_ID_BROADCOM_43455 0xa9bf #define SDIO_DEVICE_ID_BROADCOM_4354 0x4354 #define SDIO_DEVICE_ID_BROADCOM_4356 0x4356 +#define SDIO_DEVICE_ID_BROADCOM_4359 0x4355 #define SDIO_DEVICE_ID_CYPRESS_4373 0x4373 +#define SDIO_DEVICE_ID_CYPRESS_43012 43012 +#define SDIO_DEVICE_ID_CYPRESS_89359 0x4359 #define SDIO_VENDOR_ID_INTEL 0x0089 #define SDIO_DEVICE_ID_INTEL_IWMC3200WIMAX 0x1402 diff --git a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h index 35942084cd40d..24f5fd1a789de 100644 --- a/include/linux/mnt_namespace.h +++ b/include/linux/mnt_namespace.h @@ -6,11 +6,14 @@ struct mnt_namespace; struct fs_struct; struct user_namespace; +struct vfsmount; extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *, struct user_namespace *, struct fs_struct *); extern void put_mnt_ns(struct mnt_namespace *ns); +extern int is_current_mnt_ns(struct vfsmount *mnt); + extern const struct file_operations proc_mounts_operations; extern const struct file_operations proc_mountinfo_operations; extern const struct file_operations proc_mountstats_operations; diff --git a/include/linux/of.h b/include/linux/of.h index 70b7dacf92387..c6c2271866863 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -52,6 +54,7 @@ struct device_node { phandle phandle; const char *full_name; struct fwnode_handle fwnode; + struct rhash_head ht_node; struct property *properties; struct property *deadprops; /* removed properties */ @@ -1227,6 +1230,8 @@ enum of_reconfig_change { }; #ifdef CONFIG_OF_DYNAMIC +#include + extern int of_reconfig_notifier_register(struct notifier_block *); extern int of_reconfig_notifier_unregister(struct notifier_block *); extern int of_reconfig_notify(unsigned long, struct of_reconfig_data *rd); @@ -1270,6 +1275,26 @@ static inline int of_changeset_update_property(struct of_changeset *ocs, { return of_changeset_action(ocs, OF_RECONFIG_UPDATE_PROPERTY, np, prop); } + +struct device_node *of_changeset_create_device_nodev( + struct of_changeset *ocs, struct device_node *parent, + const char *fmt, va_list vargs); + +__printf(3, 4) struct device_node * +of_changeset_create_device_node(struct of_changeset *ocs, + struct device_node *parent, const char *fmt, ...); + +int __of_changeset_add_update_property_copy(struct of_changeset *ocs, + struct device_node *np, const char *name, const void *value, + int length, bool update); + +int __of_changeset_add_update_property_string_list( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char **strs, int count, bool update); + +int of_changeset_node_move(struct of_changeset *ocs, + struct device_node *np, struct device_node *new_parent); + #else /* CONFIG_OF_DYNAMIC */ static inline int of_reconfig_notifier_register(struct notifier_block *nb) { @@ -1289,8 +1314,323 @@ static inline int of_reconfig_get_state_change(unsigned long action, { return -EINVAL; } + +static inline struct device_node *of_changeset_create_device_nodev( + struct of_changeset *ocs, struct device_node *parent, + const char *fmt, va_list vargs) +{ + return ERR_PTR(-EINVAL); +} + +static inline __printf(3, 4) struct device_node * +of_changeset_create_device_node(struct of_changeset *ocs, + struct device_node *parent, const char *fmt, ...) +{ + return ERR_PTR(-EINVAL); +} + +static inline int __of_changeset_add_update_property_copy( + struct of_changeset *ocs, struct device_node *np, + const char *name, const void *value, int length, bool update) +{ + return -EINVAL; +} + +static inline __printf(4, 5) int of_changeset_add_property_stringf( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *fmt, ...) +{ + return -EINVAL; +} + +static inline int of_changeset_update_property_stringf( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *fmt, ...) +{ + return -EINVAL; +} + +static inline int __of_changeset_add_update_property_string_list( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char **strs, int count, bool update) +{ + return -EINVAL; +} + +static inline int of_changeset_node_move(struct of_changeset *ocs, + struct device_node *np, struct device_node *new_parent) +{ + return -EINVAL; +} + #endif /* CONFIG_OF_DYNAMIC */ +/** + * of_changeset_add_property_copy - Create a new property copying name & value + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @value: pointer to the value data + * @length: length of the value in bytes + * + * Adds a property to the changeset by making copies of the name & value + * entries. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_add_property_copy(struct of_changeset *ocs, + struct device_node *np, const char *name, + const void *value, int length) +{ + return __of_changeset_add_update_property_copy(ocs, np, name, value, + length, false); +} + +/** + * of_changeset_update_property_copy - Update a property copying name & value + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @value: pointer to the value data + * @length: length of the value in bytes + * + * Update a property to the changeset by making copies of the name & value + * entries. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_update_property_copy(struct of_changeset *ocs, + struct device_node *np, const char *name, + const void *value, int length) +{ + return __of_changeset_add_update_property_copy(ocs, np, name, value, + length, true); +} + +/** + * __of_changeset_add_update_property_string - Create/update a string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @str: string property value + * @update: True on update operation + * + * Adds/updates a string property to the changeset by making copies of the name + * and the given value. The @update parameter controls whether an add or + * update takes place. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int __of_changeset_add_update_property_string( + struct of_changeset *ocs, struct device_node *np, const char *name, + const char *str, bool update) +{ + return __of_changeset_add_update_property_copy(ocs, np, name, str, + strlen(str) + 1, update); +} + +/** + * __of_changeset_add_update_property_stringv - Create/update a formatted + * string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @fmt: format of string property + * @vargs: arguments of the format string + * @update: True on update operation + * + * Adds/updates a string property to the changeset by making copies of the name + * and the formatted value. The @update parameter controls whether an add or + * update takes place. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int __of_changeset_add_update_property_stringv( + struct of_changeset *ocs, struct device_node *np, const char *name, + const char *fmt, va_list vargs, bool update) +{ + char *str; + int ret; + + str = kvasprintf(GFP_KERNEL, fmt, vargs); + if (!str) + return -ENOMEM; + ret = __of_changeset_add_update_property_string(ocs, np, name, str, + update); + kfree(str); + + return ret; +} + +/** + * of_changeset_add_property_string_list - Create a new string list property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @strs: pointer to the string list + * @count: string count + * + * Adds a string list property to the changeset. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_add_property_string_list( + struct of_changeset *ocs, struct device_node *np, const char *name, + const char **strs, int count) +{ + return __of_changeset_add_update_property_string_list(ocs, np, name, + strs, count, false); +} + +/** + * of_changeset_update_property_string_list - Update string list property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @strs: pointer to the string list + * @count: string count + * + * Updates a string list property to the changeset. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_update_property_string_list( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char **strs, int count) +{ + return __of_changeset_add_update_property_string_list(ocs, np, name, + strs, count, true); +} + +/** + * of_changeset_add_property_string - Adds a string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @str: string property + * + * Adds a string property to the changeset by making copies of the name + * and the string value. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_add_property_string( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *str) +{ + return __of_changeset_add_update_property_string(ocs, np, name, str, + false); +} + +/** + * of_changeset_update_property_string - Update a string property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @str: string property + * + * Updates a string property to the changeset by making copies of the name + * and the string value. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_update_property_string( + struct of_changeset *ocs, struct device_node *np, + const char *name, const char *str) +{ + return __of_changeset_add_update_property_string(ocs, np, name, str, + true); +} + +/** + * of_changeset_add_property_u32 - Create a new u32 property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @val: value in host endian format + * + * Adds a u32 property to the changeset. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_add_property_u32(struct of_changeset *ocs, + struct device_node *np, const char *name, u32 val) +{ + val = cpu_to_be32(val); + return __of_changeset_add_update_property_copy(ocs, np, name, &val, + sizeof(val), false); +} + +/** + * of_changeset_update_property_u32 - Update u32 property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * @val: value in host endian format + * + * Updates a u32 property to the changeset. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_update_property_u32( + struct of_changeset *ocs, struct device_node *np, + const char *name, u32 val) +{ + val = cpu_to_be32(val); + return __of_changeset_add_update_property_copy(ocs, np, name, &val, + sizeof(val), true); +} + +/** + * of_changeset_add_property_bool - Create a new u32 property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * + * Adds a bool property to the changeset. Note that there is + * no option to set the value to false, since the property + * existing sets it to true. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_add_property_bool( + struct of_changeset *ocs, struct device_node *np, const char *name) +{ + return __of_changeset_add_update_property_copy(ocs, np, name, "", 0, + false); +} + +/** + * of_changeset_update_property_bool - Update a bool property + * + * @ocs: changeset pointer + * @np: device node pointer + * @name: name of the property + * + * Updates a property to the changeset. Note that there is + * no option to set the value to false, since the property + * existing sets it to true. + * + * Returns zero on success, a negative error value otherwise. + */ +static inline int of_changeset_update_property_bool(struct of_changeset *ocs, + struct device_node *np, const char *name) +{ + return __of_changeset_add_update_property_copy(ocs, np, name, "", 0, + true); +} + /* CONFIG_OF_RESOLVE api */ extern int of_resolve_phandles(struct device_node *tree); @@ -1331,6 +1671,10 @@ int of_overlay_destroy_all(void); int of_overlay_notifier_register(struct notifier_block *nb); int of_overlay_notifier_unregister(struct notifier_block *nb); +int of_overlay_create_target_index(struct device_node *tree, int index); +int of_overlay_create_target_root(struct device_node *tree, + struct device_node *target_root); + #else static inline int of_overlay_create(struct device_node *tree) @@ -1358,6 +1702,18 @@ static inline int of_overlay_notifier_unregister(struct notifier_block *nb) return 0; } +static inline int of_overlay_create_target_index(struct device_node *tree, + int index) +{ + return -ENOTSUPP; +} + +static inline int of_overlay_create_target_root(struct device_node *tree, + struct device_node *target_root) +{ + return -ENOTSUPP; +} + #endif #endif /* _LINUX_OF_H */ diff --git a/include/linux/splice.h b/include/linux/splice.h index 74b4911ac16dd..19789fbea5675 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -87,4 +87,10 @@ extern void splice_shrink_spd(struct splice_pipe_desc *); extern const struct pipe_buf_operations page_cache_pipe_buf_ops; extern const struct pipe_buf_operations default_pipe_buf_ops; + +extern long do_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags); +extern long do_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); #endif diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index ea0ed58db97e2..0a74f7f168d9f 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -650,6 +650,9 @@ struct survey_info { * CFG80211_MAX_WEP_KEYS WEP keys * @wep_tx_key: key index (0..3) of the default TX static WEP key * @psk: PSK (for devices supporting 4-way-handshake offload) + * @sae_pwd: password for SAE authentication (for devices supporting SAE + * offload) + * @sae_pwd_len: length of SAE password (for devices supporting SAE offload) */ struct cfg80211_crypto_settings { u32 wpa_versions; @@ -664,6 +667,8 @@ struct cfg80211_crypto_settings { struct key_params *wep_keys; int wep_tx_key; const u8 *psk; + const u8 *sae_pwd; + u8 sae_pwd_len; }; /** @@ -5282,6 +5287,8 @@ static inline void cfg80211_testmode_event(struct sk_buff *skb, gfp_t gfp) * not known. This value is used only if @status < 0 to indicate that the * failure is due to a timeout and not due to explicit rejection by the AP. * This value is ignored in other cases (@status >= 0). + * @authorized: Indicates whether the connection is ready to transport + * data packets. */ struct cfg80211_connect_resp_params { int status; @@ -5299,6 +5306,7 @@ struct cfg80211_connect_resp_params { size_t pmk_len; const u8 *pmkid; enum nl80211_timeout_reason timeout_reason; + bool authorized; }; /** @@ -5479,6 +5487,23 @@ struct cfg80211_roam_info { void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info, gfp_t gfp); +/** + * cfg80211_port_authorized - notify cfg80211 of successful security association + * + * @dev: network device + * @bssid: the BSSID of the AP + * @gfp: allocation flags + * + * This function should be called by a driver that supports 4 way handshake + * offload after a security association was successfully established (i.e., + * the 4 way handshake was completed successfully). The call to this function + * should be preceded with a call to cfg80211_connect_result(), + * cfg80211_connect_done(), cfg80211_connect_bss() or cfg80211_roamed() to + * indicate the 802.11 association. + */ +void cfg80211_port_authorized(struct net_device *dev, const u8 *bssid, + gfp_t gfp); + /** * cfg80211_disconnected - notify cfg80211 that connection was dropped * diff --git a/include/uapi/linux/aufs_type.h b/include/uapi/linux/aufs_type.h new file mode 100644 index 0000000000000..d8e39df984330 --- /dev/null +++ b/include/uapi/linux/aufs_type.h @@ -0,0 +1,452 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2019 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __AUFS_TYPE_H__ +#define __AUFS_TYPE_H__ + +#define AUFS_NAME "aufs" + +#ifdef __KERNEL__ +/* + * define it before including all other headers. + * sched.h may use pr_* macros before defining "current", so define the + * no-current version first, and re-define later. + */ +#define pr_fmt(fmt) AUFS_NAME " %s:%d: " fmt, __func__, __LINE__ +#include +#undef pr_fmt +#define pr_fmt(fmt) \ + AUFS_NAME " %s:%d:%.*s[%d]: " fmt, __func__, __LINE__, \ + (int)sizeof(current->comm), current->comm, current->pid +#else +#include +#include +#endif /* __KERNEL__ */ + +#include + +#define AUFS_VERSION "4.14.73+-20190902" + +/* todo? move this to linux-2.6.19/include/magic.h */ +#define AUFS_SUPER_MAGIC ('a' << 24 | 'u' << 16 | 'f' << 8 | 's') + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_BRANCH_MAX_127 +typedef int8_t aufs_bindex_t; +#define AUFS_BRANCH_MAX 127 +#else +typedef int16_t aufs_bindex_t; +#ifdef CONFIG_AUFS_BRANCH_MAX_511 +#define AUFS_BRANCH_MAX 511 +#elif defined(CONFIG_AUFS_BRANCH_MAX_1023) +#define AUFS_BRANCH_MAX 1023 +#elif defined(CONFIG_AUFS_BRANCH_MAX_32767) +#define AUFS_BRANCH_MAX 32767 +#endif +#endif + +#ifdef __KERNEL__ +#ifndef AUFS_BRANCH_MAX +#error unknown CONFIG_AUFS_BRANCH_MAX value +#endif +#endif /* __KERNEL__ */ + +/* ---------------------------------------------------------------------- */ + +#define AUFS_FSTYPE AUFS_NAME + +#define AUFS_ROOT_INO 2 +#define AUFS_FIRST_INO 11 + +#define AUFS_WH_PFX ".wh." +#define AUFS_WH_PFX_LEN ((int)sizeof(AUFS_WH_PFX) - 1) +#define AUFS_WH_TMP_LEN 4 +/* a limit for rmdir/rename a dir and copyup */ +#define AUFS_MAX_NAMELEN (NAME_MAX \ + - AUFS_WH_PFX_LEN * 2 /* doubly whiteouted */\ + - 1 /* dot */\ + - AUFS_WH_TMP_LEN) /* hex */ +#define AUFS_XINO_FNAME "." AUFS_NAME ".xino" +#define AUFS_XINO_DEFPATH "/tmp/" AUFS_XINO_FNAME +#define AUFS_XINO_DEF_SEC 30 /* seconds */ +#define AUFS_XINO_DEF_TRUNC 45 /* percentage */ +#define AUFS_DIRWH_DEF 3 +#define AUFS_RDCACHE_DEF 10 /* seconds */ +#define AUFS_RDCACHE_MAX 3600 /* seconds */ +#define AUFS_RDBLK_DEF 512 /* bytes */ +#define AUFS_RDHASH_DEF 32 +#define AUFS_WKQ_NAME AUFS_NAME "d" +#define AUFS_MFS_DEF_SEC 30 /* seconds */ +#define AUFS_MFS_MAX_SEC 3600 /* seconds */ +#define AUFS_FHSM_CACHE_DEF_SEC 30 /* seconds */ +#define AUFS_PLINK_WARN 50 /* number of plinks in a single bucket */ + +/* pseudo-link maintenace under /proc */ +#define AUFS_PLINK_MAINT_NAME "plink_maint" +#define AUFS_PLINK_MAINT_DIR "fs/" AUFS_NAME +#define AUFS_PLINK_MAINT_PATH AUFS_PLINK_MAINT_DIR "/" AUFS_PLINK_MAINT_NAME + +/* dirren, renamed dir */ +#define AUFS_DR_INFO_PFX AUFS_WH_PFX ".dr." +#define AUFS_DR_BRHINO_NAME AUFS_WH_PFX "hino" +/* whiteouted doubly */ +#define AUFS_WH_DR_INFO_PFX AUFS_WH_PFX AUFS_DR_INFO_PFX +#define AUFS_WH_DR_BRHINO AUFS_WH_PFX AUFS_DR_BRHINO_NAME + +#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq" /* whiteouted doubly */ +#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME + +#define AUFS_BASE_NAME AUFS_WH_PFX AUFS_NAME +#define AUFS_PLINKDIR_NAME AUFS_WH_PFX "plnk" +#define AUFS_ORPHDIR_NAME AUFS_WH_PFX "orph" + +/* doubly whiteouted */ +#define AUFS_WH_BASE AUFS_WH_PFX AUFS_BASE_NAME +#define AUFS_WH_PLINKDIR AUFS_WH_PFX AUFS_PLINKDIR_NAME +#define AUFS_WH_ORPHDIR AUFS_WH_PFX AUFS_ORPHDIR_NAME + +/* branch permissions and attributes */ +#define AUFS_BRPERM_RW "rw" +#define AUFS_BRPERM_RO "ro" +#define AUFS_BRPERM_RR "rr" +#define AUFS_BRATTR_COO_REG "coo_reg" +#define AUFS_BRATTR_COO_ALL "coo_all" +#define AUFS_BRATTR_FHSM "fhsm" +#define AUFS_BRATTR_UNPIN "unpin" +#define AUFS_BRATTR_ICEX "icex" +#define AUFS_BRATTR_ICEX_SEC "icexsec" +#define AUFS_BRATTR_ICEX_SYS "icexsys" +#define AUFS_BRATTR_ICEX_TR "icextr" +#define AUFS_BRATTR_ICEX_USR "icexusr" +#define AUFS_BRATTR_ICEX_OTH "icexoth" +#define AUFS_BRRATTR_WH "wh" +#define AUFS_BRWATTR_NLWH "nolwh" +#define AUFS_BRWATTR_MOO "moo" + +#define AuBrPerm_RW 1 /* writable, hardlinkable wh */ +#define AuBrPerm_RO (1 << 1) /* readonly */ +#define AuBrPerm_RR (1 << 2) /* natively readonly */ +#define AuBrPerm_Mask (AuBrPerm_RW | AuBrPerm_RO | AuBrPerm_RR) + +#define AuBrAttr_COO_REG (1 << 3) /* copy-up on open */ +#define AuBrAttr_COO_ALL (1 << 4) +#define AuBrAttr_COO_Mask (AuBrAttr_COO_REG | AuBrAttr_COO_ALL) + +#define AuBrAttr_FHSM (1 << 5) /* file-based hsm */ +#define AuBrAttr_UNPIN (1 << 6) /* rename-able top dir of + branch. meaningless since + linux-3.18-rc1 */ + +/* ignore error in copying XATTR */ +#define AuBrAttr_ICEX_SEC (1 << 7) +#define AuBrAttr_ICEX_SYS (1 << 8) +#define AuBrAttr_ICEX_TR (1 << 9) +#define AuBrAttr_ICEX_USR (1 << 10) +#define AuBrAttr_ICEX_OTH (1 << 11) +#define AuBrAttr_ICEX (AuBrAttr_ICEX_SEC \ + | AuBrAttr_ICEX_SYS \ + | AuBrAttr_ICEX_TR \ + | AuBrAttr_ICEX_USR \ + | AuBrAttr_ICEX_OTH) + +#define AuBrRAttr_WH (1 << 12) /* whiteout-able */ +#define AuBrRAttr_Mask AuBrRAttr_WH + +#define AuBrWAttr_NoLinkWH (1 << 13) /* un-hardlinkable whiteouts */ +#define AuBrWAttr_MOO (1 << 14) /* move-up on open */ +#define AuBrWAttr_Mask (AuBrWAttr_NoLinkWH | AuBrWAttr_MOO) + +#define AuBrAttr_CMOO_Mask (AuBrAttr_COO_Mask | AuBrWAttr_MOO) + +/* #warning test userspace */ +#ifdef __KERNEL__ +#ifndef CONFIG_AUFS_FHSM +#undef AuBrAttr_FHSM +#define AuBrAttr_FHSM 0 +#endif +#ifndef CONFIG_AUFS_XATTR +#undef AuBrAttr_ICEX +#define AuBrAttr_ICEX 0 +#undef AuBrAttr_ICEX_SEC +#define AuBrAttr_ICEX_SEC 0 +#undef AuBrAttr_ICEX_SYS +#define AuBrAttr_ICEX_SYS 0 +#undef AuBrAttr_ICEX_TR +#define AuBrAttr_ICEX_TR 0 +#undef AuBrAttr_ICEX_USR +#define AuBrAttr_ICEX_USR 0 +#undef AuBrAttr_ICEX_OTH +#define AuBrAttr_ICEX_OTH 0 +#endif +#endif + +/* the longest combination */ +/* AUFS_BRATTR_ICEX and AUFS_BRATTR_ICEX_TR don't affect here */ +#define AuBrPermStrSz sizeof(AUFS_BRPERM_RW \ + "+" AUFS_BRATTR_COO_REG \ + "+" AUFS_BRATTR_FHSM \ + "+" AUFS_BRATTR_UNPIN \ + "+" AUFS_BRATTR_ICEX_SEC \ + "+" AUFS_BRATTR_ICEX_SYS \ + "+" AUFS_BRATTR_ICEX_USR \ + "+" AUFS_BRATTR_ICEX_OTH \ + "+" AUFS_BRWATTR_NLWH) + +typedef struct { + char a[AuBrPermStrSz]; +} au_br_perm_str_t; + +static inline int au_br_writable(int brperm) +{ + return brperm & AuBrPerm_RW; +} + +static inline int au_br_whable(int brperm) +{ + return brperm & (AuBrPerm_RW | AuBrRAttr_WH); +} + +static inline int au_br_wh_linkable(int brperm) +{ + return !(brperm & AuBrWAttr_NoLinkWH); +} + +static inline int au_br_cmoo(int brperm) +{ + return brperm & AuBrAttr_CMOO_Mask; +} + +static inline int au_br_fhsm(int brperm) +{ + return brperm & AuBrAttr_FHSM; +} + +/* ---------------------------------------------------------------------- */ + +/* ioctl */ +enum { + /* readdir in userspace */ + AuCtl_RDU, + AuCtl_RDU_INO, + + AuCtl_WBR_FD, /* pathconf wrapper */ + AuCtl_IBUSY, /* busy inode */ + AuCtl_MVDOWN, /* move-down */ + AuCtl_BR, /* info about branches */ + AuCtl_FHSM_FD /* connection for fhsm */ +}; + +/* borrowed from linux/include/linux/kernel.h */ +#ifndef ALIGN +#ifdef _GNU_SOURCE +#define ALIGN(x, a) __ALIGN_MASK(x, (typeof(x))(a)-1) +#else +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#endif +#define __ALIGN_MASK(x, mask) (((x)+(mask))&~(mask)) +#endif + +/* borrowed from linux/include/linux/compiler-gcc3.h */ +#ifndef __aligned +#define __aligned(x) __attribute__((aligned(x))) +#endif + +#ifdef __KERNEL__ +#ifndef __packed +#define __packed __attribute__((packed)) +#endif +#endif + +struct au_rdu_cookie { + uint64_t h_pos; + int16_t bindex; + uint8_t flags; + uint8_t pad; + uint32_t generation; +} __aligned(8); + +struct au_rdu_ent { + uint64_t ino; + int16_t bindex; + uint8_t type; + uint8_t nlen; + uint8_t wh; + char name[0]; +} __aligned(8); + +static inline int au_rdu_len(int nlen) +{ + /* include the terminating NULL */ + return ALIGN(sizeof(struct au_rdu_ent) + nlen + 1, + sizeof(uint64_t)); +} + +union au_rdu_ent_ul { + struct au_rdu_ent __user *e; + uint64_t ul; +}; + +enum { + AufsCtlRduV_SZ, + AufsCtlRduV_End +}; + +struct aufs_rdu { + /* input */ + union { + uint64_t sz; /* AuCtl_RDU */ + uint64_t nent; /* AuCtl_RDU_INO */ + }; + union au_rdu_ent_ul ent; + uint16_t verify[AufsCtlRduV_End]; + + /* input/output */ + uint32_t blk; + + /* output */ + union au_rdu_ent_ul tail; + /* number of entries which were added in a single call */ + uint64_t rent; + uint8_t full; + uint8_t shwh; + + struct au_rdu_cookie cookie; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +/* dirren. the branch is identified by the filename who contains this */ +struct au_drinfo { + uint64_t ino; + union { + uint8_t oldnamelen; + uint64_t _padding; + }; + uint8_t oldname[0]; +} __aligned(8); + +struct au_drinfo_fdata { + uint32_t magic; + struct au_drinfo drinfo; +} __aligned(8); + +#define AUFS_DRINFO_MAGIC_V1 ('a' << 24 | 'd' << 16 | 'r' << 8 | 0x01) +/* future */ +#define AUFS_DRINFO_MAGIC_V2 ('a' << 24 | 'd' << 16 | 'r' << 8 | 0x02) + +/* ---------------------------------------------------------------------- */ + +struct aufs_wbr_fd { + uint32_t oflags; + int16_t brid; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +struct aufs_ibusy { + uint64_t ino, h_ino; + int16_t bindex; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +/* error code for move-down */ +/* the actual message strings are implemented in aufs-util.git */ +enum { + EAU_MVDOWN_OPAQUE = 1, + EAU_MVDOWN_WHITEOUT, + EAU_MVDOWN_UPPER, + EAU_MVDOWN_BOTTOM, + EAU_MVDOWN_NOUPPER, + EAU_MVDOWN_NOLOWERBR, + EAU_Last +}; + +/* flags for move-down */ +#define AUFS_MVDOWN_DMSG 1 +#define AUFS_MVDOWN_OWLOWER (1 << 1) /* overwrite lower */ +#define AUFS_MVDOWN_KUPPER (1 << 2) /* keep upper */ +#define AUFS_MVDOWN_ROLOWER (1 << 3) /* do even if lower is RO */ +#define AUFS_MVDOWN_ROLOWER_R (1 << 4) /* did on lower RO */ +#define AUFS_MVDOWN_ROUPPER (1 << 5) /* do even if upper is RO */ +#define AUFS_MVDOWN_ROUPPER_R (1 << 6) /* did on upper RO */ +#define AUFS_MVDOWN_BRID_UPPER (1 << 7) /* upper brid */ +#define AUFS_MVDOWN_BRID_LOWER (1 << 8) /* lower brid */ +#define AUFS_MVDOWN_FHSM_LOWER (1 << 9) /* find fhsm attr for lower */ +#define AUFS_MVDOWN_STFS (1 << 10) /* req. stfs */ +#define AUFS_MVDOWN_STFS_FAILED (1 << 11) /* output: stfs is unusable */ +#define AUFS_MVDOWN_BOTTOM (1 << 12) /* output: no more lowers */ + +/* index for move-down */ +enum { + AUFS_MVDOWN_UPPER, + AUFS_MVDOWN_LOWER, + AUFS_MVDOWN_NARRAY +}; + +/* + * additional info of move-down + * number of free blocks and inodes. + * subset of struct kstatfs, but smaller and always 64bit. + */ +struct aufs_stfs { + uint64_t f_blocks; + uint64_t f_bavail; + uint64_t f_files; + uint64_t f_ffree; +}; + +struct aufs_stbr { + int16_t brid; /* optional input */ + int16_t bindex; /* output */ + struct aufs_stfs stfs; /* output when AUFS_MVDOWN_STFS set */ +} __aligned(8); + +struct aufs_mvdown { + uint32_t flags; /* input/output */ + struct aufs_stbr stbr[AUFS_MVDOWN_NARRAY]; /* input/output */ + int8_t au_errno; /* output */ +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +union aufs_brinfo { + /* PATH_MAX may differ between kernel-space and user-space */ + char _spacer[4096]; + struct { + int16_t id; + int perm; + char path[0]; + }; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +#define AuCtlType 'A' +#define AUFS_CTL_RDU _IOWR(AuCtlType, AuCtl_RDU, struct aufs_rdu) +#define AUFS_CTL_RDU_INO _IOWR(AuCtlType, AuCtl_RDU_INO, struct aufs_rdu) +#define AUFS_CTL_WBR_FD _IOW(AuCtlType, AuCtl_WBR_FD, \ + struct aufs_wbr_fd) +#define AUFS_CTL_IBUSY _IOWR(AuCtlType, AuCtl_IBUSY, struct aufs_ibusy) +#define AUFS_CTL_MVDOWN _IOWR(AuCtlType, AuCtl_MVDOWN, \ + struct aufs_mvdown) +#define AUFS_CTL_BRINFO _IOW(AuCtlType, AuCtl_BR, union aufs_brinfo) +#define AUFS_CTL_FHSM_FD _IOW(AuCtlType, AuCtl_FHSM_FD, int) + +#endif /* __AUFS_TYPE_H__ */ diff --git a/include/uapi/linux/can/isotp.h b/include/uapi/linux/can/isotp.h new file mode 100644 index 0000000000000..548e732f33a8e --- /dev/null +++ b/include/uapi/linux/can/isotp.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * socketcan/can/isotp.h + * + * Definitions for isotp CAN sockets + * + * Author: Oliver Hartkopp + * Copyright (c) 2008 Volkswagen Group Electronic Research + * All rights reserved. + * + * Send feedback to + * + */ + +#ifndef CAN_ISOTP_H +#define CAN_ISOTP_H + +#include + +#define SOL_CAN_ISOTP (SOL_CAN_BASE + CAN_ISOTP) + +/* for socket options affecting the socket (not the global system) */ + +#define CAN_ISOTP_OPTS 1 /* pass struct can_isotp_options */ + +#define CAN_ISOTP_RECV_FC 2 /* pass struct can_isotp_fc_options */ + +/* sockopts to force stmin timer values for protocol regression tests */ + +#define CAN_ISOTP_TX_STMIN 3 /* pass __u32 value in nano secs */ + /* use this time instead of value */ + /* provided in FC from the receiver */ + +#define CAN_ISOTP_RX_STMIN 4 /* pass __u32 value in nano secs */ + /* ignore received CF frames which */ + /* timestamps differ less than val */ + +#define CAN_ISOTP_LL_OPTS 5 /* pass struct can_isotp_ll_options */ + +struct can_isotp_options { + + __u32 flags; /* set flags for isotp behaviour. */ + /* __u32 value : flags see below */ + + __u32 frame_txtime; /* frame transmission time (N_As/N_Ar) */ + /* __u32 value : time in nano secs */ + + __u8 ext_address; /* set address for extended addressing */ + /* __u8 value : extended address */ + + __u8 txpad_content; /* set content of padding byte (tx) */ + /* __u8 value : content on tx path */ + + __u8 rxpad_content; /* set content of padding byte (rx) */ + /* __u8 value : content on rx path */ + + __u8 rx_ext_address; /* set address for extended addressing */ + /* __u8 value : extended address (rx) */ +}; + +struct can_isotp_fc_options { + + __u8 bs; /* blocksize provided in FC frame */ + /* __u8 value : blocksize. 0 = off */ + + __u8 stmin; /* separation time provided in FC frame */ + /* __u8 value : */ + /* 0x00 - 0x7F : 0 - 127 ms */ + /* 0x80 - 0xF0 : reserved */ + /* 0xF1 - 0xF9 : 100 us - 900 us */ + /* 0xFA - 0xFF : reserved */ + + __u8 wftmax; /* max. number of wait frame transmiss. */ + /* __u8 value : 0 = omit FC N_PDU WT */ +}; + +struct can_isotp_ll_options { + + __u8 mtu; /* generated & accepted CAN frame type */ + /* __u8 value : */ + /* CAN_MTU (16) -> standard CAN 2.0 */ + /* CANFD_MTU (72) -> CAN FD frame */ + + __u8 tx_dl; /* tx link layer data length in bytes */ + /* (configured maximum payload length) */ + /* __u8 value : 8,12,16,20,24,32,48,64 */ + /* => rx path supports all LL_DL values */ + + __u8 tx_flags; /* set into struct canfd_frame.flags */ + /* at frame creation: e.g. CANFD_BRS */ + /* Obsolete when the BRS flag is fixed */ + /* by the CAN netdriver configuration */ +}; + +/* flags for isotp behaviour */ + +#define CAN_ISOTP_LISTEN_MODE 0x001 /* listen only (do not send FC) */ +#define CAN_ISOTP_EXTEND_ADDR 0x002 /* enable extended addressing */ +#define CAN_ISOTP_TX_PADDING 0x004 /* enable CAN frame padding tx path */ +#define CAN_ISOTP_RX_PADDING 0x008 /* enable CAN frame padding rx path */ +#define CAN_ISOTP_CHK_PAD_LEN 0x010 /* check received CAN frame padding */ +#define CAN_ISOTP_CHK_PAD_DATA 0x020 /* check received CAN frame padding */ +#define CAN_ISOTP_HALF_DUPLEX 0x040 /* half duplex error state handling */ +#define CAN_ISOTP_FORCE_TXSTMIN 0x080 /* ignore stmin from received FC */ +#define CAN_ISOTP_FORCE_RXSTMIN 0x100 /* ignore CFs depending on rx stmin */ +#define CAN_ISOTP_RX_EXT_ADDR 0x200 /* different rx extended addressing */ + + +/* default values */ + +#define CAN_ISOTP_DEFAULT_FLAGS 0 +#define CAN_ISOTP_DEFAULT_EXT_ADDRESS 0x00 +#define CAN_ISOTP_DEFAULT_PAD_CONTENT 0xCC /* prevent bit-stuffing */ +#define CAN_ISOTP_DEFAULT_FRAME_TXTIME 0 +#define CAN_ISOTP_DEFAULT_RECV_BS 0 +#define CAN_ISOTP_DEFAULT_RECV_STMIN 0x00 +#define CAN_ISOTP_DEFAULT_RECV_WFTMAX 0 + +#define CAN_ISOTP_DEFAULT_LL_MTU CAN_MTU +#define CAN_ISOTP_DEFAULT_LL_TX_DL CAN_MAX_DLEN +#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS 0 + +/* + * Remark on CAN_ISOTP_DEFAULT_RECV_* values: + * + * We can strongly assume, that the Linux Kernel implementation of + * CAN_ISOTP is capable to run with BS=0, STmin=0 and WFTmax=0. + * But as we like to be able to behave as a commonly available ECU, + * these default settings can be changed via sockopts. + * For that reason the STmin value is intentionally _not_ checked for + * consistency and copied directly into the flow control (FC) frame. + * + */ + +#endif diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h index 61769d4b7dbac..85387c76c24f3 100644 --- a/include/uapi/linux/input-event-codes.h +++ b/include/uapi/linux/input-event-codes.h @@ -407,6 +407,7 @@ #define BTN_TOOL_MOUSE 0x146 #define BTN_TOOL_LENS 0x147 #define BTN_TOOL_QUINTTAP 0x148 /* Five fingers on trackpad */ +#define BTN_STYLUS3 0x149 #define BTN_TOUCH 0x14a #define BTN_STYLUS 0x14b #define BTN_STYLUS2 0x14c @@ -438,10 +439,12 @@ #define KEY_TITLE 0x171 #define KEY_SUBTITLE 0x172 #define KEY_ANGLE 0x173 -#define KEY_ZOOM 0x174 +#define KEY_FULL_SCREEN 0x174 /* AC View Toggle */ +#define KEY_ZOOM KEY_FULL_SCREEN #define KEY_MODE 0x175 #define KEY_KEYBOARD 0x176 -#define KEY_SCREEN 0x177 +#define KEY_ASPECT_RATIO 0x177 /* HUTRR37: Aspect */ +#define KEY_SCREEN KEY_ASPECT_RATIO #define KEY_PC 0x178 /* Media Select Computer */ #define KEY_TV 0x179 /* Media Select TV */ #define KEY_TV2 0x17a /* Media Select Cable */ @@ -593,6 +596,7 @@ #define BTN_DPAD_RIGHT 0x223 #define KEY_ALS_TOGGLE 0x230 /* Ambient light sensor */ +#define KEY_ROTATE_LOCK_TOGGLE 0x231 /* Display rotation lock */ #define KEY_BUTTONCONFIG 0x240 /* AL Button Configuration */ #define KEY_TASKMANAGER 0x241 /* AL Task/Project Manager */ @@ -602,6 +606,7 @@ #define KEY_SCREENSAVER 0x245 /* AL Screen Saver */ #define KEY_VOICECOMMAND 0x246 /* Listening Voice Command */ #define KEY_ASSISTANT 0x247 /* AL Context-aware desktop assistant */ +#define KEY_KBD_LAYOUT_NEXT 0x248 /* AC Next Keyboard Layout Select */ #define KEY_BRIGHTNESS_MIN 0x250 /* Set Brightness to Minimum */ #define KEY_BRIGHTNESS_MAX 0x251 /* Set Brightness to Maximum */ @@ -706,6 +711,16 @@ #define REL_DIAL 0x07 #define REL_WHEEL 0x08 #define REL_MISC 0x09 +/* + * 0x0a is reserved and should not be used in input drivers. + * It was used by HID as REL_MISC+1 and userspace needs to detect if + * the next REL_* event is correct or is just REL_MISC + n. + * We define here REL_RESERVED so userspace can rely on it and detect + * the situation described above. + */ +#define REL_RESERVED 0x0a +#define REL_WHEEL_HI_RES 0x0b +#define REL_HWHEEL_HI_RES 0x0c #define REL_MAX 0x0f #define REL_CNT (REL_MAX+1) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index f41ea5af22eed..a0210a71abbdc 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -177,18 +177,27 @@ * * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK flag drivers * can indicate they support offloading EAPOL handshakes for WPA/WPA2 - * preshared key authentication. In %NL80211_CMD_CONNECT the preshared - * key should be specified using %NL80211_ATTR_PMK. Drivers supporting - * this offload may reject the %NL80211_CMD_CONNECT when no preshared - * key material is provided, for example when that driver does not - * support setting the temporal keys through %CMD_NEW_KEY. + * preshared key authentication in station mode. In %NL80211_CMD_CONNECT + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_CONNECT when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * Similarly @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X flag can be * set by drivers indicating offload support of the PTK/GTK EAPOL - * handshakes during 802.1X authentication. In order to use the offload - * the %NL80211_CMD_CONNECT should have %NL80211_ATTR_WANT_1X_4WAY_HS - * attribute flag. Drivers supporting this offload may reject the - * %NL80211_CMD_CONNECT when the attribute flag is not present. + * handshakes during 802.1X authentication in station mode. In order to + * use the offload the %NL80211_CMD_CONNECT should have + * %NL80211_ATTR_WANT_1X_4WAY_HS attribute flag. Drivers supporting this + * offload may reject the %NL80211_CMD_CONNECT when the attribute flag is + * not present. + * + * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag drivers + * can indicate they support offloading EAPOL handshakes for WPA/WPA2 + * preshared key authentication in AP mode. In %NL80211_CMD_START_AP + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_START_AP when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * For 802.1X the PMK or PMK-R0 are set by providing %NL80211_ATTR_PMK * using %NL80211_CMD_SET_PMK. For offloaded FT support also @@ -231,6 +240,16 @@ * use in a FILS shared key connection with PMKSA caching. */ +/** + * DOC: SAE authentication offload + * + * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they + * support offloading SAE authentication for WPA3-Personal networks. The + * password for SAE should be specified using %NL80211_ATTR_SAE_PASSWORD in + * %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP for station and AP mode + * respectively. + */ + /** * enum nl80211_commands - supported nl80211 commands * @@ -569,13 +588,14 @@ * authentication/association or not receiving a response from the AP. * Non-zero %NL80211_ATTR_STATUS_CODE value is indicated in that case as * well to remain backwards compatible. - * @NL80211_CMD_ROAM: notifcation indicating the card/driver roamed by itself. - * When the driver roamed in a network that requires 802.1X authentication, - * %NL80211_ATTR_PORT_AUTHORIZED should be set if the 802.1X authentication - * was done by the driver or if roaming was done using Fast Transition - * protocol (in which case 802.1X authentication is not needed). If - * %NL80211_ATTR_PORT_AUTHORIZED is not set, user space is responsible for - * the 802.1X authentication. + * When establishing a security association, drivers that support 4 way + * handshake offload should send %NL80211_CMD_PORT_AUTHORIZED event when + * the 4 way handshake is completed successfully. + * @NL80211_CMD_ROAM: Notification indicating the card/driver roamed by itself. + * When a security association was established with the new AP (e.g. if + * the FT protocol was used for roaming or the driver completed the 4 way + * handshake), this event should be followed by an + * %NL80211_CMD_PORT_AUTHORIZED event. * @NL80211_CMD_DISCONNECT: drop a given connection; also used to notify * userspace that a connection was dropped by the AP or due to other * reasons, for this the %NL80211_ATTR_DISCONNECTED_BY_AP and @@ -982,6 +1002,12 @@ * @NL80211_CMD_DEL_PMK: For offloaded 4-Way handshake, delete the previously * configured PMK for the authenticator address identified by * &NL80211_ATTR_MAC. + * @NL80211_CMD_PORT_AUTHORIZED: An event that indicates that the 4 way + * handshake was completed successfully by the driver. The BSSID is + * specified with &NL80211_ATTR_MAC. Drivers that support 4 way handshake + * offload should send this event after indicating 802.11 association with + * &NL80211_CMD_CONNECT or &NL80211_CMD_ROAM. If the 4 way handshake failed + * &NL80211_CMD_DISCONNECT should be indicated instead. * * @NL80211_CMD_MAX: highest used command number * @__NL80211_CMD_AFTER_LAST: internal use @@ -1185,6 +1211,8 @@ enum nl80211_commands { NL80211_CMD_SET_PMK, NL80211_CMD_DEL_PMK, + NL80211_CMD_PORT_AUTHORIZED, + /* add new commands above here */ /* used to define NL80211_CMD_MAX below */ @@ -2120,10 +2148,11 @@ enum nl80211_commands { * * @NL80211_ATTR_PMK: attribute for passing PMK key material. Used with * %NL80211_CMD_SET_PMKSA for the PMKSA identified by %NL80211_ATTR_PMKID. - * For %NL80211_CMD_CONNECT it is used to provide PSK for offloading 4-way - * handshake for WPA/WPA2-PSK networks. For 802.1X authentication it is - * used with %NL80211_CMD_SET_PMK. For offloaded FT support this attribute - * specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME is included as well. + * For %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP it is used to provide + * PSK for offloading 4-way handshake for WPA/WPA2-PSK networks. For 802.1X + * authentication it is used with %NL80211_CMD_SET_PMK. For offloaded FT + * support this attribute specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME + * is included as well. * * @NL80211_ATTR_SCHED_SCAN_MULTI: flag attribute which user-space shall use to * indicate that it supports multiple active scheduled scan requests. @@ -2139,6 +2168,66 @@ enum nl80211_commands { * the driver or is not needed (because roaming used the Fast Transition * protocol). * + * @NL80211_ATTR_EXTERNAL_AUTH_ACTION: Identify the requested external + * authentication operation (u32 attribute with an + * &enum nl80211_external_auth_action value). This is used with the + * %NL80211_CMD_EXTERNAL_AUTH request event. + * @NL80211_ATTR_EXTERNAL_AUTH_SUPPORT: Flag attribute indicating that the user + * space supports external authentication. This attribute shall be used + * only with %NL80211_CMD_CONNECT request. The driver may offload + * authentication processing to user space if this capability is indicated + * in NL80211_CMD_CONNECT requests from the user space. + * + * @NL80211_ATTR_NSS: Station's New/updated RX_NSS value notified using this + * u8 attribute. This is used with %NL80211_CMD_STA_OPMODE_CHANGED. + * + * @NL80211_ATTR_TXQ_STATS: TXQ statistics (nested attribute, see &enum + * nl80211_txq_stats) + * @NL80211_ATTR_TXQ_LIMIT: Total packet limit for the TXQ queues for this phy. + * The smaller of this and the memory limit is enforced. + * @NL80211_ATTR_TXQ_MEMORY_LIMIT: Total memory memory limit (in bytes) for the + * TXQ queues for this phy. The smaller of this and the packet limit is + * enforced. + * @NL80211_ATTR_TXQ_QUANTUM: TXQ scheduler quantum (bytes). Number of bytes + * a flow is assigned on each round of the DRR scheduler. + * @NL80211_ATTR_HE_CAPABILITY: HE Capability information element (from + * association request when used with NL80211_CMD_NEW_STATION). Can be set + * only if %NL80211_STA_FLAG_WME is set. + * + * @NL80211_ATTR_FTM_RESPONDER: nested attribute which user-space can include + * in %NL80211_CMD_START_AP or %NL80211_CMD_SET_BEACON for fine timing + * measurement (FTM) responder functionality and containing parameters as + * possible, see &enum nl80211_ftm_responder_attr + * + * @NL80211_ATTR_FTM_RESPONDER_STATS: Nested attribute with FTM responder + * statistics, see &enum nl80211_ftm_responder_stats. + * + * @NL80211_ATTR_TIMEOUT: Timeout for the given operation in milliseconds (u32), + * if the attribute is not given no timeout is requested. Note that 0 is an + * invalid value. + * + * @NL80211_ATTR_PEER_MEASUREMENTS: peer measurements request (and result) + * data, uses nested attributes specified in + * &enum nl80211_peer_measurement_attrs. + * This is also used for capability advertisement in the wiphy information, + * with the appropriate sub-attributes. + * + * @NL80211_ATTR_AIRTIME_WEIGHT: Station's weight when scheduled by the airtime + * scheduler. + * + * @NL80211_ATTR_STA_TX_POWER_SETTING: Transmit power setting type (u8) for + * station associated with the AP. See &enum nl80211_tx_power_setting for + * possible values. + * @NL80211_ATTR_STA_TX_POWER: Transmit power level (s16) in dBm units. This + * allows to set Tx power for a station. If this attribute is not included, + * the default per-interface tx power setting will be overriding. Driver + * should be picking up the lowest tx power, either tx power per-interface + * or per-station. + * + * @NL80211_ATTR_SAE_PASSWORD: attribute for passing SAE password material. It + * is used with %NL80211_CMD_CONNECT to provide password for offloading + * SAE authentication for WPA3-Personal networks. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2565,6 +2654,35 @@ enum nl80211_attrs { NL80211_ATTR_PMKR0_NAME, NL80211_ATTR_PORT_AUTHORIZED, + NL80211_ATTR_EXTERNAL_AUTH_ACTION, + NL80211_ATTR_EXTERNAL_AUTH_SUPPORT, + + NL80211_ATTR_NSS, + NL80211_ATTR_ACK_SIGNAL, + + NL80211_ATTR_CONTROL_PORT_OVER_NL80211, + + NL80211_ATTR_TXQ_STATS, + NL80211_ATTR_TXQ_LIMIT, + NL80211_ATTR_TXQ_MEMORY_LIMIT, + NL80211_ATTR_TXQ_QUANTUM, + + NL80211_ATTR_HE_CAPABILITY, + + NL80211_ATTR_FTM_RESPONDER, + + NL80211_ATTR_FTM_RESPONDER_STATS, + + NL80211_ATTR_TIMEOUT, + + NL80211_ATTR_PEER_MEASUREMENTS, + + NL80211_ATTR_AIRTIME_WEIGHT, + NL80211_ATTR_STA_TX_POWER_SETTING, + NL80211_ATTR_STA_TX_POWER, + + NL80211_ATTR_SAE_PASSWORD, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -3958,6 +4076,7 @@ enum nl80211_mfp { enum nl80211_wpa_versions { NL80211_WPA_VERSION_1 = 1 << 0, NL80211_WPA_VERSION_2 = 1 << 1, + NL80211_WPA_VERSION_3 = 1 << 2, }; /** @@ -4916,6 +5035,68 @@ enum nl80211_feature_flags { * handshake with 802.1X in station mode (will pass EAP frames to the host * and accept the set_pmk/del_pmk commands), doing it in the host might not * be supported. + * @NL80211_EXT_FEATURE_FILS_MAX_CHANNEL_TIME: Driver is capable of overriding + * the max channel attribute in the FILS request params IE with the + * actual dwell time. + * @NL80211_EXT_FEATURE_ACCEPT_BCAST_PROBE_RESP: Driver accepts broadcast probe + * response + * @NL80211_EXT_FEATURE_OCE_PROBE_REQ_HIGH_TX_RATE: Driver supports sending + * the first probe request in each channel at rate of at least 5.5Mbps. + * @NL80211_EXT_FEATURE_OCE_PROBE_REQ_DEFERRAL_SUPPRESSION: Driver supports + * probe request tx deferral and suppression + * @NL80211_EXT_FEATURE_MFP_OPTIONAL: Driver supports the %NL80211_MFP_OPTIONAL + * value in %NL80211_ATTR_USE_MFP. + * @NL80211_EXT_FEATURE_LOW_SPAN_SCAN: Driver supports low span scan. + * @NL80211_EXT_FEATURE_LOW_POWER_SCAN: Driver supports low power scan. + * @NL80211_EXT_FEATURE_HIGH_ACCURACY_SCAN: Driver supports high accuracy scan. + * @NL80211_EXT_FEATURE_DFS_OFFLOAD: HW/driver will offload DFS actions. + * Device or driver will do all DFS-related actions by itself, + * informing user-space about CAC progress, radar detection event, + * channel change triggered by radar detection event. + * No need to start CAC from user-space, no need to react to + * "radar detected" event. + * @NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211: Driver supports sending and + * receiving control port frames over nl80211 instead of the netdevice. + * @NL80211_EXT_FEATURE_DATA_ACK_SIGNAL_SUPPORT: This Driver support data ack + * rssi if firmware support, this flag is to intimate about ack rssi + * support to nl80211. + * @NL80211_EXT_FEATURE_TXQS: Driver supports FQ-CoDel-enabled intermediate + * TXQs. + * @NL80211_EXT_FEATURE_SCAN_RANDOM_SN: Driver/device supports randomizing the + * SN in probe request frames if requested by %NL80211_SCAN_FLAG_RANDOM_SN. + * @NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT: Driver/device can omit all data + * except for supported rates from the probe request content if requested + * by the %NL80211_SCAN_FLAG_MIN_PREQ_CONTENT flag. + * @NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER: Driver supports enabling fine + * timing measurement responder role. + * + * @NL80211_EXT_FEATURE_CAN_REPLACE_PTK0: Driver/device confirm that they are + * able to rekey an in-use key correctly. Userspace must not rekey PTK keys + * if this flag is not set. Ignoring this can leak clear text packets and/or + * freeze the connection. + * @NL80211_EXT_FEATURE_EXT_KEY_ID: Driver supports "Extended Key ID for + * Individually Addressed Frames" from IEEE802.11-2016. + * + * @NL80211_EXT_FEATURE_AIRTIME_FAIRNESS: Driver supports getting airtime + * fairness for transmitted packets and has enabled airtime fairness + * scheduling. + * + * @NL80211_EXT_FEATURE_AP_PMKSA_CACHING: Driver/device supports PMKSA caching + * (set/del PMKSA operations) in AP mode. + * + * @NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD: Driver supports + * filtering of sched scan results using band specific RSSI thresholds. + * + * @NL80211_EXT_FEATURE_STA_TX_PWR: This driver supports controlling tx power + * to a station. + * + * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in + * station mode (SAE password is passed as part of the connect command) + * or AP mode (SAE password is passed as part of the start AP command). + * + * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way + * handshake with PSK in AP mode (PSK is passed as part of the start AP + * command). * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. @@ -4938,6 +5119,29 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_FILS_SK_OFFLOAD, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X, + NL80211_EXT_FEATURE_FILS_MAX_CHANNEL_TIME, + NL80211_EXT_FEATURE_ACCEPT_BCAST_PROBE_RESP, + NL80211_EXT_FEATURE_OCE_PROBE_REQ_HIGH_TX_RATE, + NL80211_EXT_FEATURE_OCE_PROBE_REQ_DEFERRAL_SUPPRESSION, + NL80211_EXT_FEATURE_MFP_OPTIONAL, + NL80211_EXT_FEATURE_LOW_SPAN_SCAN, + NL80211_EXT_FEATURE_LOW_POWER_SCAN, + NL80211_EXT_FEATURE_HIGH_ACCURACY_SCAN, + NL80211_EXT_FEATURE_DFS_OFFLOAD, + NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211, + NL80211_EXT_FEATURE_DATA_ACK_SIGNAL_SUPPORT, + NL80211_EXT_FEATURE_TXQS, + NL80211_EXT_FEATURE_SCAN_RANDOM_SN, + NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT, + NL80211_EXT_FEATURE_CAN_REPLACE_PTK0, + NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER, + NL80211_EXT_FEATURE_AIRTIME_FAIRNESS, + NL80211_EXT_FEATURE_AP_PMKSA_CACHING, + NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD, + NL80211_EXT_FEATURE_EXT_KEY_ID, + NL80211_EXT_FEATURE_STA_TX_PWR, + NL80211_EXT_FEATURE_SAE_OFFLOAD, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/jenkins_build.sh b/jenkins_build.sh new file mode 100755 index 0000000000000..8fba37bce2880 --- /dev/null +++ b/jenkins_build.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +#gcc: + +# +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.05/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.08/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.08-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabihf/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabihf/gcc-linaro-7.3.1-2018.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf.tar.xz +# + +gcc_html_path="https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/" +gcc_filename_prefix="gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf" +gcc_banner="arm-linux-gnueabihf-gcc (Linaro GCC 7.5-2019.12) 7.5.0" +gcc_copyright="2017" +datestamp="2019.12-gcc-arm-linux-gnueabihf" + +# + +if [ ! -d ${gcc_filename_prefix}/ ] ; then + rm -rf ./gcc-* || true + #wget -c ${site}/${version}/${filename} + wget -c http://gfnd.rcn-ee.org/farm/jenkins-dl/${gcc_filename_prefix}.tar.xz + tar xf ${gcc_filename_prefix}.tar.xz +fi + +export CC=`pwd`/${gcc_filename_prefix}/bin/arm-linux-gnueabihf- + +make ARCH=arm clean +make ARCH=arm bb.org_defconfig + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" zImage]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" zImage +if [ ! -f arch/arm/boot/zImage ] ; then + echo "failed: [arch/arm/boot/zImage]" + exit 1 +fi + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" modules]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" modules +if [ ! -f drivers/spi/spidev.ko ] ; then + echo "failed: [drivers/spi/spidev.ko]" + exit 1 +fi + +echo "[make ARCH=arm CROSS_COMPILE=\"${binary}\" dtbs]" +make ARCH=arm CROSS_COMPILE="ccache ${CC}" dtbs +if [ ! -f arch/arm/boot/dts/am335x-boneblack.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-boneblack.dtb]" + exit 1 +else + if [ -f arch/arm/boot/dts/am335x-pocketbeagle.dts ] ; then + if [ ! -f arch/arm/boot/dts/am335x-pocketbeagle.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-pocketbeagle.dtb]" + exit 1 + fi + fi +fi + +make ARCH=arm clean diff --git a/kernel/fork.c b/kernel/fork.c index 6d6ce2c3a364c..172054521e4e8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -675,7 +675,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; - get_file(file); + vma_get_file(tmp); if (tmp->vm_flags & VM_DENYWRITE) atomic_dec(&inode->i_writecount); i_mmap_lock_write(mapping); diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index bf694c709b96f..28e6c758e49ca 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -143,7 +143,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; unsigned long nr_lock_classes; static struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; -static inline struct lock_class *hlock_class(struct held_lock *hlock) +inline struct lock_class *lockdep_hlock_class(struct held_lock *hlock) { if (!hlock->class_idx) { /* @@ -154,6 +154,8 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock) } return lock_classes + hlock->class_idx - 1; } +EXPORT_SYMBOL_GPL(lockdep_hlock_class); +#define hlock_class(hlock) lockdep_hlock_class(hlock) #ifdef CONFIG_LOCK_STAT static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats); diff --git a/kernel/softirq.c b/kernel/softirq.c index a4c87cf27f9da..aa3b313977779 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -386,7 +386,7 @@ static inline void tick_irq_exit(void) /* Make sure that timer wheel updates are propagated */ if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { - if (!in_irq()) + if (!in_interrupt()) tick_nohz_irq_exit(); } #endif diff --git a/kernel/task_work.c b/kernel/task_work.c index 0fef395662a6e..83fb1ecfc33de 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -116,3 +116,4 @@ void task_work_run(void) } while (work); } } +EXPORT_SYMBOL_GPL(task_work_run); diff --git a/lib/Kconfig b/lib/Kconfig index b1445b22a6def..6c1423c5c1886 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -10,6 +10,14 @@ menu "Library routines" config RAID6_PQ tristate +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. + config BITREVERSE tristate diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 4769947232582..7e4f7a8ffa8e2 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -34,63 +34,64 @@ struct raid6_calls raid6_call; EXPORT_SYMBOL_GPL(raid6_call); const struct raid6_calls * const raid6_algos[] = { -#if defined(__ia64__) - &raid6_intx16, - &raid6_intx32, -#endif #if defined(__i386__) && !defined(__arch_um__) - &raid6_mmxx1, - &raid6_mmxx2, - &raid6_sse1x1, - &raid6_sse1x2, - &raid6_sse2x1, - &raid6_sse2x2, -#ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, - &raid6_avx2x2, -#endif #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, &raid6_avx512x2, + &raid6_avx512x1, #endif -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_sse2x1, - &raid6_sse2x2, - &raid6_sse2x4, #ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, &raid6_avx2x2, - &raid6_avx2x4, + &raid6_avx2x1, +#endif + &raid6_sse2x2, + &raid6_sse2x1, + &raid6_sse1x2, + &raid6_sse1x1, + &raid6_mmxx2, + &raid6_mmxx1, #endif +#if defined(__x86_64__) && !defined(__arch_um__) #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, - &raid6_avx512x2, &raid6_avx512x4, + &raid6_avx512x2, + &raid6_avx512x1, #endif +#ifdef CONFIG_AS_AVX2 + &raid6_avx2x4, + &raid6_avx2x2, + &raid6_avx2x1, +#endif + &raid6_sse2x4, + &raid6_sse2x2, + &raid6_sse2x1, #endif #ifdef CONFIG_ALTIVEC - &raid6_altivec1, - &raid6_altivec2, - &raid6_altivec4, + &raid6_vpermxor8, + &raid6_vpermxor4, + &raid6_vpermxor2, + &raid6_vpermxor1, &raid6_altivec8, -#endif -#if defined(CONFIG_TILEGX) - &raid6_tilegx8, + &raid6_altivec4, + &raid6_altivec2, + &raid6_altivec1, #endif #if defined(CONFIG_S390) &raid6_s390vx8, #endif - &raid6_intx1, - &raid6_intx2, - &raid6_intx4, - &raid6_intx8, #ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx1, - &raid6_neonx2, - &raid6_neonx4, &raid6_neonx8, + &raid6_neonx4, + &raid6_neonx2, + &raid6_neonx1, +#endif +#if defined(__ia64__) + &raid6_intx32, + &raid6_intx16, #endif + &raid6_intx8, + &raid6_intx4, + &raid6_intx2, + &raid6_intx1, NULL }; @@ -162,6 +163,11 @@ static inline const struct raid6_calls *raid6_choose_gen( if ((*algo)->valid && !(*algo)->valid()) continue; + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + best = *algo; + break; + } + perf = 0; preempt_disable(); diff --git a/mm/Makefile b/mm/Makefile index e7ebd176fb935..d6bcfb01b5283 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -39,7 +39,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ compaction.o vmacache.o swap_slots.o \ interval_tree.o list_lru.o workingset.o \ - debug.o $(mmu-y) + prfile.o debug.o $(mmu-y) obj-y += init-mm.o diff --git a/mm/filemap.c b/mm/filemap.c index e2e738cc08b1d..2a410810f2333 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2589,7 +2589,7 @@ int filemap_page_mkwrite(struct vm_fault *vmf) int ret = VM_FAULT_LOCKED; sb_start_pagefault(inode->i_sb); - file_update_time(vmf->vma->vm_file); + vma_file_update_time(vmf->vma); lock_page(page); if (page->mapping != inode->i_mapping) { unlock_page(page); diff --git a/mm/mmap.c b/mm/mmap.c index 00dab291e61df..2f9b902b59867 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -171,7 +171,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); mpol_put(vma_policy(vma)); kmem_cache_free(vm_area_cachep, vma); return next; @@ -896,7 +896,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, if (remove_next) { if (file) { uprobe_munmap(next, next->vm_start, next->vm_end); - fput(file); + vma_fput(vma); } if (next->anon_vma) anon_vma_merge(vma, next); @@ -1778,8 +1778,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return addr; unmap_and_free_vma: + vma_fput(vma); vma->vm_file = NULL; - fput(file); /* Undo any partial mapping done by a device driver. */ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); @@ -2602,7 +2602,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, goto out_free_mpol; if (new->vm_file) - get_file(new->vm_file); + vma_get_file(new); if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); @@ -2621,7 +2621,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->close) new->vm_ops->close(new); if (new->vm_file) - fput(new->vm_file); + vma_fput(new); unlink_anon_vmas(new); out_free_mpol: mpol_put(vma_policy(new)); @@ -2783,7 +2783,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, struct vm_area_struct *vma; unsigned long populate = 0; unsigned long ret = -EINVAL; - struct file *file; + struct file *file, *prfile; pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n", current->comm, current->pid); @@ -2858,10 +2858,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } } - file = get_file(vma->vm_file); + vma_get_file(vma); + file = vma->vm_file; + prfile = vma->vm_prfile; ret = do_mmap_pgoff(vma->vm_file, start, size, prot, flags, pgoff, &populate, NULL); + if (!IS_ERR_VALUE(ret) && file && prfile) { + struct vm_area_struct *new_vma; + + new_vma = find_vma(mm, ret); + if (!new_vma->vm_prfile) + new_vma->vm_prfile = prfile; + if (new_vma != vma) + get_file(prfile); + } + /* + * two fput()s instead of vma_fput(vma), + * coz vma may not be available anymore. + */ fput(file); + if (prfile) + fput(prfile); out: up_write(&mm->mmap_sem); if (populate) @@ -3172,7 +3189,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (anon_vma_clone(new_vma, vma)) goto out_free_mempol; if (new_vma->vm_file) - get_file(new_vma->vm_file); + vma_get_file(new_vma); if (new_vma->vm_ops && new_vma->vm_ops->open) new_vma->vm_ops->open(new_vma); vma_link(mm, new_vma, prev, rb_link, rb_parent); diff --git a/mm/nommu.c b/mm/nommu.c index 17c00d93de2e4..4bcdf9448dd0b 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -641,7 +641,7 @@ static void __put_nommu_region(struct vm_region *region) up_write(&nommu_region_sem); if (region->vm_file) - fput(region->vm_file); + vmr_fput(region); /* IO memory and memory shared directly out of the pagecache * from ramfs/tmpfs mustn't be released here */ @@ -799,7 +799,7 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); put_nommu_region(vma->vm_region); kmem_cache_free(vm_area_cachep, vma); } @@ -1321,7 +1321,7 @@ unsigned long do_mmap(struct file *file, goto error_just_free; } } - fput(region->vm_file); + vmr_fput(region); kmem_cache_free(vm_region_jar, region); region = pregion; result = start; @@ -1396,10 +1396,10 @@ unsigned long do_mmap(struct file *file, up_write(&nommu_region_sem); error: if (region->vm_file) - fput(region->vm_file); + vmr_fput(region); kmem_cache_free(vm_region_jar, region); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); kmem_cache_free(vm_area_cachep, vma); return ret; diff --git a/mm/prfile.c b/mm/prfile.c new file mode 100644 index 0000000000000..024cdcfae1b1e --- /dev/null +++ b/mm/prfile.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Mainly for aufs which mmap(2) different file and wants to print different + * path in /proc/PID/maps. + * Call these functions via macros defined in linux/mm.h. + * + * See Documentation/filesystems/aufs/design/06mmap.txt + * + * Copyright (c) 2014-2019 Junjro R. Okajima + * Copyright (c) 2014 Ian Campbell + */ + +#include +#include +#include + +/* #define PRFILE_TRACE */ +static inline void prfile_trace(struct file *f, struct file *pr, + const char func[], int line, const char func2[]) +{ +#ifdef PRFILE_TRACE + if (pr) + pr_info("%s:%d: %s, %pD2\n", func, line, func2, f); +#endif +} + +void vma_do_file_update_time(struct vm_area_struct *vma, const char func[], + int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + file_update_time(f); + if (f && pr) + file_update_time(pr); +} + +struct file *vma_do_pr_or_file(struct vm_area_struct *vma, const char func[], + int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + return (f && pr) ? pr : f; +} + +void vma_do_get_file(struct vm_area_struct *vma, const char func[], int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + get_file(f); + if (f && pr) + get_file(pr); +} + +void vma_do_fput(struct vm_area_struct *vma, const char func[], int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + fput(f); + if (f && pr) + fput(pr); +} + +#ifndef CONFIG_MMU +struct file *vmr_do_pr_or_file(struct vm_region *region, const char func[], + int line) +{ + struct file *f = region->vm_file, *pr = region->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + return (f && pr) ? pr : f; +} + +void vmr_do_fput(struct vm_region *region, const char func[], int line) +{ + struct file *f = region->vm_file, *pr = region->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + fput(f); + if (f && pr) + fput(pr); +} +#endif /* !CONFIG_MMU */ diff --git a/net/Kconfig b/net/Kconfig index b18a043358ae5..80910d210d1e2 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -85,6 +85,7 @@ config INET Short answer: say Y. if INET +source "net/wireguard/Kconfig" source "net/ipv4/Kconfig" source "net/ipv6/Kconfig" source "net/netlabel/Kconfig" diff --git a/net/Makefile b/net/Makefile index c431ddf0e8945..41c4be2f9b2f5 100644 --- a/net/Makefile +++ b/net/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_NET) += $(tmp-y) obj-$(CONFIG_LLC) += llc/ obj-$(CONFIG_NET) += ethernet/ 802/ sched/ netlink/ bpf/ obj-$(CONFIG_NETFILTER) += netfilter/ +obj-$(CONFIG_WIREGUARD) += wireguard/ obj-$(CONFIG_INET) += ipv4/ obj-$(CONFIG_TLS) += tls/ obj-$(CONFIG_XFRM) += xfrm/ diff --git a/net/can/Kconfig b/net/can/Kconfig index a15c0e0d1fc77..8774a591398bc 100644 --- a/net/can/Kconfig +++ b/net/can/Kconfig @@ -51,6 +51,16 @@ config CAN_GW They can be modified with AND/OR/XOR/SET operations as configured by the netlink configuration interface known e.g. from iptables. +config CAN_ISOTP + tristate "ISO 15765-2:2016 CAN transport protocol" + default y + ---help--- + ISO 15765-2 CAN transport protocol for protocol family CAN + + This implementation is already widely used in automotive use-cases, e.g. + for UDS based OBD diagnosis. Although some small adaptions may be applied + to make it ready for Linux Mainline. Feedback is welcome. + source "drivers/net/can/Kconfig" endif diff --git a/net/can/Makefile b/net/can/Makefile index 1242bbbfe57f5..fde676965946a 100644 --- a/net/can/Makefile +++ b/net/can/Makefile @@ -15,3 +15,6 @@ can-bcm-y := bcm.o obj-$(CONFIG_CAN_GW) += can-gw.o can-gw-y := gw.o + +obj-$(CONFIG_CAN_ISOTP) += can-isotp.o +can-isotp-y := isotp.o diff --git a/net/can/isotp.c b/net/can/isotp.c new file mode 100644 index 0000000000000..a0d20d9443bf5 --- /dev/null +++ b/net/can/isotp.c @@ -0,0 +1,1537 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * isotp.c - ISO 15765-2 CAN transport protocol for protocol family CAN + * + * WARNING: This is ALPHA code for discussions and first tests that should + * not be used in production environments. + * + * In the discussion the Socket-API to the userspace or the ISO-TP socket + * options or the return values we may change! Current behaviour: + * + * - no ISO-TP specific return values are provided to the userspace + * - when a transfer (tx) is on the run the next write() blocks until it's done + * - no support for sending wait frames to the data source in the rx path + * + * Copyright (c) 2017 Volkswagen Group Electronic Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Volkswagen nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General + * Public License ("GPL") version 2, in which case the provisions of the + * GPL apply INSTEAD OF those given above. + * + * The provided data structures and external interfaces from this code + * are not restricted to be used by modules with a GPL compatible license. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + * Send feedback to + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) +#include +#define CAN_SKBRES sizeof(struct can_skb_priv) +#else +#define CAN_SKBRES 0 +#endif +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25) +#include "compat.h" +#endif + +#define CAN_ISOTP_VERSION "20181217" +static __initdata const char banner[] = + KERN_INFO "can: isotp protocol (rev " CAN_ISOTP_VERSION " alpha)\n"; + +MODULE_DESCRIPTION("PF_CAN isotp 15765-2:2016 protocol"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Oliver Hartkopp "); +MODULE_ALIAS("can-proto-6"); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#error This modules needs hrtimers (available since Kernel 2.6.22) +#endif + +#define DBG(fmt, args...) (printk( KERN_DEBUG "can-isotp: %s: " fmt, \ + __func__, ##args)) +#undef DBG +#define DBG(fmt, args...) + +#define SINGLE_MASK(id) ((id & CAN_EFF_FLAG) ? \ + (CAN_EFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG) : \ + (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG)) + +/* + ISO 15765-2:2016 supports more than 4095 byte per ISO PDU as the FF_DL can + take full 32 bit values (4 Gbyte). We would need some good concept to handle + this between user space and kernel space. For now increase the static buffer + to something about 8 kbyte to be able to test this new functionality. +*/ +#define MAX_MSG_LENGTH 8200 + +/* N_PCI type values in bits 7-4 of N_PCI bytes */ +#define N_PCI_SF 0x00 /* single frame */ +#define N_PCI_FF 0x10 /* first frame */ +#define N_PCI_CF 0x20 /* consecutive frame */ +#define N_PCI_FC 0x30 /* flow control */ + +#define N_PCI_SZ 1 /* size of the PCI byte #1 */ +#define SF_PCI_SZ4 1 /* size of SingleFrame PCI including 4 bit SF_DL */ +#define SF_PCI_SZ8 2 /* size of SingleFrame PCI including 8 bit SF_DL */ +#define FF_PCI_SZ12 2 /* size of FirstFrame PCI including 12 bit FF_DL */ +#define FF_PCI_SZ32 6 /* size of FirstFrame PCI including 32 bit FF_DL */ +#define FC_CONTENT_SZ 3 /* flow control content size in byte (FS/BS/STmin) */ + +#define ISOTP_CHECK_PADDING (CAN_ISOTP_CHK_PAD_LEN | CAN_ISOTP_CHK_PAD_DATA) + +/* Flow Status given in FC frame */ +#define ISOTP_FC_CTS 0 /* clear to send */ +#define ISOTP_FC_WT 1 /* wait */ +#define ISOTP_FC_OVFLW 2 /* overflow */ + +enum { + ISOTP_IDLE = 0, + ISOTP_WAIT_FIRST_FC, + ISOTP_WAIT_FC, + ISOTP_WAIT_DATA, + ISOTP_SENDING +}; + +struct tpcon { + int idx; + int len; + u8 state; + u8 bs; + u8 sn; + u8 ll_dl; + u8 buf[MAX_MSG_LENGTH+1]; +}; + +struct isotp_sock { + struct sock sk; + int bound; + int ifindex; + canid_t txid; + canid_t rxid; + ktime_t tx_gap; + ktime_t lastrxcf_tstamp; + struct hrtimer rxtimer, txtimer; + struct tasklet_struct txtsklet; + struct can_isotp_options opt; + struct can_isotp_fc_options rxfc, txfc; + struct can_isotp_ll_options ll; + __u32 force_tx_stmin; + __u32 force_rx_stmin; + struct tpcon rx, tx; + struct notifier_block notifier; + wait_queue_head_t wait; +}; + +static inline struct isotp_sock *isotp_sk(const struct sock *sk) +{ + return (struct isotp_sock *)sk; +} + +static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + rxtimer); + if (so->rx.state == ISOTP_WAIT_DATA) { + DBG("we did not get new data frames in time.\n"); + + /* reset tx state */ + so->rx.state = ISOTP_IDLE; + } + + return HRTIMER_NORESTART; +} + +static void isotp_skb_reserve(struct sk_buff *skb, struct net_device *dev) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) + can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,5) + can_skb_prv(skb)->skbcnt = 0; +#endif +} + +static void isotp_skb_destructor(struct sk_buff *skb) +{ + sock_put(skb->sk); +} + +static inline void isotp_skb_set_owner(struct sk_buff *skb, struct sock *sk) +{ + if (sk) { + sock_hold(sk); + skb->destructor = isotp_skb_destructor; + skb->sk = sk; + } +} + +static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus) +{ + struct net_device *dev; + struct sk_buff *nskb; + struct canfd_frame *ncf; + struct isotp_sock *so = isotp_sk(sk); + + nskb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any()); + if (!nskb) + return 1; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) { + kfree_skb(nskb); + return 1; + } + isotp_skb_reserve(nskb, dev); + nskb->dev = dev; + isotp_skb_set_owner(nskb, sk); + ncf = (struct canfd_frame *) nskb->data; + skb_put(nskb, so->ll.mtu); + + /* create & send flow control reply */ + ncf->can_id = so->txid; + + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + memset(ncf->data, so->opt.txpad_content, CAN_MAX_DLEN); + ncf->len = CAN_MAX_DLEN; + } else + ncf->len = ae + FC_CONTENT_SZ; + + ncf->data[ae] = N_PCI_FC | flowstatus; + ncf->data[ae + 1] = so->rxfc.bs; + ncf->data[ae + 2] = so->rxfc.stmin; + + if (ae) + ncf->data[0] = so->opt.ext_address; + + if (so->ll.mtu == CANFD_MTU) + ncf->flags = so->ll.tx_flags; + + can_send(nskb, 1); + dev_put(dev); + + /* reset blocksize counter */ + so->rx.bs = 0; + + /* reset last CF frame rx timestamp for rx stmin enforcement */ + so->lastrxcf_tstamp = ktime_set(0,0); + + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1,0), HRTIMER_MODE_REL); + return 0; +} + +static void isotp_rcv_skb(struct sk_buff *skb, struct sock *sk) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)skb->cb; + + BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct sockaddr_can)); + + memset(addr, 0, sizeof(*addr)); + addr->can_family = AF_CAN; + addr->can_ifindex = skb->dev->ifindex; + + if (sock_queue_rcv_skb(sk, skb) < 0) + kfree_skb(skb); +} + +static u8 padlen(u8 datalen) +{ + const u8 plen[] = {8, 8, 8, 8, 8, 8, 8, 8, 8, /* 0 - 8 */ + 12, 12, 12, 12, /* 9 - 12 */ + 16, 16, 16, 16, /* 13 - 16 */ + 20, 20, 20, 20, /* 17 - 20 */ + 24, 24, 24, 24, /* 21 - 24 */ + 32, 32, 32, 32, 32, 32, 32, 32, /* 25 - 32 */ + 48, 48, 48, 48, 48, 48, 48, 48, /* 33 - 40 */ + 48, 48, 48, 48, 48, 48, 48, 48}; /* 41 - 48 */ + + if (datalen > 48) + return 64; + + return plen[datalen]; +} + +/* check for length optimization and return 1/true when the check fails */ +static int check_optimized(struct canfd_frame *cf, int start_index) +{ + /* + * for CAN_DL <= 8 the start_index is equal to the CAN_DL as the + * padding would start at this point. E.g. if the padding would + * start at cf.data[7] cf->len has to be 7 to be optimal. + * Note: The data[] index starts with zero. + */ + if (cf->len <= CAN_MAX_DLEN) + return (cf->len != start_index); + + /* + * This relation is also valid in the non-linear DLC range, where + * we need to take care of the minimal next possible CAN_DL. + * The correct check would be (padlen(cf->len) != padlen(start_index)). + * But as cf->len can only take discrete values from 12, .., 64 at this + * point the padlen(cf->len) is always equal to cf->len. + */ + return (cf->len != padlen(start_index)); +} + +/* check padding and return 1/true when the check fails */ +static int check_pad(struct isotp_sock *so, struct canfd_frame *cf, + int start_index, __u8 content) +{ + int i; + + /* no RX_PADDING value => check length of optimized frame length */ + if (!(so->opt.flags & CAN_ISOTP_RX_PADDING)) { + + if (so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) + return check_optimized(cf, start_index); + + /* no valid test against empty value => ignore frame */ + return 1; + } + + /* check datalength of correctly padded CAN frame */ + if ((so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) && + cf->len != padlen(cf->len)) + return 1; + + /* check padding content */ + if (so->opt.flags & CAN_ISOTP_CHK_PAD_DATA) { + for (i = start_index; i < cf->len; i++) + if (cf->data[i] != content) + return 1; + } + return 0; +} + +static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) +{ + if (so->tx.state != ISOTP_WAIT_FC && + so->tx.state != ISOTP_WAIT_FIRST_FC) + return 0; + + hrtimer_cancel(&so->txtimer); + + if ((cf->len < ae + FC_CONTENT_SZ) || + ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) { + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + return 1; + } + + /* get communication parameters only from the first FC frame */ + if (so->tx.state == ISOTP_WAIT_FIRST_FC) { + + so->txfc.bs = cf->data[ae + 1]; + so->txfc.stmin = cf->data[ae + 2]; + + /* fix wrong STmin values according spec */ + if ((so->txfc.stmin > 0x7F) && + ((so->txfc.stmin < 0xF1) || (so->txfc.stmin > 0xF9))) + so->txfc.stmin = 0x7F; + + so->tx_gap = ktime_set(0,0); + /* add transmission time for CAN frame N_As */ + so->tx_gap = ktime_add_ns(so->tx_gap, so->opt.frame_txtime); + /* add waiting time for consecutive frames N_Cs */ + if (so->opt.flags & CAN_ISOTP_FORCE_TXSTMIN) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->force_tx_stmin); + else if (so->txfc.stmin < 0x80) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->txfc.stmin * 1000000); + else + so->tx_gap = ktime_add_ns(so->tx_gap, + (so->txfc.stmin - 0xF0) + * 100000); + so->tx.state = ISOTP_WAIT_FC; + } + + DBG("FC frame: FS %d, BS %d, STmin 0x%02X, tx_gap %lld\n", + cf->data[ae] & 0x0F & 0x0F, so->txfc.bs, so->txfc.stmin, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + (long long)so->tx_gap); +#else + (long long)so->tx_gap.tv64); +#endif + + switch (cf->data[ae] & 0x0F) { + + case ISOTP_FC_CTS: + so->tx.bs = 0; + so->tx.state = ISOTP_SENDING; + DBG("starting txtimer for sending\n"); + /* start cyclic timer for sending CF frame */ + hrtimer_start(&so->txtimer, so->tx_gap, + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_WT: + DBG("starting waiting for next FC\n"); + /* start timer to wait for next FC frame */ + hrtimer_start(&so->txtimer, ktime_set(1,0), + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_OVFLW: + DBG("overflow in receiver side\n"); + + default: + /* stop this tx job. TODO: error reporting? */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + } + return 0; +} + +static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen, + struct sk_buff *skb, int len) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + if (!len || len > cf->len - pcilen) + return 1; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, pcilen + len, so->opt.rxpad_content)) + return 1; + + nskb = alloc_skb(len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, len), &cf->data[pcilen], len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; +} + +static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) +{ + struct isotp_sock *so = isotp_sk(sk); + int i; + int off; + int ff_pci_sz; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + /* get the used sender LL_DL from the (first) CAN frame data length */ + so->rx.ll_dl = padlen(cf->len); + + /* the first frame has to use the entire frame up to LL_DL length */ + if (cf->len != so->rx.ll_dl) + return 1; + + /* get the FF_DL */ + so->rx.len = (cf->data[ae] & 0x0F) << 8; + so->rx.len += cf->data[ae + 1]; + + /* Check for FF_DL escape sequence supporting 32 bit PDU length */ + if (so->rx.len) + ff_pci_sz = FF_PCI_SZ12; + else { + /* FF_DL = 0 => get real length from next 4 bytes */ + so->rx.len = cf->data[ae + 2] << 24; + so->rx.len += cf->data[ae + 3] << 16; + so->rx.len += cf->data[ae + 4] << 8; + so->rx.len += cf->data[ae + 5]; + ff_pci_sz = FF_PCI_SZ32; + } + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->rx.ll_dl > CAN_MAX_DLEN)? 1:0; + + if (so->rx.len + ae + off + ff_pci_sz < so->rx.ll_dl) + return 1; + + if (so->rx.len > MAX_MSG_LENGTH) { + /* send FC frame with overflow status */ + isotp_send_fc(sk, ae, ISOTP_FC_OVFLW); + return 1; + } + + /* copy the first received data bytes */ + so->rx.idx = 0; + for (i = ae + ff_pci_sz; i < so->rx.ll_dl; i++) + so->rx.buf[so->rx.idx++] = cf->data[i]; + + /* initial setup for this pdu receiption */ + so->rx.sn = 1; + so->rx.state = ISOTP_WAIT_DATA; + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* send our first FC frame */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, + struct sk_buff *skb) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + int i; + + if (so->rx.state != ISOTP_WAIT_DATA) + return 0; + + /* drop if timestamp gap is less than force_rx_stmin nano secs */ + if (so->opt.flags & CAN_ISOTP_FORCE_RXSTMIN) { + + if (ktime_to_ns(ktime_sub(skb->tstamp, so->lastrxcf_tstamp)) < + so->force_rx_stmin) + return 0; + + so->lastrxcf_tstamp = skb->tstamp; + } + + hrtimer_cancel(&so->rxtimer); + + /* CFs are never longer than the FF */ + if (cf->len > so->rx.ll_dl) + return 1; + + /* CFs have usually the LL_DL length */ + if (cf->len < so->rx.ll_dl) { + /* this is only allowed for the last CF */ + if (so->rx.len - so->rx.idx > so->rx.ll_dl - ae - N_PCI_SZ) + return 1; + } + + if ((cf->data[ae] & 0x0F) != so->rx.sn) { + DBG("wrong sn %d. expected %d.\n", + cf->data[ae] & 0x0F, so->rx.sn); + /* some error reporting? */ + so->rx.state = ISOTP_IDLE; + return 1; + } + so->rx.sn++; + so->rx.sn %= 16; + + for (i = ae + N_PCI_SZ; i < cf->len; i++) { + so->rx.buf[so->rx.idx++] = cf->data[i]; + if (so->rx.idx >= so->rx.len) + break; + } + + if (so->rx.idx >= so->rx.len) { + + /* we are done */ + so->rx.state = ISOTP_IDLE; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, i+1, so->opt.rxpad_content)) + return 1; + + nskb = alloc_skb(so->rx.len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, so->rx.len), so->rx.buf, + so->rx.len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; + } + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* perform blocksize handling, if enabled */ + if (!so->rxfc.bs || ++so->rx.bs < so->rxfc.bs) { + + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1,0), + HRTIMER_MODE_REL); + return 0; + } + + /* we reached the specified blocksize so->rxfc.bs */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static void isotp_rcv(struct sk_buff *skb, void *data) +{ + struct sock *sk = (struct sock *)data; + struct isotp_sock *so = isotp_sk(sk); + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + u8 n_pci_type, sf_dl; + + BUG_ON(skb->len != CAN_MTU && skb->len != CANFD_MTU); + + /* + * Strictly receive only frames with the configured MTU size + * => clear separation of CAN2.0 / CAN FD transport channels + */ + if (skb->len != so->ll.mtu) + return; + + cf = (struct canfd_frame *) skb->data; + + /* if enabled: check receiption of my configured extended address */ + if (ae && cf->data[0] != so->opt.rx_ext_address) + return; + + n_pci_type = cf->data[ae] & 0xF0; + + if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) { + /* check rx/tx path half duplex expectations */ + if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) || + (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC)) + return; + } + + switch (n_pci_type) { + case N_PCI_FC: + /* tx path: flow control frame containing the FC parameters */ + isotp_rcv_fc(so, cf, ae); + break; + + case N_PCI_SF: + /* + * rx path: single frame + * + * As we do not have a rx.ll_dl configuration, we can only test + * if the CAN frames payload length matches the LL_DL == 8 + * requirements - no matter if it's CAN 2.0 or CAN FD + */ + + /* get the SF_DL from the N_PCI byte */ + sf_dl = cf->data[ae] & 0x0F; + + if (cf->len <= CAN_MAX_DLEN) + isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl); + else if (skb->len == CANFD_MTU) { + /* + * We have a CAN FD frame and CAN_DL is greater than 8: + * Only frames with the SF_DL == 0 ESC value are valid. + * + * If so take care of the increased SF PCI size + * (SF_PCI_SZ8) to point to the message content behind + * the extended SF PCI info and get the real SF_DL + * length value from the formerly first data byte. + */ + if (sf_dl == 0) + isotp_rcv_sf(sk, cf, SF_PCI_SZ8 + ae, skb, + cf->data[SF_PCI_SZ4 + ae]); + } + break; + + case N_PCI_FF: + /* rx path: first frame */ + isotp_rcv_ff(sk, cf, ae); + break; + + case N_PCI_CF: + /* rx path: consecutive frame */ + isotp_rcv_cf(sk, cf, ae, skb); + break; + + } +} + +static void isotp_fill_dataframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae, int off) +{ + int pcilen = N_PCI_SZ + ae + off; + int space = so->tx.ll_dl - pcilen; + int num = min_t(int, so->tx.len - so->tx.idx, space); + int i; + + cf->can_id = so->txid; + cf->len = num + pcilen; + + if (num < space) { + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + /* user requested padding */ + cf->len = padlen(cf->len); + memset(cf->data, so->opt.txpad_content, cf->len); + } else if (cf->len > CAN_MAX_DLEN) { + /* mandatory padding for CAN FD frames */ + cf->len = padlen(cf->len); + memset(cf->data, CAN_ISOTP_DEFAULT_PAD_CONTENT, + cf->len); + } + } + + for (i = 0; i < num; i++) + cf->data[pcilen + i] = so->tx.buf[so->tx.idx++]; + + if (ae) + cf->data[0] = so->opt.ext_address; +} + +static void isotp_create_fframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae) +{ + int i; + int ff_pci_sz; + + cf->can_id = so->txid; + cf->len = so->tx.ll_dl; + if (ae) + cf->data[0] = so->opt.ext_address; + + /* create N_PCI bytes with 12/32 bit FF_DL data length */ + if (so->tx.len > 4095) { + /* use 32 bit FF_DL notation */ + cf->data[ae] = N_PCI_FF; + cf->data[ae + 1] = 0; + cf->data[ae + 2] = (u8) (so->tx.len >> 24) & 0xFFU; + cf->data[ae + 3] = (u8) (so->tx.len >> 16) & 0xFFU; + cf->data[ae + 4] = (u8) (so->tx.len >> 8) & 0xFFU; + cf->data[ae + 5] = (u8) so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ32; + } else { + /* use 12 bit FF_DL notation */ + cf->data[ae] = (u8) (so->tx.len>>8) | N_PCI_FF; + cf->data[ae + 1] = (u8) so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ12; + } + + /* add first data bytes depending on ae */ + for (i = ae + ff_pci_sz; i < so->tx.ll_dl; i++) + cf->data[i] = so->tx.buf[so->tx.idx++]; + + so->tx.sn = 1; + so->tx.state = ISOTP_WAIT_FIRST_FC; +} + +static void isotp_tx_timer_tsklet(unsigned long data) +{ + struct isotp_sock *so = (struct isotp_sock *)data; + struct sock *sk = &so->sk; + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + + switch (so->tx.state) { + + case ISOTP_WAIT_FC: + case ISOTP_WAIT_FIRST_FC: + + /* we did not get any flow control frame in time */ + + DBG("we did not get FC frame in time.\n"); + + /* reset tx state */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + break; + + case ISOTP_SENDING: + + /* push out the next segmented pdu */ + + DBG("next pdu to send.\n"); + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + break; + +isotp_tx_burst: + skb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any()); + if (!skb) { + dev_put(dev); + break; + } + + isotp_skb_reserve(skb, dev); + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* create consecutive frame */ + isotp_fill_dataframe(cf, so, ae, 0); + + /* place consecutive frame N_PCI in appropriate index */ + cf->data[ae] = N_PCI_CF | so->tx.sn++; + so->tx.sn %= 16; + so->tx.bs++; + + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + isotp_skb_set_owner(skb, sk); + can_send(skb, 1); + + if (so->tx.idx >= so->tx.len) { + /* we are done */ + DBG("we are done\n"); + so->tx.state = ISOTP_IDLE; + dev_put(dev); + wake_up_interruptible(&so->wait); + break; + } + + if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { + /* stop and wait for FC */ + DBG("BS stop and wait for FC\n"); + so->tx.state = ISOTP_WAIT_FC; + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), ktime_set(1,0)), + HRTIMER_MODE_ABS); + break; + } + + /* no gap between data frames needed => use burst mode */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (!so->tx_gap) + goto isotp_tx_burst; +#else + if (!so->tx_gap.tv64) + goto isotp_tx_burst; +#endif + + /* start timer to send next data frame with correct delay */ + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), so->tx_gap), + HRTIMER_MODE_ABS); + break; + + default: + BUG_ON(1); + } +} + +static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + txtimer); + tasklet_schedule(&so->txtsklet); + + return HRTIMER_NORESTART; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) +#else +static int isotp_sendmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + int off; + int err; + + if (!so->bound) + return -EADDRNOTAVAIL; + + /* we do not support multiple buffers - for now */ + if (so->tx.state != ISOTP_IDLE) { + if (msg->msg_flags & MSG_DONTWAIT) + return -EAGAIN; + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + } + + if (!size || size > MAX_MSG_LENGTH) + return -EINVAL; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_from_msg(so->tx.buf, msg, size); +#else + err = memcpy_fromiovec(so->tx.buf, msg->msg_iov, size); +#endif + if (err < 0) + return err; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + return -ENXIO; + + skb = sock_alloc_send_skb(sk, so->ll.mtu + CAN_SKBRES, + msg->msg_flags & MSG_DONTWAIT, &err); + if (!skb) { + dev_put(dev); + return err; + } + + isotp_skb_reserve(skb, dev); + + so->tx.state = ISOTP_SENDING; + so->tx.len = size; + so->tx.idx = 0; + + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->tx.ll_dl > CAN_MAX_DLEN)? 1:0; + + /* check for single frame transmission depending on TX_DL */ + if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { + + /* + * The message size generally fits into a SingleFrame - good. + * + * SF_DL ESC offset optimization: + * + * When TX_DL is greater 8 but the message would still fit + * into a 8 byte CAN frame, we can omit the offset. + * This prevents a protocol caused length extension from + * CAN_DL = 8 to CAN_DL = 12 due to the SF_SL ESC handling. + */ + if (size <= CAN_MAX_DLEN - SF_PCI_SZ4 - ae) + off = 0; + + isotp_fill_dataframe(cf, so, ae, off); + + /* place single frame N_PCI w/o length in appropriate index */ + cf->data[ae] = N_PCI_SF; + + /* place SF_DL size value depending on the SF_DL ESC offset */ + if (off) + cf->data[SF_PCI_SZ4 + ae] = size; + else + cf->data[ae] |= size; + + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + } else { + /* send first frame and wait for FC */ + + isotp_create_fframe(cf, so, ae); + + DBG("starting txtimer for fc\n"); + /* start timeout for FC */ + hrtimer_start(&so->txtimer, ktime_set(1,0), HRTIMER_MODE_REL); + } + + /* send the first or only CAN frame */ + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + skb->sk = sk; + err = can_send(skb, 1); + dev_put(dev); + if (err) + return err; + + return size; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, + int flags) +#else +static int isotp_recvmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size, int flags) +#endif +{ + struct sock *sk = sock->sk; + struct sk_buff *skb; + int err = 0; + int noblock; + + noblock = flags & MSG_DONTWAIT; + flags &= ~MSG_DONTWAIT; + + skb = skb_recv_datagram(sk, flags, noblock, &err); + if (!skb) + return err; + + if (size < skb->len) + msg->msg_flags |= MSG_TRUNC; + else + size = skb->len; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_to_msg(msg, skb->data, size); +#else + err = memcpy_toiovec(msg->msg_iov, skb->data, size); +#endif + if (err < 0) { + skb_free_datagram(sk, skb); + return err; + } + + sock_recv_timestamp(msg, sk, skb); + + if (msg->msg_name) { + msg->msg_namelen = sizeof(struct sockaddr_can); + memcpy(msg->msg_name, skb->cb, msg->msg_namelen); + } + + skb_free_datagram(sk, skb); + + return size; +} + +static int isotp_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so; + struct net *net; + + if (!sk) + return 0; + + so = isotp_sk(sk); + net = sock_net(sk); + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + + unregister_netdevice_notifier(&so->notifier); + + lock_sock(sk); + + hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->rxtimer); + tasklet_kill(&so->txtsklet); + + /* remove current filters & unregister */ + if (so->bound) { + if (so->ifindex) { + struct net_device *dev; + + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + so->ifindex = 0; + so->bound = 0; + + sock_orphan(sk); + sock->sk = NULL; + + release_sock(sk); + sock_put(sk); + + return 0; +} + +static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct net *net = sock_net(sk); + int ifindex; + struct net_device *dev; + int err = 0; + int notify_enetdown = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) + if (len < CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.tp)) +#else + if (len < sizeof(*addr)) +#endif + return -EINVAL; + + if (addr->can_addr.tp.rx_id == addr->can_addr.tp.tx_id) + return -EADDRNOTAVAIL; + + if ((addr->can_addr.tp.rx_id | addr->can_addr.tp.tx_id) & + (CAN_ERR_FLAG | CAN_RTR_FLAG)) + return -EADDRNOTAVAIL; + + if (!addr->can_ifindex) + return -ENODEV; + + lock_sock(sk); + + if (so->bound && addr->can_ifindex == so->ifindex && + addr->can_addr.tp.rx_id == so->rxid && + addr->can_addr.tp.tx_id == so->txid) + goto out; + + dev = dev_get_by_index(net, addr->can_ifindex); + if (!dev) { + err = -ENODEV; + goto out; + } + if (dev->type != ARPHRD_CAN) { + dev_put(dev); + err = -ENODEV; + goto out; + } + if (dev->mtu < so->ll.mtu) { + dev_put(dev); + err = -EINVAL; + goto out; + } + if (!(dev->flags & IFF_UP)) + notify_enetdown = 1; + + ifindex = dev->ifindex; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_register(net, dev, addr->can_addr.tp.rx_id, +#else + can_rx_register(dev, addr->can_addr.tp.rx_id, +#endif + SINGLE_MASK(addr->can_addr.tp.rx_id), isotp_rcv, sk, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,11)) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,50)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,49)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,42)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))) + "isotp", sk); +#else + "isotp"); +#endif + dev_put(dev); + + if (so->bound) { + /* unregister old filter */ + if (so->ifindex) { + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + /* switch to new settings */ + so->ifindex = ifindex; + so->rxid = addr->can_addr.tp.rx_id; + so->txid = addr->can_addr.tp.tx_id; + so->bound = 1; + +out: + release_sock(sk); + + if (notify_enetdown) { + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + } + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, int peer) +#else +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, + int *len, int peer) +#endif +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + + if (peer) + return -EOPNOTSUPP; + + addr->can_family = AF_CAN; + addr->can_ifindex = so->ifindex; + addr->can_addr.tp.rx_id = so->rxid; + addr->can_addr.tp.tx_id = so->txid; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) + return sizeof(*addr); +#else + *len = sizeof(*addr); + + return 0; +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, unsigned int optlen) +#else +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, int optlen) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int ret = 0; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + if (optlen < 0) + return -EINVAL; + + switch (optname) { + + case CAN_ISOTP_OPTS: + if (optlen != sizeof(struct can_isotp_options)) + return -EINVAL; + + if (copy_from_user(&so->opt, optval, optlen)) + return -EFAULT; + + /* no separate rx_ext_address is given => use ext_address */ + if (!(so->opt.flags & CAN_ISOTP_RX_EXT_ADDR)) + so->opt.rx_ext_address = so->opt.ext_address; + break; + + case CAN_ISOTP_RECV_FC: + if (optlen != sizeof(struct can_isotp_fc_options)) + return -EINVAL; + + if (copy_from_user(&so->rxfc, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_TX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_tx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_RX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_rx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_LL_OPTS: + if (optlen != sizeof(struct can_isotp_ll_options)) + return -EINVAL; + else { + struct can_isotp_ll_options ll; + + if (copy_from_user(&ll, optval, optlen)) + return -EFAULT; + + /* check for correct ISO 11898-1 DLC data lentgh */ + if (ll.tx_dl != padlen(ll.tx_dl)) + return -EINVAL; + + if (ll.mtu != CAN_MTU && ll.mtu != CANFD_MTU) + return -EINVAL; + + if (ll.mtu == CAN_MTU && ll.tx_dl > CAN_MAX_DLEN) + return -EINVAL; + + memcpy(&so->ll, &ll, sizeof(ll)); + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = ll.tx_dl; + } + break; + + default: + ret = -ENOPROTOOPT; + } + + return ret; +} + +static int isotp_getsockopt(struct socket *sock, int level, int optname, + char __user *optval, int __user *optlen) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int len; + void *val; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + switch (optname) { + + case CAN_ISOTP_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_options)); + val = &so->opt; + break; + + case CAN_ISOTP_RECV_FC: + len = min_t(int, len, sizeof(struct can_isotp_fc_options)); + val = &so->rxfc; + break; + + case CAN_ISOTP_TX_STMIN: + len = min_t(int, len, sizeof(__u32)); + val = &so->force_tx_stmin; + break; + + case CAN_ISOTP_RX_STMIN: + len = min_t(int, len, sizeof(__u32)); + val = &so->force_rx_stmin; + break; + + case CAN_ISOTP_LL_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_ll_options)); + val = &so->ll; + break; + + default: + return -ENOPROTOOPT; + } + + if (put_user(len, optlen)) + return -EFAULT; + if (copy_to_user(optval, val, len)) + return -EFAULT; + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0) +static int isotp_notifier(struct notifier_block *nb, unsigned long msg, + void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); +#else +static int isotp_notifier(struct notifier_block *nb, + unsigned long msg, void *data) +{ + struct net_device *dev = (struct net_device *)data; +#endif + struct isotp_sock *so = container_of(nb, struct isotp_sock, notifier); + struct sock *sk = &so->sk; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + if (!net_eq(dev_net(dev), sock_net(sk))) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + if (dev_net(dev) != &init_net) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) + if (dev->nd_net != &init_net) + return NOTIFY_DONE; +#endif + + if (dev->type != ARPHRD_CAN) + return NOTIFY_DONE; + + if (so->ifindex != dev->ifindex) + return NOTIFY_DONE; + + switch (msg) { + + case NETDEV_UNREGISTER: + lock_sock(sk); + /* remove current filters & unregister */ + if (so->bound) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(dev_net(dev), dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + + so->ifindex = 0; + so->bound = 0; + release_sock(sk); + + sk->sk_err = ENODEV; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + + case NETDEV_DOWN: + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + } + + return NOTIFY_DONE; +} + +static int isotp_init(struct sock *sk) +{ + struct isotp_sock *so = isotp_sk(sk); + + so->ifindex = 0; + so->bound = 0; + + so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS; + so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rx_ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rxpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.txpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.frame_txtime = CAN_ISOTP_DEFAULT_FRAME_TXTIME; + so->rxfc.bs = CAN_ISOTP_DEFAULT_RECV_BS; + so->rxfc.stmin = CAN_ISOTP_DEFAULT_RECV_STMIN; + so->rxfc.wftmax = CAN_ISOTP_DEFAULT_RECV_WFTMAX; + so->ll.mtu = CAN_ISOTP_DEFAULT_LL_MTU; + so->ll.tx_dl = CAN_ISOTP_DEFAULT_LL_TX_DL; + so->ll.tx_flags = CAN_ISOTP_DEFAULT_LL_TX_FLAGS; + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = so->ll.tx_dl; + + so->rx.state = ISOTP_IDLE; + so->tx.state = ISOTP_IDLE; + + hrtimer_init(&so->rxtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->rxtimer.function = isotp_rx_timer_handler; + hrtimer_init(&so->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->txtimer.function = isotp_tx_timer_handler; + + tasklet_init(&so->txtsklet, isotp_tx_timer_tsklet, (unsigned long)so); + + init_waitqueue_head(&so->wait); + + so->notifier.notifier_call = isotp_notifier; + register_netdevice_notifier(&so->notifier); + + return 0; +} + +int isotp_sock_no_ioctlcmd(struct socket *sock, unsigned int cmd, + unsigned long arg) +{ + /* no ioctls for socket layer -> hand it down to NIC layer */ + return -ENOIOCTLCMD; +} + +static const struct proto_ops isotp_ops = { + .family = PF_CAN, + .release = isotp_release, + .bind = isotp_bind, + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .accept = sock_no_accept, + .getname = isotp_getname, + .poll = datagram_poll, + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,2,0) + .ioctl = isotp_sock_no_ioctlcmd, + .gettstamp = sock_gettstamp, +#else + .ioctl = can_ioctl, /* use can_ioctl() from af_can.c */ +#endif + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .setsockopt = isotp_setsockopt, + .getsockopt = isotp_getsockopt, + .sendmsg = isotp_sendmsg, + .recvmsg = isotp_recvmsg, + .mmap = sock_no_mmap, + .sendpage = sock_no_sendpage, +}; + +static struct proto isotp_proto __read_mostly = { + .name = "CAN_ISOTP", + .owner = THIS_MODULE, + .obj_size = sizeof(struct isotp_sock), + .init = isotp_init, +}; + +static const struct can_proto isotp_can_proto = { + .type = SOCK_DGRAM, + .protocol = CAN_ISOTP, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .capability = -1, +#endif + .ops = &isotp_ops, + .prot = &isotp_proto, +}; + +static __init int isotp_module_init(void) +{ + int err; + + printk(banner); + + err = can_proto_register(&isotp_can_proto); + if (err < 0) + printk(KERN_ERR "can: registration of isotp protocol failed\n"); + + return err; +} + +static __exit void isotp_module_exit(void) +{ + can_proto_unregister(&isotp_can_proto); +} + +module_init(isotp_module_init); +module_exit(isotp_module_exit); diff --git a/net/wireguard/Kconfig b/net/wireguard/Kconfig new file mode 100644 index 0000000000000..156e9dbfc0516 --- /dev/null +++ b/net/wireguard/Kconfig @@ -0,0 +1,33 @@ +config WIREGUARD + tristate "IP: WireGuard secure network tunnel" + depends on NET && INET + depends on IPV6 || !IPV6 + select NET_UDP_TUNNEL + select DST_CACHE + select CRYPTO + select CRYPTO_ALGAPI + select VFP + select VFPv3 if CPU_V7 + select NEON if CPU_V7 + select KERNEL_MODE_NEON if CPU_V7 + default m + help + WireGuard is a secure, fast, and easy to use replacement for IPsec + that uses modern cryptography and clever networking tricks. It's + designed to be fairly general purpose and abstract enough to fit most + use cases, while at the same time remaining extremely simple to + configure. See www.wireguard.com for more info. + + It's safe to say Y or M here, as the driver is very lightweight and + is only in use when an administrator chooses to add an interface. + +config WIREGUARD_DEBUG + bool "Debugging checks and verbose messages" + depends on WIREGUARD + help + This will write log messages for handshake and other events + that occur for a WireGuard interface. It will also perform some + extra validation checks and unit tests at various points. This is + only useful for debugging. + + Say N here unless you know what you're doing. diff --git a/net/wireguard/Makefile b/net/wireguard/Makefile new file mode 100644 index 0000000000000..364deb7053766 --- /dev/null +++ b/net/wireguard/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +ccflags-y := -O3 -fvisibility=hidden +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG -g +ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' +ccflags-y += -Wframe-larger-than=2048 + +wireguard-y := main.o noise.o device.o peer.o timers.o queueing.o send.o receive.o socket.o peerlookup.o allowedips.o ratelimiter.o cookie.o netlink.o + +include $(src)/crypto/Makefile.include +include $(src)/compat/Makefile.include + +obj-$(if $(KBUILD_EXTMOD),m,$(CONFIG_WIREGUARD)) := wireguard.o diff --git a/net/wireguard/allowedips.c b/net/wireguard/allowedips.c new file mode 100644 index 0000000000000..72667d5399c34 --- /dev/null +++ b/net/wireguard/allowedips.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "allowedips.h" +#include "peer.h" + +static void swap_endian(u8 *dst, const u8 *src, u8 bits) +{ + if (bits == 32) { + *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); + } else if (bits == 128) { + ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); + ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); + } +} + +static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, + u8 cidr, u8 bits) +{ + node->cidr = cidr; + node->bit_at_a = cidr / 8U; +#ifdef __LITTLE_ENDIAN + node->bit_at_a ^= (bits / 8U - 1U) % 8U; +#endif + node->bit_at_b = 7U - (cidr % 8U); + node->bitlen = bits; + memcpy(node->bits, src, bits / 8U); +} +#define CHOOSE_NODE(parent, key) \ + parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1] + +static void node_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct allowedips_node, rcu)); +} + +static void push_rcu(struct allowedips_node **stack, + struct allowedips_node __rcu *p, unsigned int *len) +{ + if (rcu_access_pointer(p)) { + WARN_ON(IS_ENABLED(DEBUG) && *len >= 128); + stack[(*len)++] = rcu_dereference_raw(p); + } +} + +static void root_free_rcu(struct rcu_head *rcu) +{ + struct allowedips_node *node, *stack[128] = { + container_of(rcu, struct allowedips_node, rcu) }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + kfree(node); + } +} + +static void root_remove_peer_lists(struct allowedips_node *root) +{ + struct allowedips_node *node, *stack[128] = { root }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + if (rcu_access_pointer(node->peer)) + list_del(&node->peer_list); + } +} + +static void walk_remove_by_peer(struct allowedips_node __rcu **top, + struct wg_peer *peer, struct mutex *lock) +{ +#define REF(p) rcu_access_pointer(p) +#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock)) +#define PUSH(p) ({ \ + WARN_ON(IS_ENABLED(DEBUG) && len >= 128); \ + stack[len++] = p; \ + }) + + struct allowedips_node __rcu **stack[128], **nptr; + struct allowedips_node *node, *prev; + unsigned int len; + + if (unlikely(!peer || !REF(*top))) + return; + + for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) { + nptr = stack[len - 1]; + node = DEREF(nptr); + if (!node) { + --len; + continue; + } + if (!prev || REF(prev->bit[0]) == node || + REF(prev->bit[1]) == node) { + if (REF(node->bit[0])) + PUSH(&node->bit[0]); + else if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else if (REF(node->bit[0]) == prev) { + if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else { + if (rcu_dereference_protected(node->peer, + lockdep_is_held(lock)) == peer) { + RCU_INIT_POINTER(node->peer, NULL); + list_del_init(&node->peer_list); + if (!node->bit[0] || !node->bit[1]) { + rcu_assign_pointer(*nptr, DEREF( + &node->bit[!REF(node->bit[0])])); + call_rcu(&node->rcu, node_free_rcu); + node = DEREF(nptr); + } + } + --len; + } + } + +#undef REF +#undef DEREF +#undef PUSH +} + +static unsigned int fls128(u64 a, u64 b) +{ + return a ? fls64(a) + 64U : fls64(b); +} + +static u8 common_bits(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + if (bits == 32) + return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); + else if (bits == 128) + return 128U - fls128( + *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], + *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); + return 0; +} + +static bool prefix_matches(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + /* This could be much faster if it actually just compared the common + * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and + * the rest, but it turns out that common_bits is already super fast on + * modern processors, even taking into account the unfortunate bswap. + * So, we just inline it like this instead. + */ + return common_bits(node, key, bits) >= node->cidr; +} + +static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, + const u8 *key) +{ + struct allowedips_node *node = trie, *found = NULL; + + while (node && prefix_matches(node, key, bits)) { + if (rcu_access_pointer(node->peer)) + found = node; + if (node->cidr == bits) + break; + node = rcu_dereference_bh(CHOOSE_NODE(node, key)); + } + return found; +} + +/* Returns a strong reference to a peer */ +static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, + const void *be_ip) +{ + /* Aligned so it can be passed to fls/fls64 */ + u8 ip[16] __aligned(__alignof(u64)); + struct allowedips_node *node; + struct wg_peer *peer = NULL; + + swap_endian(ip, be_ip, bits); + + rcu_read_lock_bh(); +retry: + node = find_node(rcu_dereference_bh(root), bits, ip); + if (node) { + peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer)); + if (!peer) + goto retry; + } + rcu_read_unlock_bh(); + return peer; +} + +static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, + u8 cidr, u8 bits, struct allowedips_node **rnode, + struct mutex *lock) +{ + struct allowedips_node *node = rcu_dereference_protected(trie, + lockdep_is_held(lock)); + struct allowedips_node *parent = NULL; + bool exact = false; + + while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) { + parent = node; + if (parent->cidr == cidr) { + exact = true; + break; + } + node = rcu_dereference_protected(CHOOSE_NODE(parent, key), + lockdep_is_held(lock)); + } + *rnode = parent; + return exact; +} + +static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + struct allowedips_node *node, *parent, *down, *newnode; + + if (unlikely(cidr > bits || !peer)) + return -EINVAL; + + if (!rcu_access_pointer(*trie)) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) + return -ENOMEM; + RCU_INIT_POINTER(node->peer, peer); + list_add_tail(&node->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(node, key, cidr, bits); + rcu_assign_pointer(*trie, node); + return 0; + } + if (node_placement(*trie, key, cidr, bits, &node, lock)) { + rcu_assign_pointer(node->peer, peer); + list_move_tail(&node->peer_list, &peer->allowedips_list); + return 0; + } + + newnode = kzalloc(sizeof(*newnode), GFP_KERNEL); + if (unlikely(!newnode)) + return -ENOMEM; + RCU_INIT_POINTER(newnode->peer, peer); + list_add_tail(&newnode->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(newnode, key, cidr, bits); + + if (!node) { + down = rcu_dereference_protected(*trie, lockdep_is_held(lock)); + } else { + down = rcu_dereference_protected(CHOOSE_NODE(node, key), + lockdep_is_held(lock)); + if (!down) { + rcu_assign_pointer(CHOOSE_NODE(node, key), newnode); + return 0; + } + } + cidr = min(cidr, common_bits(down, key, bits)); + parent = node; + + if (newnode->cidr == cidr) { + rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down); + if (!parent) + rcu_assign_pointer(*trie, newnode); + else + rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits), + newnode); + } else { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) { + kfree(newnode); + return -ENOMEM; + } + INIT_LIST_HEAD(&node->peer_list); + copy_and_assign_cidr(node, newnode->bits, cidr, bits); + + rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down); + rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode); + if (!parent) + rcu_assign_pointer(*trie, node); + else + rcu_assign_pointer(CHOOSE_NODE(parent, node->bits), + node); + } + return 0; +} + +void wg_allowedips_init(struct allowedips *table) +{ + table->root4 = table->root6 = NULL; + table->seq = 1; +} + +void wg_allowedips_free(struct allowedips *table, struct mutex *lock) +{ + struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; + + ++table->seq; + RCU_INIT_POINTER(table->root4, NULL); + RCU_INIT_POINTER(table->root6, NULL); + if (rcu_access_pointer(old4)) { + struct allowedips_node *node = rcu_dereference_protected(old4, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } + if (rcu_access_pointer(old6)) { + struct allowedips_node *node = rcu_dereference_protected(old6, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } +} + +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls */ + u8 key[4] __aligned(__alignof(u32)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 32); + return add(&table->root4, 32, key, cidr, peer, lock); +} + +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls64 */ + u8 key[16] __aligned(__alignof(u64)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 128); + return add(&table->root6, 128, key, cidr, peer, lock); +} + +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock) +{ + ++table->seq; + walk_remove_by_peer(&table->root4, peer, lock); + walk_remove_by_peer(&table->root6, peer, lock); +} + +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) +{ + const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); + swap_endian(ip, node->bits, node->bitlen); + memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); + if (node->cidr) + ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); + + *cidr = node->cidr; + return node->bitlen == 32 ? AF_INET : AF_INET6; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); + return NULL; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->saddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); + return NULL; +} + +#include "selftest/allowedips.c" diff --git a/net/wireguard/allowedips.h b/net/wireguard/allowedips.h new file mode 100644 index 0000000000000..e5c83cafcef4c --- /dev/null +++ b/net/wireguard/allowedips.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ALLOWEDIPS_H +#define _WG_ALLOWEDIPS_H + +#include +#include +#include + +struct wg_peer; + +struct allowedips_node { + struct wg_peer __rcu *peer; + struct allowedips_node __rcu *bit[2]; + /* While it may seem scandalous that we waste space for v4, + * we're alloc'ing to the nearest power of 2 anyway, so this + * doesn't actually make a difference. + */ + u8 bits[16] __aligned(__alignof(u64)); + u8 cidr, bit_at_a, bit_at_b, bitlen; + + /* Keep rarely used list at bottom to be beyond cache line. */ + union { + struct list_head peer_list; + struct rcu_head rcu; + }; +}; + +struct allowedips { + struct allowedips_node __rcu *root4; + struct allowedips_node __rcu *root6; + u64 seq; +}; + +void wg_allowedips_init(struct allowedips *table); +void wg_allowedips_free(struct allowedips *table, struct mutex *mutex); +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock); +/* The ip input pointer should be __aligned(__alignof(u64))) */ +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr); + +/* These return a strong reference to a peer: */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb); +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb); + +#ifdef DEBUG +bool wg_allowedips_selftest(void); +#endif + +#endif /* _WG_ALLOWEDIPS_H */ diff --git a/net/wireguard/compat/Makefile.include b/net/wireguard/compat/Makefile.include new file mode 100644 index 0000000000000..7bf345b5db524 --- /dev/null +++ b/net/wireguard/compat/Makefile.include @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) + +ccflags-y += -include $(kbuild-dir)/compat/compat.h +asflags-y += -include $(kbuild-dir)/compat/compat-asm.h + +ifeq ($(wildcard $(srctree)/include/linux/ptr_ring.h),) +ccflags-y += -I$(kbuild-dir)/compat/ptr_ring/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/siphash.h),) +ccflags-y += -I$(kbuild-dir)/compat/siphash/include +wireguard-y += compat/siphash/siphash.o +endif + +ifeq ($(wildcard $(srctree)/include/net/dst_cache.h),) +ccflags-y += -I$(kbuild-dir)/compat/dst_cache/include +wireguard-y += compat/dst_cache/dst_cache.o +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/intel-family.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/intel-family-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/fpu/api.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/fpu-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/simd.h)$(shell grep -s -F "generic-y += simd.h" "$(srctree)/arch/$(SRCARCH)/Makefile" "$(srctree)/arch/$(SRCARCH)/Makefile"),) +ccflags-y += -I$(kbuild-dir)/compat/simd-asm/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/simd.h),) +ccflags-y += -I$(kbuild-dir)/compat/simd/include +endif + +ifeq ($(wildcard $(srctree)/include/net/udp_tunnel.h),) +ccflags-y += -I$(kbuild-dir)/compat/udp_tunnel/include +wireguard-y += compat/udp_tunnel/udp_tunnel.o +endif + +ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h"),) +ccflags-y += -include $(kbuild-dir)/compat/memneq/include.h +wireguard-y += compat/memneq/memneq.o +endif + +ifeq ($(shell grep -s -F "addr_gen_mode" "$(srctree)/include/linux/ipv6.h"),) +ccflags-y += -DCOMPAT_CANNOT_USE_DEV_CNF +endif + +ifdef CONFIG_HZ +ifeq ($(wildcard $(srctree)/include/generated/timeconst.h),) +ccflags-y += $(shell echo 'define gcd(a,b){auto t;while(b){t=b;b=a%b;a=t;};return a;};hz=$(CONFIG_HZ);cd=gcd(hz,1000000);print "-DHZ_TO_USEC_NUM=",1000000/cd," -DHZ_TO_USEC_DEN=",hz/cd;halt;' | bc -q) +endif +endif + +ifeq ($(wildcard $(srctree)/arch/arm/include/asm/neon.h)$(CONFIG_ARM),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif +ifeq ($(wildcard $(srctree)/arch/arm64/include/asm/neon.h)$(CONFIG_ARM64),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif + +ifeq ($(CONFIG_X86_64),y) + ifeq ($(ssse3_instr),) + ssse3_instr := $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) + ccflags-y += $(ssse3_instr) + asflags-y += $(ssse3_instr) + endif + ifeq ($(avx_instr),) + avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1) + ccflags-y += $(avx_instr) + asflags-y += $(avx_instr) + endif + ifeq ($(avx2_instr),) + avx2_instr := $(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) + ccflags-y += $(avx2_instr) + asflags-y += $(avx2_instr) + endif + ifeq ($(avx512_instr),) + avx512_instr := $(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) + ccflags-y += $(avx512_instr) + asflags-y += $(avx512_instr) + endif + ifeq ($(bmi2_instr),) + bmi2_instr :=$(call as-instr,mulx %rax$(comma)%rax$(comma)%rax,-DCONFIG_AS_BMI2=1) + ccflags-y += $(bmi2_instr) + asflags-y += $(bmi2_instr) + endif + ifeq ($(adx_instr),) + adx_instr :=$(call as-instr,adcx %rax$(comma)%rax,-DCONFIG_AS_ADX=1) + ccflags-y += $(adx_instr) + asflags-y += $(adx_instr) + endif +endif diff --git a/net/wireguard/compat/checksum/checksum_partial_compat.h b/net/wireguard/compat/checksum/checksum_partial_compat.h new file mode 100644 index 0000000000000..3dfe5397a94f5 --- /dev/null +++ b/net/wireguard/compat/checksum/checksum_partial_compat.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include +#include + +#define IP6_MF 0x0001 +#define IP6_OFFSET 0xFFF8 +static inline int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, unsigned int max) +{ + if (skb_headlen(skb) >= len) + return 0; + if (max > skb->len) + max = skb->len; + if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) + return -ENOMEM; + if (skb_headlen(skb) < len) + return -EPROTO; + return 0; +} +#define MAX_IP_HDR_LEN 128 +static inline int skb_checksum_setup_ip(struct sk_buff *skb, bool recalculate) +{ + unsigned int off; + bool fragment; + int err; + fragment = false; + err = skb_maybe_pull_tail(skb, sizeof(struct iphdr), MAX_IP_HDR_LEN); + if (err < 0) + goto out; + if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) + fragment = true; + off = ip_hdrlen(skb); + err = -EPROTO; + if (fragment) + goto out; + switch (ip_hdr(skb)->protocol) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +#define MAX_IPV6_HDR_LEN 256 +#define OPT_HDR(type, skb, off) \ + (type *)(skb_network_header(skb) + (off)) +static inline int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) +{ + int err; + u8 nexthdr; + unsigned int off; + unsigned int len; + bool fragment; + bool done; + fragment = false; + done = false; + off = sizeof(struct ipv6hdr); + err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + nexthdr = ipv6_hdr(skb)->nexthdr; + len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); + while (off <= len && !done) { + switch (nexthdr) { + case IPPROTO_DSTOPTS: + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: { + struct ipv6_opt_hdr *hp; + + err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); + nexthdr = hp->nexthdr; + off += ipv6_optlen(hp); + break; + } + case IPPROTO_FRAGMENT: { + struct frag_hdr *hp; + err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct frag_hdr, skb, off); + if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) + fragment = true; + nexthdr = hp->nexthdr; + off += sizeof(struct frag_hdr); + break; + } + default: + done = true; + break; + } + } + err = -EPROTO; + if (!done || fragment) + goto out; + switch (nexthdr) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +static inline int skb_checksum_setup(struct sk_buff *skb, bool recalculate) +{ + int err; + switch (skb->protocol) { + case htons(ETH_P_IP): + err = skb_checksum_setup_ip(skb, recalculate); + break; + + case htons(ETH_P_IPV6): + err = skb_checksum_setup_ipv6(skb, recalculate); + break; + default: + err = -EPROTO; + break; + } + return err; +} diff --git a/net/wireguard/compat/compat-asm.h b/net/wireguard/compat/compat-asm.h new file mode 100644 index 0000000000000..bafd70beb68f4 --- /dev/null +++ b/net/wireguard/compat/compat-asm.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPATASM_H +#define _WG_COMPATASM_H + +#include +#include +#include + +/* PaX compatibility */ +#if defined(RAP_PLUGIN) +#undef ENTRY +#define ENTRY RAP_ENTRY +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo + .macro ret\c, reg +#if __LINUX_ARM_ARCH__ < 6 + mov\c pc, \reg +#else + .ifeqs "\reg", "lr" + bx\c \reg + .else + mov\c pc, \reg + .endif +#endif + .endm + .endr +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#include +#define lspush push +#define lspull pull +#undef push +#undef pull +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define SYM_FUNC_START ENTRY +#define SYM_FUNC_END ENDPROC +#endif + +#endif /* _WG_COMPATASM_H */ diff --git a/net/wireguard/compat/compat.h b/net/wireguard/compat/compat.h new file mode 100644 index 0000000000000..824f57cad01f2 --- /dev/null +++ b/net/wireguard/compat/compat.h @@ -0,0 +1,945 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPAT_H +#define _WG_COMPAT_H + +#include +#include +#include +#include + +#ifdef RHEL_MAJOR +#if RHEL_MAJOR == 7 +#define ISRHEL7 +#elif RHEL_MAJOR == 8 +#define ISRHEL8 +#endif +#endif +#ifdef UTS_UBUNTU_RELEASE_ABI +#if LINUX_VERSION_CODE == KERNEL_VERSION(3, 13, 11) +#define ISUBUNTU1404 +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#define ISUBUNTU1604 +#endif +#endif +#ifdef CONFIG_SUSE_KERNEL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#define ISOPENSUSE42 +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) +#define ISOPENSUSE15 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#error "WireGuard requires Linux >= 3.10" +#endif + +#if defined(ISRHEL7) +#include +#define headers_end headers_start +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define headers_start data +#define headers_end data +#endif + +#include +#include +#ifndef __ro_after_init +#define __ro_after_init __read_mostly +#endif + +#include +#ifndef READ_ONCE +#define READ_ONCE ACCESS_ONCE +#endif +#ifndef WRITE_ONCE +#ifdef ACCESS_ONCE_RW +#define WRITE_ONCE(p, v) (ACCESS_ONCE_RW(p) = (v)) +#else +#define WRITE_ONCE(p, v) (ACCESS_ONCE(p) = (v)) +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include "udp_tunnel/udp_tunnel_partial_compat.h" +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(DEBUG) && defined(net_dbg_ratelimited) +#undef net_dbg_ratelimited +#define net_dbg_ratelimited(fmt, ...) do { if (0) no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef RCU_LOCKDEP_WARN +#define RCU_LOCKDEP_WARN(cond, message) rcu_lockdep_assert(!(cond), message) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(ISRHEL7) +#define ipv6_dst_lookup(a, b, c, d) ipv6_dst_lookup(b, c, d) +#endif + +#if (LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 17) && LINUX_VERSION_CODE > KERNEL_VERSION(3, 19, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 27) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 40) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 54))) && !defined(ISUBUNTU1404) && (!defined(ISRHEL7) || RHEL_MINOR < 7) /* TODO: remove < 7 workaround once CentOS 7.7 comes out. */ +#include +#include +#define IP6_ECN_set_ce(a, b) IP6_ECN_set_ce(b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISRHEL7) +#include +struct ipv6_stub_type { + void *udpv6_encap_enable; + int (*ipv6_dst_lookup)(struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6); +}; +static const struct ipv6_stub_type ipv6_stub_impl = { + .udpv6_encap_enable = (void *)1, + .ipv6_dst_lookup = ip6_dst_lookup +}; +static const struct ipv6_stub_type *ipv6_stub = &ipv6_stub_impl; +#endif + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISOPENSUSE42) && !defined(ISRHEL7) +#include +static inline bool ipv6_mod_enabled(void) +{ + return ipv6_stub != NULL && ipv6_stub->udpv6_encap_enable != NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_reset_tc(struct sk_buff *skb) +{ +#ifdef CONFIG_NET_CLS_ACT + skb->tc_verd = 0; +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#include +#include +static inline u32 __compat_get_random_u32(void) +{ + static siphash_key_t key; + static u32 counter = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + static bool has_seeded = false; + if (unlikely(!has_seeded)) { + get_random_bytes(&key, sizeof(key)); + has_seeded = true; + } +#else + get_random_once(&key, sizeof(key)); +#endif + return siphash_2u32(counter++, get_random_int(), &key); +} +#define get_random_u32 __compat_get_random_u32 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) && !defined(ISRHEL7) +static inline void netif_keep_dst(struct net_device *dev) +{ + dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; +} +#define COMPAT_CANNOT_USE_CSUM_LEVEL +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define pcpu_sw_netstats pcpu_tstats +#endif +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats alloc_percpu +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats(type) \ +({ \ + typeof(type) __percpu *pcpu_stats = alloc_percpu(type); \ + if (pcpu_stats) { \ + int __cpu; \ + for_each_possible_cpu(__cpu) { \ + typeof(type) *stat; \ + stat = per_cpu_ptr(pcpu_stats, __cpu); \ + u64_stats_init(&stat->syncp); \ + } \ + } \ + pcpu_stats; \ +}) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include "checksum/checksum_partial_compat.h" +static inline void *__compat_pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) +{ + if (tail != skb) { + skb->data_len += len; + skb->len += len; + } + return skb_put(tail, len); +} +#define pskb_put __compat_pskb_put +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_scrub_packet(struct sk_buff *skb, bool xnet) +{ +#ifdef CONFIG_CAVIUM_OCTEON_IPFWD_OFFLOAD + memset(&skb->cvm_info, 0, sizeof(skb->cvm_info)); + skb->cvm_reserved = 0; +#endif + skb->tstamp.tv64 = 0; + skb->pkt_type = PACKET_HOST; + skb->skb_iif = 0; + skb_dst_drop(skb); + secpath_reset(skb); + nf_reset(skb); + nf_reset_trace(skb); + if (!xnet) + return; + skb_orphan(skb); + skb->mark = 0; +} +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) || defined(ISUBUNTU1404)) && !defined(ISRHEL7) +#include +static inline u32 __compat_prandom_u32_max(u32 ep_ro) +{ + return (u32)(((u64)prandom_u32() * ep_ro) >> 32); +} +#define prandom_u32_max __compat_prandom_u32_max +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 75) && !defined(ISRHEL7) +#ifndef U8_MAX +#define U8_MAX ((u8)~0U) +#endif +#ifndef S8_MAX +#define S8_MAX ((s8)(U8_MAX >> 1)) +#endif +#ifndef S8_MIN +#define S8_MIN ((s8)(-S8_MAX - 1)) +#endif +#ifndef U16_MAX +#define U16_MAX ((u16)~0U) +#endif +#ifndef S16_MAX +#define S16_MAX ((s16)(U16_MAX >> 1)) +#endif +#ifndef S16_MIN +#define S16_MIN ((s16)(-S16_MAX - 1)) +#endif +#ifndef U32_MAX +#define U32_MAX ((u32)~0U) +#endif +#ifndef S32_MAX +#define S32_MAX ((s32)(U32_MAX >> 1)) +#endif +#ifndef S32_MIN +#define S32_MIN ((s32)(-S32_MAX - 1)) +#endif +#ifndef U64_MAX +#define U64_MAX ((u64)~0ULL) +#endif +#ifndef S64_MAX +#define S64_MAX ((s64)(U64_MAX >> 1)) +#endif +#ifndef S64_MIN +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 60) && !defined(ISRHEL7) +/* Making this static may very well invalidate its usefulness, + * but so it goes with compat code. */ +static inline void memzero_explicit(void *s, size_t count) +{ + memset(s, 0, count); + barrier(); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && !defined(ISRHEL7) +static const struct in6_addr __compat_in6addr_any = IN6ADDR_ANY_INIT; +#define in6addr_any __compat_in6addr_any +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISOPENSUSE15) +#include +#include +#include +struct rng_initializer { + struct completion done; + struct random_ready_callback cb; +}; +static inline void rng_initialized_callback(struct random_ready_callback *cb) +{ + complete(&container_of(cb, struct rng_initializer, cb)->done); +} +static inline int wait_for_random_bytes(void) +{ + static bool rng_is_initialized = false; + int ret; + if (unlikely(!rng_is_initialized)) { + struct rng_initializer rng = { + .done = COMPLETION_INITIALIZER(rng.done), + .cb = { .owner = THIS_MODULE, .func = rng_initialized_callback } + }; + ret = add_random_ready_callback(&rng.cb); + if (!ret) { + ret = wait_for_completion_interruptible(&rng.done); + if (ret) { + del_random_ready_callback(&rng.cb); + return ret; + } + } else if (ret != -EALREADY) + return ret; + rng_is_initialized = true; + } + return 0; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline int wait_for_random_bytes(void) +{ + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISRHEL8) +#include +#include +struct rng_is_initialized_callback { + struct random_ready_callback cb; + atomic_t *rng_state; +}; +static inline void rng_is_initialized_callback(struct random_ready_callback *cb) +{ + struct rng_is_initialized_callback *rdy = container_of(cb, struct rng_is_initialized_callback, cb); + atomic_set(rdy->rng_state, 2); + kfree(rdy); +} +static inline bool rng_is_initialized(void) +{ + static atomic_t rng_state = ATOMIC_INIT(0); + + if (atomic_read(&rng_state) == 2) + return true; + + if (atomic_cmpxchg(&rng_state, 0, 1) == 0) { + int ret; + struct rng_is_initialized_callback *rdy = kmalloc(sizeof(*rdy), GFP_ATOMIC); + if (!rdy) { + atomic_set(&rng_state, 0); + return false; + } + rdy->cb.owner = THIS_MODULE; + rdy->cb.func = rng_is_initialized_callback; + rdy->rng_state = &rng_state; + ret = add_random_ready_callback(&rdy->cb); + if (ret) + kfree(rdy); + if (ret == -EALREADY) { + atomic_set(&rng_state, 2); + return true; + } else if (ret) + atomic_set(&rng_state, 0); + return false; + } + return false; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline bool rng_is_initialized(void) +{ + return true; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +static inline int get_random_bytes_wait(void *buf, int nbytes) +{ + int ret = wait_for_random_bytes(); + if (unlikely(ret)) + return ret; + get_random_bytes(buf, nbytes); + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#define system_power_efficient_wq system_unbound_wq +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#include +#ifndef ktime_get_real_ts64 +#define timespec64 timespec +#define ktime_get_real_ts64 ktime_get_real_ts +#endif +#else +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +static inline u64 __compat_jiffies64_to_nsecs(u64 j) +{ +#if !(NSEC_PER_SEC % HZ) + return (NSEC_PER_SEC / HZ) * j; +#else + return div_u64(j * HZ_TO_USEC_NUM, HZ_TO_USEC_DEN) * 1000; +#endif +} +#define jiffies64_to_nsecs __compat_jiffies64_to_nsecs +#endif +static inline u64 ktime_get_coarse_boottime_ns(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return ktime_to_ns(ktime_get_boottime()); +#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 12) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 53) + return ktime_to_ns(ktime_mono_to_any(ns_to_ktime(jiffies64_to_nsecs(get_jiffies_64())), TK_OFFS_BOOT)); +#else + return ktime_to_ns(ktime_get_coarse_boottime()); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +static inline __be32 __compat_confirm_addr_indev(struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + int same = 0; + __be32 addr = 0; + for_ifa(in_dev) { + if (!addr && (local == ifa->ifa_local || !local) && ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + if (same) + break; + } + if (!same) { + same = (!local || inet_ifa_match(local, ifa)) && (!dst || inet_ifa_match(dst, ifa)); + if (same && addr) { + if (local || !dst) + break; + if (inet_ifa_match(addr, ifa)) + break; + if (ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + break; + } + same = 0; + } + } + } endfor_ifa(in_dev); + return same ? addr : 0; +} +static inline __be32 __compat_inet_confirm_addr(struct net *net, struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + __be32 addr = 0; + struct net_device *dev; + if (in_dev) + return __compat_confirm_addr_indev(in_dev, dst, local, scope); + rcu_read_lock(); + for_each_netdev_rcu(net, dev) { + in_dev = __in_dev_get_rcu(dev); + if (in_dev) { + addr = __compat_confirm_addr_indev(in_dev, dst, local, scope); + if (addr) + break; + } + } + rcu_read_unlock(); + return addr; +} +#define inet_confirm_addr __compat_inet_confirm_addr +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#include +static inline void *__compat_kvmalloc(size_t size, gfp_t flags) +{ + gfp_t kmalloc_flags = flags; + void *ret; + if (size > PAGE_SIZE) { + kmalloc_flags |= __GFP_NOWARN; + if (!(kmalloc_flags & __GFP_REPEAT) || (size <= PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + kmalloc_flags |= __GFP_NORETRY; + } + ret = kmalloc(size, kmalloc_flags); + if (ret || size <= PAGE_SIZE) + return ret; + return __vmalloc(size, flags, PAGE_KERNEL); +} +static inline void *__compat_kvzalloc(size_t size, gfp_t flags) +{ + return __compat_kvmalloc(size, flags | __GFP_ZERO); +} +#define kvmalloc __compat_kvmalloc +#define kvzalloc __compat_kvzalloc +#endif + +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 41)) && !defined(ISUBUNTU1404) +#include +#include +static inline void __compat_kvfree(const void *addr) +{ + if (is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} +#define kvfree __compat_kvfree +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 9) +#include +#define priv_destructor destructor +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +#define wg_newlink(a,b,c,d,e) wg_newlink(a,b,c,d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#define nlmsg_parse(a, b, c, d, e, f) nlmsg_parse(a, b, c, d, e) +#define nla_parse_nested(a, b, c, d, e) nla_parse_nested(a, b, c, d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +static inline struct nlattr **genl_family_attrbuf(const struct genl_family *family) +{ + return family->attrbuf; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define PTR_ERR_OR_ZERO(p) PTR_RET(p) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) +#include +#define nla_put_u64_64bit(a, b, c, d) nla_put_u64(a, b, c) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#include +#ifndef GENL_UNS_ADMIN_PERM +#define GENL_UNS_ADMIN_PERM GENL_ADMIN_PERM +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops, ARRAY_SIZE(genl_ops)) +#define COMPAT_CANNOT_USE_CONST_GENL_OPS +#else +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops) +#endif +#define COMPAT_CANNOT_USE_GENL_NOPS +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 2) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 16) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 65) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 101) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 84) +#define __COMPAT_NETLINK_DUMP_BLOCK { \ + int ret; \ + skb->end -= nlmsg_total_size(sizeof(int)); \ + ret = wg_get_device_dump_real(skb, cb); \ + skb->end += nlmsg_total_size(sizeof(int)); \ + return ret; \ +} +#define __COMPAT_NETLINK_DUMP_OVERRIDE +#else +#define __COMPAT_NETLINK_DUMP_BLOCK return wg_get_device_dump_real(skb, cb); +#endif +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 25) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 87) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + struct wg_device *wg = (struct wg_device *)cb->args[0]; \ + if (!wg) { \ + int ret = wg_get_device_start(cb); \ + if (ret) \ + return ret; \ + } \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#define COMPAT_CANNOT_USE_NETLINK_START +#elif defined(__COMPAT_NETLINK_DUMP_OVERRIDE) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define COMPAT_CANNOT_USE_IN6_DEV_GET +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define COMPAT_CANNOT_USE_IFF_NO_QUEUE +#endif + +#if defined(CONFIG_X86_64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +#include +#include +static inline int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) +{ + return boot_cpu_has(X86_FEATURE_XSAVE) && xgetbv(XCR_XFEATURE_ENABLED_MASK) & xfeatures_needed; +} +#endif +#ifndef XFEATURE_MASK_YMM +#define XFEATURE_MASK_YMM XSTATE_YMM +#endif +#ifndef XFEATURE_MASK_SSE +#define XFEATURE_MASK_SSE XSTATE_SSE +#endif +#ifndef XSTATE_AVX512 +#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM) +#endif +#ifndef XFEATURE_MASK_AVX512 +#define XFEATURE_MASK_AVX512 XSTATE_AVX512 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && defined(CONFIG_X86_64) +/* This is incredibly dumb and reckless, but as it turns out, there's + * not really hardware Linux runs properly on that supports F but not BW + * and VL, so in practice this isn't so bad. Plus, this is compat layer, + * so the bar remains fairly low. + */ +#include +#ifndef X86_FEATURE_AVX512BW +#define X86_FEATURE_AVX512BW X86_FEATURE_AVX512F +#endif +#ifndef X86_FEATURE_AVX512VL +#define X86_FEATURE_AVX512VL X86_FEATURE_AVX512F +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) +struct __compat_dummy_container { char dev; }; +#define netdev_notifier_info net_device *)data); __attribute((unused)) char __compat_dummy_variable = ((struct __compat_dummy_container +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#define from_timer(var, callback_timer, timer_fieldname) container_of(callback_timer, typeof(*var), timer_fieldname) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 3) +#define COMPAT_CANNOT_USE_AVX512 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define genl_dump_check_consistent(a, b) genl_dump_check_consistent(a, b, &genl_family) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISRHEL7) && !defined(ISOPENSUSE15) +static inline void *skb_put_data(struct sk_buff *skb, const void *data, unsigned int len) +{ + void *tmp = skb_put(skb, len); + memcpy(tmp, data, len); + return tmp; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) && !defined(ISRHEL7) +#define napi_complete_done(n, work_done) napi_complete(n) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#include +/* NAPI_STATE_SCHED gets set by netif_napi_add anyway, so this is safe. + * Also, kernels without NAPI_STATE_NO_BUSY_POLL don't have a call to + * napi_hash_add inside of netif_napi_add. + */ +#define NAPI_STATE_NO_BUSY_POLL NAPI_STATE_SCHED +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) ({ int __compat_p1 = atomic_read(v); smp_rmb(); __compat_p1; }) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) ({ smp_wmb(); atomic_set(v, i); }) +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0) +static inline void le32_to_cpu_array(u32 *buf, unsigned int words) +{ + while (words--) { + __le32_to_cpus(buf); + buf++; + } +} +static inline void cpu_to_le32_array(u32 *buf, unsigned int words) +{ + while (words--) { + __cpu_to_le32s(buf); + buf++; + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#include +static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2, + unsigned int size) +{ + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + __builtin_constant_p(size) && + (size % sizeof(unsigned long)) == 0) { + unsigned long *d = (unsigned long *)dst; + unsigned long *s1 = (unsigned long *)src1; + unsigned long *s2 = (unsigned long *)src2; + + while (size > 0) { + *d++ = *s1++ ^ *s2++; + size -= sizeof(unsigned long); + } + } else { + if (unlikely(dst != src1)) + memmove(dst, src1, size); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + crypto_xor(dst, src2, size); +#else + __crypto_xor(dst, src2, size); +#endif + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define read_cpuid_part() read_cpuid_part_number() +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) && !defined(ISRHEL7) +#define hlist_add_behind(a, b) hlist_add_after(b, a) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) +#define totalram_pages() totalram_pages +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0) +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#ifdef __kernel_timespec +#undef __kernel_timespec +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#define skb_probe_transport_header(a) skb_probe_transport_header(a, 0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && !defined(ISRHEL7) +#define ignore_df local_df +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +/* Note that all intentional uses of the non-_bh variety need to explicitly + * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU. + */ +#include +static __always_inline void old_synchronize_rcu(void) +{ + synchronize_rcu(); +} +static __always_inline void old_call_rcu(void *a, void *b) +{ + call_rcu(a, b); +} +static __always_inline void old_rcu_barrier(void) +{ + rcu_barrier(); +} +#ifdef synchronize_rcu +#undef synchronize_rcu +#endif +#ifdef call_rcu +#undef call_rcu +#endif +#ifdef rcu_barrier +#undef rcu_barrier +#endif +#define synchronize_rcu synchronize_rcu_bh +#define call_rcu call_rcu_bh +#define rcu_barrier rcu_barrier_bh +#define COMPAT_CANNOT_DEPRECIATE_BH_RCU +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 10) && !defined(ISRHEL8) +static inline void skb_mark_not_on_list(struct sk_buff *skb) +{ + skb->next = NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && !defined(ISRHEL8) +#define NLA_EXACT_LEN NLA_UNSPEC +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) +#define NLA_MIN_LEN NLA_UNSPEC +#define COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) && defined(__aarch64__) +#define cpu_have_named_feature(name) (elf_hwcap & (HWCAP_ ## name)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) +#include +#ifndef offsetofend +#define offsetofend(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define genl_dumpit_info(cb) ({ \ + struct { struct nlattr **attrs; } *a = (void *)((u8 *)cb->args + offsetofend(struct dump_ctx, next_allowedip)); \ + BUILD_BUG_ON(sizeof(cb->args) < offsetofend(struct dump_ctx, next_allowedip) + sizeof(*a)); \ + a->attrs = genl_family_attrbuf(&genl_family); \ + if (nlmsg_parse(cb->nlh, GENL_HDRLEN + genl_family.hdrsize, a->attrs, genl_family.maxattr, device_policy, NULL) < 0) \ + memset(a->attrs, 0, (genl_family.maxattr + 1) * sizeof(struct nlattr *)); \ + a; \ +}) +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 18) +#define ipv6_dst_lookup_flow(a, b, c, d) ipv6_dst_lookup(a, b, &dst, c) + (void *)0 ?: dst +#endif + +#if defined(ISUBUNTU1604) +#include +#ifndef _WG_LINUX_SIPHASH_H +#define hsiphash_2u32 siphash_2u32 +#define hsiphash_3u32 siphash_3u32 +#define hsiphash_key_t siphash_key_t +#endif +#endif + +#ifdef CONFIG_VE +#include +#ifdef NETIF_F_VIRTUAL +#undef NETIF_F_LLTX +#define NETIF_F_LLTX (__NETIF_F(LLTX) | __NETIF_F(VIRTUAL)) +#endif +#endif + +/* https://github.com/ClangBuiltLinux/linux/issues/7 */ +#if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000) +#include +#undef BUILD_BUG_ON +#define BUILD_BUG_ON(x) +#endif + +/* https://lkml.kernel.org/r/20170624021727.17835-1-Jason@zx2c4.com */ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#endif +static inline void new_icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb_in, &ctinfo); + if (skb_network_header(skb_in) < skb_in->head || (skb_network_header(skb_in) + sizeof(struct iphdr)) > skb_tail_pointer(skb_in)) + return; + if (ct) + ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; + icmp_send(skb_in, type, code, info); +} +static inline void new_icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb, &ctinfo); + if (skb_network_header(skb) < skb->head || (skb_network_header(skb) + sizeof(struct ipv6hdr)) > skb_tail_pointer(skb)) + return; + if (ct) + ipv6_hdr(skb)->saddr = ct->tuplehash[0].tuple.src.u3.in6; + icmpv6_send(skb, type, code, info); +} +#define icmp_send(a,b,c,d) new_icmp_send(a,b,c,d) +#define icmpv6_send(a,b,c,d) new_icmpv6_send(a,b,c,d) +#endif + +/* PaX compatibility */ +#ifdef CONSTIFY_PLUGIN +#include +#undef __read_mostly +#define __read_mostly +#endif +#if (defined(RAP_PLUGIN) || defined(CONFIG_CFI_CLANG)) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define wg_expired_retransmit_handshake(a) wg_expired_retransmit_handshake(unsigned long timer) +#define wg_expired_send_keepalive(a) wg_expired_send_keepalive(unsigned long timer) +#define wg_expired_new_handshake(a) wg_expired_new_handshake(unsigned long timer) +#define wg_expired_zero_key_material(a) wg_expired_zero_key_material(unsigned long timer) +#define wg_expired_send_persistent_keepalive(a) wg_expired_send_persistent_keepalive(unsigned long timer) +#undef timer_setup +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#undef from_timer +#define from_timer(var, callback_timer, timer_fieldname) container_of((struct timer_list *)callback_timer, typeof(*var), timer_fieldname) +#endif + +#endif /* _WG_COMPAT_H */ diff --git a/net/wireguard/compat/dst_cache/dst_cache.c b/net/wireguard/compat/dst_cache/dst_cache.c new file mode 100644 index 0000000000000..7ec22f768a8fb --- /dev/null +++ b/net/wireguard/compat/dst_cache/dst_cache.c @@ -0,0 +1,175 @@ +/* + * net/core/dst_cache.c - dst entry cache + * + * Copyright (c) 2016 Paolo Abeni + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 50) +static inline u32 rt6_get_cookie(const struct rt6_info *rt) +{ + if ((unlikely(rt->dst.flags & DST_NOCACHE) && rt->dst.from)) + rt = (struct rt6_info *)(rt->dst.from); + + return rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0; +} +#endif +#endif +#include + +struct dst_cache_pcpu { + unsigned long refresh_ts; + struct dst_entry *dst; + u32 cookie; + union { + struct in_addr in_saddr; + struct in6_addr in6_saddr; + }; +}; + +static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, + struct dst_entry *dst, u32 cookie) +{ + dst_release(dst_cache->dst); + if (dst) + dst_hold(dst); + + dst_cache->cookie = cookie; + dst_cache->dst = dst; +} + +static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, + struct dst_cache_pcpu *idst) +{ + struct dst_entry *dst; + + dst = idst->dst; + if (!dst) + goto fail; + + /* the cache already hold a dst reference; it can't go away */ + dst_hold(dst); + + if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) || + (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) { + dst_cache_per_cpu_dst_set(idst, NULL, 0); + dst_release(dst); + goto fail; + } + return dst; + +fail: + idst->refresh_ts = jiffies; + return NULL; +} + +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) +{ + if (!dst_cache->cache) + return NULL; + + return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); +} + +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in_saddr.s_addr; + return container_of(dst, struct rtable, dst); +} + +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(idst, dst, 0); + idst->in_saddr.s_addr = saddr; +} + +#if IS_ENABLED(CONFIG_IPV6) +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst, + rt6_get_cookie((struct rt6_info *)dst)); + idst->in6_saddr = *addr; +} + +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in6_saddr; + return dst; +} +#endif + +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) + BUG_ON(gfp & GFP_ATOMIC); + dst_cache->cache = alloc_percpu(struct dst_cache_pcpu); +#else + dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu, + gfp | __GFP_ZERO); +#endif + if (!dst_cache->cache) + return -ENOMEM; + + dst_cache_reset(dst_cache); + return 0; +} + +void dst_cache_destroy(struct dst_cache *dst_cache) +{ + int i; + + if (!dst_cache->cache) + return; + + for_each_possible_cpu(i) + dst_release(per_cpu_ptr(dst_cache->cache, i)->dst); + + free_percpu(dst_cache->cache); +} diff --git a/net/wireguard/compat/dst_cache/include/net/dst_cache.h b/net/wireguard/compat/dst_cache/include/net/dst_cache.h new file mode 100644 index 0000000000000..48021c0d6be14 --- /dev/null +++ b/net/wireguard/compat/dst_cache/include/net/dst_cache.h @@ -0,0 +1,97 @@ +#ifndef _WG_NET_DST_CACHE_H +#define _WG_NET_DST_CACHE_H + +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#endif + +struct dst_cache { + struct dst_cache_pcpu __percpu *cache; + unsigned long reset_ts; +}; + +/** + * dst_cache_get - perform cache lookup + * @dst_cache: the cache + * + * The caller should use dst_cache_get_ip4() if it need to retrieve the + * source address to be used when xmitting to the cached dst. + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache); + +/** + * dst_cache_get_ip4 - perform cache lookup and fetch ipv4 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr); + +/** + * dst_cache_set_ip4 - store the ipv4 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr); + +#if IS_ENABLED(CONFIG_IPV6) + +/** + * dst_cache_set_ip6 - store the ipv6 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr); + +/** + * dst_cache_get_ip6 - perform cache lookup and fetch ipv6 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr); +#endif + +/** + * dst_cache_reset - invalidate the cache contents + * @dst_cache: the cache + * + * This do not free the cached dst to avoid races and contentions. + * the dst will be freed on later cache lookup. + */ +static inline void dst_cache_reset(struct dst_cache *dst_cache) +{ + dst_cache->reset_ts = jiffies; +} + +/** + * dst_cache_init - initialize the cache, allocating the required storage + * @dst_cache: the cache + * @gfp: allocation flags + */ +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp); + +/** + * dst_cache_destroy - empty the cache and free the allocated storage + * @dst_cache: the cache + * + * No synchronization is enforced: it must be called only when the cache + * is unused. + */ +void dst_cache_destroy(struct dst_cache *dst_cache); + +#endif /* _WG_NET_DST_CACHE_H */ diff --git a/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h new file mode 100644 index 0000000000000..f3f9117bcb885 --- /dev/null +++ b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h @@ -0,0 +1 @@ +#include diff --git a/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h new file mode 100644 index 0000000000000..35a6bc4da8adf --- /dev/null +++ b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_INTEL_FAMILY_H +#define _ASM_X86_INTEL_FAMILY_H + +/* + * "Big Core" Processors (Branded as Core, Xeon, etc...) + * + * The "_X" parts are generally the EP and EX Xeons, or the + * "Extreme" ones, like Broadwell-E. + * + * Things ending in "2" are usually because we have no better + * name for them. There's no processor called "SILVERMONT2". + */ + +#define INTEL_FAM6_CORE_YONAH 0x0E + +#define INTEL_FAM6_CORE2_MEROM 0x0F +#define INTEL_FAM6_CORE2_MEROM_L 0x16 +#define INTEL_FAM6_CORE2_PENRYN 0x17 +#define INTEL_FAM6_CORE2_DUNNINGTON 0x1D + +#define INTEL_FAM6_NEHALEM 0x1E +#define INTEL_FAM6_NEHALEM_G 0x1F /* Auburndale / Havendale */ +#define INTEL_FAM6_NEHALEM_EP 0x1A +#define INTEL_FAM6_NEHALEM_EX 0x2E + +#define INTEL_FAM6_WESTMERE 0x25 +#define INTEL_FAM6_WESTMERE_EP 0x2C +#define INTEL_FAM6_WESTMERE_EX 0x2F + +#define INTEL_FAM6_SANDYBRIDGE 0x2A +#define INTEL_FAM6_SANDYBRIDGE_X 0x2D +#define INTEL_FAM6_IVYBRIDGE 0x3A +#define INTEL_FAM6_IVYBRIDGE_X 0x3E + +#define INTEL_FAM6_HASWELL_CORE 0x3C +#define INTEL_FAM6_HASWELL_X 0x3F +#define INTEL_FAM6_HASWELL_ULT 0x45 +#define INTEL_FAM6_HASWELL_GT3E 0x46 + +#define INTEL_FAM6_BROADWELL_CORE 0x3D +#define INTEL_FAM6_BROADWELL_GT3E 0x47 +#define INTEL_FAM6_BROADWELL_X 0x4F +#define INTEL_FAM6_BROADWELL_XEON_D 0x56 + +#define INTEL_FAM6_SKYLAKE_MOBILE 0x4E +#define INTEL_FAM6_SKYLAKE_DESKTOP 0x5E +#define INTEL_FAM6_SKYLAKE_X 0x55 +#define INTEL_FAM6_KABYLAKE_MOBILE 0x8E +#define INTEL_FAM6_KABYLAKE_DESKTOP 0x9E + +/* "Small Core" Processors (Atom) */ + +#define INTEL_FAM6_ATOM_PINEVIEW 0x1C +#define INTEL_FAM6_ATOM_LINCROFT 0x26 +#define INTEL_FAM6_ATOM_PENWELL 0x27 +#define INTEL_FAM6_ATOM_CLOVERVIEW 0x35 +#define INTEL_FAM6_ATOM_CEDARVIEW 0x36 +#define INTEL_FAM6_ATOM_SILVERMONT1 0x37 /* BayTrail/BYT / Valleyview */ +#define INTEL_FAM6_ATOM_SILVERMONT2 0x4D /* Avaton/Rangely */ +#define INTEL_FAM6_ATOM_AIRMONT 0x4C /* CherryTrail / Braswell */ +#define INTEL_FAM6_ATOM_MERRIFIELD 0x4A /* Tangier */ +#define INTEL_FAM6_ATOM_MOOREFIELD 0x5A /* Anniedale */ +#define INTEL_FAM6_ATOM_GOLDMONT 0x5C +#define INTEL_FAM6_ATOM_DENVERTON 0x5F /* Goldmont Microserver */ +#define INTEL_FAM6_ATOM_GEMINI_LAKE 0x7A + +/* Xeon Phi */ + +#define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ +#define INTEL_FAM6_XEON_PHI_KNM 0x85 /* Knights Mill */ + +#endif /* _ASM_X86_INTEL_FAMILY_H */ diff --git a/net/wireguard/compat/memneq/include.h b/net/wireguard/compat/memneq/include.h new file mode 100644 index 0000000000000..2d18acd9b6c87 --- /dev/null +++ b/net/wireguard/compat/memneq/include.h @@ -0,0 +1,5 @@ +extern noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size); +static inline int crypto_memneq(const void *a, const void *b, size_t size) +{ + return __crypto_memneq(a, b, size) != 0UL ? 1 : 0; +} diff --git a/net/wireguard/compat/memneq/memneq.c b/net/wireguard/compat/memneq/memneq.c new file mode 100644 index 0000000000000..1c427d4055379 --- /dev/null +++ b/net/wireguard/compat/memneq/memneq.c @@ -0,0 +1,170 @@ +/* + * Constant-time equality testing of memory regions. + * + * Authors: + * + * James Yonan + * Daniel Borkmann + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called LICENSE.GPL. + * + * BSD LICENSE + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of OpenVPN Technologies nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +/* Make the optimizer believe the variable can be manipulated arbitrarily. */ +#define COMPILER_OPTIMIZER_HIDE_VAR(var) asm("" : "=r" (var) : "0" (var)) + +#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ + +/* Generic path for arbitrary size */ +static inline unsigned long +__crypto_memneq_generic(const void *a, const void *b, size_t size) +{ + unsigned long neq = 0; + +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + while (size >= sizeof(unsigned long)) { + neq |= *(unsigned long *)a ^ *(unsigned long *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += sizeof(unsigned long); + b += sizeof(unsigned long); + size -= sizeof(unsigned long); + } +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + while (size > 0) { + neq |= *(unsigned char *)a ^ *(unsigned char *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += 1; + b += 1; + size -= 1; + } + return neq; +} + +/* Loop-free fast-path for frequently used 16-byte size */ +static inline unsigned long __crypto_memneq_16(const void *a, const void *b) +{ + unsigned long neq = 0; + +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (sizeof(unsigned long) == 8) { + neq |= *(unsigned long *)(a) ^ *(unsigned long *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned long *)(a+8) ^ *(unsigned long *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else if (sizeof(unsigned int) == 4) { + neq |= *(unsigned int *)(a) ^ *(unsigned int *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+4) ^ *(unsigned int *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+8) ^ *(unsigned int *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+12) ^ *(unsigned int *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + { + neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } + + return neq; +} + +/* Compare two areas of memory without leaking timing information, + * and with special optimizations for common sizes. Users should + * not call this function directly, but should instead use + * crypto_memneq defined in crypto/algapi.h. + */ +noinline unsigned long __crypto_memneq(const void *a, const void *b, + size_t size) +{ + switch (size) { + case 16: + return __crypto_memneq_16(a, b); + default: + return __crypto_memneq_generic(a, b, size); + } +} + +#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ diff --git a/net/wireguard/compat/neon-arm/include/asm/neon.h b/net/wireguard/compat/neon-arm/include/asm/neon.h new file mode 100644 index 0000000000000..980d831e201a4 --- /dev/null +++ b/net/wireguard/compat/neon-arm/include/asm/neon.h @@ -0,0 +1,7 @@ +#ifndef _ARCH_ARM_ASM_NEON +#define _ARCH_ARM_ASM_NEON +#define kernel_neon_begin() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#define kernel_neon_end() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#endif diff --git a/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h new file mode 100644 index 0000000000000..37b4bb2545b32 --- /dev/null +++ b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h @@ -0,0 +1,640 @@ +/* + * Definitions for the 'struct ptr_ring' datastructure. + * + * Author: + * Michael S. Tsirkin + * + * Copyright (C) 2016 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This is a limited-size FIFO maintaining pointers in FIFO order, with + * one CPU producing entries and another consuming entries from a FIFO. + * + * This implementation tries to minimize cache-contention when there is a + * single producer and a single consumer CPU. + */ + +#ifndef _LINUX_PTR_RING_H +#define _LINUX_PTR_RING_H 1 + +#ifdef __KERNEL__ +#include +#include +#include +#include +#include +#include +#include +#endif + +struct ptr_ring { + int producer ____cacheline_aligned_in_smp; + spinlock_t producer_lock; + int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */ + int consumer_tail; /* next entry to invalidate */ + spinlock_t consumer_lock; + /* Shared consumer/producer data */ + /* Read-only by both the producer and the consumer */ + int size ____cacheline_aligned_in_smp; /* max entries in queue */ + int batch; /* number of entries to consume in a batch */ + void **queue; +}; + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). If ring is ever resized, callers must hold + * producer_lock - see e.g. ptr_ring_full. Otherwise, if callers don't hold + * producer_lock, the next call to __ptr_ring_produce may fail. + */ +static inline bool __ptr_ring_full(struct ptr_ring *r) +{ + return r->queue[r->producer]; +} + +static inline bool ptr_ring_full(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_full(r); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_full_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must hold producer_lock. + */ +static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + if (unlikely(!r->size) || r->queue[r->producer]) + return -ENOSPC; + + r->queue[r->producer++] = ptr; + if (unlikely(r->producer >= r->size)) + r->producer = 0; + return 0; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * consume in interrupt or BH context, you must disable interrupts/BH when + * calling this. + */ +static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if they dereference the pointer - see e.g. PTR_RING_PEEK_CALL. + * If ring is never resized, and if the pointer is merely + * tested, there's no need to take the lock - see e.g. __ptr_ring_empty. + */ +static inline void *__ptr_ring_peek(struct ptr_ring *r) +{ + if (likely(r->size)) + return r->queue[r->consumer_head]; + return NULL; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if the ring is ever resized - see e.g. ptr_ring_empty. + */ +static inline bool __ptr_ring_empty(struct ptr_ring *r) +{ + return !__ptr_ring_peek(r); +} + +static inline bool ptr_ring_empty(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_empty(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_empty_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Must only be called after __ptr_ring_peek returned !NULL */ +static inline void __ptr_ring_discard_one(struct ptr_ring *r) +{ + /* Fundamentally, what we want to do is update consumer + * index and zero out the entry so producer can reuse it. + * Doing it naively at each consume would be as simple as: + * r->queue[r->consumer++] = NULL; + * if (unlikely(r->consumer >= r->size)) + * r->consumer = 0; + * but that is suboptimal when the ring is full as producer is writing + * out new entries in the same cache line. Defer these updates until a + * batch of entries has been consumed. + */ + int head = r->consumer_head++; + + /* Once we have processed enough entries invalidate them in + * the ring all at once so producer can reuse their space in the ring. + * We also do this when we reach end of the ring - not mandatory + * but helps keep the implementation simple. + */ + if (unlikely(r->consumer_head - r->consumer_tail >= r->batch || + r->consumer_head >= r->size)) { + /* Zero out entries in the reverse order: this way we touch the + * cache line that producer might currently be reading the last; + * producer won't make progress and touch other cache lines + * besides the first one until we write out all entries. + */ + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + } + if (unlikely(r->consumer_head >= r->size)) { + r->consumer_head = 0; + r->consumer_tail = 0; + } +} + +static inline void *__ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + ptr = __ptr_ring_peek(r); + if (ptr) + __ptr_ring_discard_one(r); + + return ptr; +} + +static inline int __ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + void *ptr; + int i; + + for (i = 0; i < n; i++) { + ptr = __ptr_ring_consume(r); + if (!ptr) + break; + array[i] = ptr; + } + + return i; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * call this in interrupt or BH context, you must disable interrupts/BH when + * producing. + */ +static inline void *ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + spin_lock(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_irq(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_irq(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_irq(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_any(struct ptr_ring *r) +{ + unsigned long flags; + void *ptr; + + spin_lock_irqsave(&r->consumer_lock, flags); + ptr = __ptr_ring_consume(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ptr; +} + +static inline void *ptr_ring_consume_bh(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_bh(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_bh(&r->consumer_lock); + + return ptr; +} + +static inline int ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_any(struct ptr_ring *r, + void **array, int n) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Cast to structure type and call a function without discarding from FIFO. + * Function must return a value. + * Callers must take consumer_lock. + */ +#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r))) + +#define PTR_RING_PEEK_CALL(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_BH(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + unsigned long __PTR_RING_PEEK_CALL_f;\ + \ + spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) +{ + return kcalloc(size, sizeof(void *), gfp); +} + +static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) +{ + r->size = size; + r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue)); + /* We need to set batch at least to 1 to make logic + * in __ptr_ring_discard_one work correctly. + * Batching too much (because ring is small) would cause a lot of + * burstiness. Needs tuning, for now disable batching. + */ + if (r->batch > r->size / 2 || !r->batch) + r->batch = 1; +} + +static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) +{ + r->queue = __ptr_ring_init_queue_alloc(size, gfp); + if (!r->queue) + return -ENOMEM; + + __ptr_ring_set_size(r, size); + r->producer = r->consumer_head = r->consumer_tail = 0; + spin_lock_init(&r->producer_lock); + spin_lock_init(&r->consumer_lock); + + return 0; +} + +/* + * Return entries into ring. Destroy entries that don't fit. + * + * Note: this is expected to be a rare slow path operation. + * + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n, + void (*destroy)(void *)) +{ + unsigned long flags; + int head; + + spin_lock_irqsave(&r->consumer_lock, flags); + spin_lock(&r->producer_lock); + + if (!r->size) + goto done; + + /* + * Clean out buffered entries (for simplicity). This way following code + * can test entries for NULL and if not assume they are valid. + */ + head = r->consumer_head - 1; + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + + /* + * Go over entries in batch, start moving head back and copy entries. + * Stop when we run into previously unconsumed entries. + */ + while (n) { + head = r->consumer_head - 1; + if (head < 0) + head = r->size - 1; + if (r->queue[head]) { + /* This batch entry will have to be destroyed. */ + goto done; + } + r->queue[head] = batch[--n]; + r->consumer_tail = r->consumer_head = head; + } + +done: + /* Destroy all entries left in the batch. */ + while (n) + destroy(batch[--n]); + spin_unlock(&r->producer_lock); + spin_unlock_irqrestore(&r->consumer_lock, flags); +} + +static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, + int size, gfp_t gfp, + void (*destroy)(void *)) +{ + int producer = 0; + void **old; + void *ptr; + + while ((ptr = __ptr_ring_consume(r))) + if (producer < size) + queue[producer++] = ptr; + else if (destroy) + destroy(ptr); + + __ptr_ring_set_size(r, size); + r->producer = producer; + r->consumer_head = 0; + r->consumer_tail = 0; + old = r->queue; + r->queue = queue; + + return old; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, + void (*destroy)(void *)) +{ + unsigned long flags; + void **queue = __ptr_ring_init_queue_alloc(size, gfp); + void **old; + + if (!queue) + return -ENOMEM; + + spin_lock_irqsave(&(r)->consumer_lock, flags); + spin_lock(&(r)->producer_lock); + + old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy); + + spin_unlock(&(r)->producer_lock); + spin_unlock_irqrestore(&(r)->consumer_lock, flags); + + kfree(old); + + return 0; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, + unsigned int nrings, + int size, + gfp_t gfp, void (*destroy)(void *)) +{ + unsigned long flags; + void ***queues; + int i; + + queues = kmalloc_array(nrings, sizeof(*queues), gfp); + if (!queues) + goto noqueues; + + for (i = 0; i < nrings; ++i) { + queues[i] = __ptr_ring_init_queue_alloc(size, gfp); + if (!queues[i]) + goto nomem; + } + + for (i = 0; i < nrings; ++i) { + spin_lock_irqsave(&(rings[i])->consumer_lock, flags); + spin_lock(&(rings[i])->producer_lock); + queues[i] = __ptr_ring_swap_queue(rings[i], queues[i], + size, gfp, destroy); + spin_unlock(&(rings[i])->producer_lock); + spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags); + } + + for (i = 0; i < nrings; ++i) + kfree(queues[i]); + + kfree(queues); + + return 0; + +nomem: + while (--i >= 0) + kfree(queues[i]); + + kfree(queues); + +noqueues: + return -ENOMEM; +} + +static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) +{ + void *ptr; + + if (destroy) + while ((ptr = ptr_ring_consume(r))) + destroy(ptr); + kfree(r->queue); +} + +#endif /* _LINUX_PTR_RING_H */ diff --git a/net/wireguard/compat/simd-asm/include/asm/simd.h b/net/wireguard/compat/simd-asm/include/asm/simd.h new file mode 100644 index 0000000000000..a975b38b55781 --- /dev/null +++ b/net/wireguard/compat/simd-asm/include/asm/simd.h @@ -0,0 +1,21 @@ +#ifndef _COMPAT_ASM_SIMD_H +#define _COMPAT_ASM_SIMD_H + +#if defined(CONFIG_X86_64) +#include +#endif + +static __must_check inline bool may_use_simd(void) +{ +#if defined(CONFIG_X86_64) + return irq_fpu_usable(); +#elif defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) + return true; +#elif defined(CONFIG_ARM) && defined(CONFIG_KERNEL_MODE_NEON) + return !in_nmi() && !in_irq() && !in_serving_softirq(); +#else + return false; +#endif +} + +#endif diff --git a/net/wireguard/compat/simd/include/linux/simd.h b/net/wireguard/compat/simd/include/linux/simd.h new file mode 100644 index 0000000000000..c75c72471e9ec --- /dev/null +++ b/net/wireguard/compat/simd/include/linux/simd.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SIMD_H +#define _WG_SIMD_H + +#include +#include +#if defined(CONFIG_X86_64) +#include +#include +#elif defined(CONFIG_KERNEL_MODE_NEON) +#include +#endif + +typedef enum { + HAVE_NO_SIMD = 1 << 0, + HAVE_FULL_SIMD = 1 << 1, + HAVE_SIMD_IN_USE = 1 << 31 +} simd_context_t; + +#define DONT_USE_SIMD ((simd_context_t []){ HAVE_NO_SIMD }) + +static inline void simd_get(simd_context_t *ctx) +{ + *ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD; +} + +static inline void simd_put(simd_context_t *ctx) +{ +#if defined(CONFIG_X86_64) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_fpu_end(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_neon_end(); +#endif + *ctx = HAVE_NO_SIMD; +} + +static inline bool simd_relax(simd_context_t *ctx) +{ +#ifdef CONFIG_PREEMPT + if ((*ctx & HAVE_SIMD_IN_USE) && need_resched()) { + simd_put(ctx); + simd_get(ctx); + return true; + } +#endif + return false; +} + +static __must_check inline bool simd_use(simd_context_t *ctx) +{ + if (!(*ctx & HAVE_FULL_SIMD)) + return false; + if (*ctx & HAVE_SIMD_IN_USE) + return true; +#if defined(CONFIG_X86_64) + kernel_fpu_begin(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + kernel_neon_begin(); +#endif + *ctx |= HAVE_SIMD_IN_USE; + return true; +} + +#endif /* _WG_SIMD_H */ diff --git a/net/wireguard/compat/siphash/include/linux/siphash.h b/net/wireguard/compat/siphash/include/linux/siphash.h new file mode 100644 index 0000000000000..1e5e337d15bf6 --- /dev/null +++ b/net/wireguard/compat/siphash/include/linux/siphash.h @@ -0,0 +1,140 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#ifndef _WG_LINUX_SIPHASH_H +#define _WG_LINUX_SIPHASH_H + +#include +#include + +#define SIPHASH_ALIGNMENT __alignof__(u64) +typedef struct { + u64 key[2]; +} siphash_key_t; + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key); +#endif + +u64 siphash_1u64(const u64 a, const siphash_key_t *key); +u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key); +u64 siphash_3u64(const u64 a, const u64 b, const u64 c, + const siphash_key_t *key); +u64 siphash_4u64(const u64 a, const u64 b, const u64 c, const u64 d, + const siphash_key_t *key); +u64 siphash_1u32(const u32 a, const siphash_key_t *key); +u64 siphash_3u32(const u32 a, const u32 b, const u32 c, + const siphash_key_t *key); + +static inline u64 siphash_2u32(const u32 a, const u32 b, + const siphash_key_t *key) +{ + return siphash_1u64((u64)b << 32 | a, key); +} +static inline u64 siphash_4u32(const u32 a, const u32 b, const u32 c, + const u32 d, const siphash_key_t *key) +{ + return siphash_2u64((u64)b << 32 | a, (u64)d << 32 | c, key); +} + + +static inline u64 ___siphash_aligned(const __le64 *data, size_t len, + const siphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return siphash_1u32(le32_to_cpup((const __le32 *)data), key); + if (__builtin_constant_p(len) && len == 8) + return siphash_1u64(le64_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 16) + return siphash_2u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 24) + return siphash_3u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 32) + return siphash_4u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), le64_to_cpu(data[3]), + key); + return __siphash_aligned(data, len, key); +} + +/** + * siphash - compute 64-bit siphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the siphash key + */ +static inline u64 siphash(const void *data, size_t len, + const siphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT)) + return __siphash_unaligned(data, len, key); +#endif + return ___siphash_aligned(data, len, key); +} + +#define HSIPHASH_ALIGNMENT __alignof__(unsigned long) +typedef struct { + unsigned long key[2]; +} hsiphash_key_t; + +u32 __hsiphash_aligned(const void *data, size_t len, + const hsiphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key); +#endif + +u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key); +u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key); +u32 hsiphash_3u32(const u32 a, const u32 b, const u32 c, + const hsiphash_key_t *key); +u32 hsiphash_4u32(const u32 a, const u32 b, const u32 c, const u32 d, + const hsiphash_key_t *key); + +static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len, + const hsiphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return hsiphash_1u32(le32_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 8) + return hsiphash_2u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 12) + return hsiphash_3u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 16) + return hsiphash_4u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), le32_to_cpu(data[3]), + key); + return __hsiphash_aligned(data, len, key); +} + +/** + * hsiphash - compute 32-bit hsiphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the hsiphash key + */ +static inline u32 hsiphash(const void *data, size_t len, + const hsiphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT)) + return __hsiphash_unaligned(data, len, key); +#endif + return ___hsiphash_aligned(data, len, key); +} + +#endif /* _WG_LINUX_SIPHASH_H */ diff --git a/net/wireguard/compat/siphash/siphash.c b/net/wireguard/compat/siphash/siphash.c new file mode 100644 index 0000000000000..58855328e6e0d --- /dev/null +++ b/net/wireguard/compat/siphash/siphash.c @@ -0,0 +1,539 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#ifdef __LITTLE_ENDIAN +#define bytemask_from_count(cnt) (~(~0ul << (cnt)*8)) +#else +#define bytemask_from_count(cnt) (~(~0ul >> (cnt)*8)) +#endif +#endif + +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 +#include +#include +#endif + +#define SIPROUND \ + do { \ + v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \ + v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \ + } while (0) + +#define PREAMBLE(len) \ + u64 v0 = 0x736f6d6570736575ULL; \ + u64 v1 = 0x646f72616e646f6dULL; \ + u64 v2 = 0x6c7967656e657261ULL; \ + u64 v3 = 0x7465646279746573ULL; \ + u64 b = ((u64)(len)) << 56; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define POSTAMBLE \ + v3 ^= b; \ + SIPROUND; \ + SIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} +#endif + +/** + * siphash_1u64 - compute 64-bit siphash PRF value of a u64 + * @first: first u64 + * @key: the siphash key + */ +u64 siphash_1u64(const u64 first, const siphash_key_t *key) +{ + PREAMBLE(8) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + POSTAMBLE +} + +/** + * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64 + * @first: first u64 + * @second: second u64 + * @key: the siphash key + */ +u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key) +{ + PREAMBLE(16) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + POSTAMBLE +} + +/** + * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @key: the siphash key + */ +u64 siphash_3u64(const u64 first, const u64 second, const u64 third, + const siphash_key_t *key) +{ + PREAMBLE(24) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + POSTAMBLE +} + +/** + * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @forth: forth u64 + * @key: the siphash key + */ +u64 siphash_4u64(const u64 first, const u64 second, const u64 third, + const u64 forth, const siphash_key_t *key) +{ + PREAMBLE(32) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + v3 ^= forth; + SIPROUND; + SIPROUND; + v0 ^= forth; + POSTAMBLE +} + +u64 siphash_1u32(const u32 first, const siphash_key_t *key) +{ + PREAMBLE(4) + b |= first; + POSTAMBLE +} + +u64 siphash_3u32(const u32 first, const u32 second, const u32 third, + const siphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + PREAMBLE(12) + v3 ^= combined; + SIPROUND; + SIPROUND; + v0 ^= combined; + b |= third; + POSTAMBLE +} + +#if BITS_PER_LONG == 64 +/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for + * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3. + */ + +#define HSIPROUND SIPROUND +#define HPREAMBLE(len) PREAMBLE(len) +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + b |= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(8) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(12) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + b |= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(16) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + combined = (u64)forth << 32 | third; + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} +#else +#define HSIPROUND \ + do { \ + v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \ + v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \ + v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \ + v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \ + } while (0) + +#define HPREAMBLE(len) \ + u32 v0 = 0; \ + u32 v1 = 0; \ + u32 v2 = 0x6c796765U; \ + u32 v3 = 0x74656462U; \ + u32 b = ((u32)(len)) << 24; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return v1 ^ v3; + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = le32_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = get_unaligned_le32(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + v3 ^= first; + HSIPROUND; + v0 ^= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + HPREAMBLE(8) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + HPREAMBLE(12) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + HPREAMBLE(16) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + v3 ^= forth; + HSIPROUND; + v0 ^= forth; + HPOSTAMBLE +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h new file mode 100644 index 0000000000000..8999527d69521 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h @@ -0,0 +1,94 @@ +#ifndef _WG_NET_UDP_TUNNEL_H +#define _WG_NET_UDP_TUNNEL_H + +#include +#include + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#endif + +struct udp_port_cfg { + u8 family; + + /* Used only for kernel-created sockets */ + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, + use_udp6_tx_checksums:1, + use_udp6_rx_checksums:1, + ipv6_v6only:1; +}; + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); +#else +static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + return 0; +} +#endif + +static inline int udp_sock_create(struct net *net, + struct udp_port_cfg *cfg, + struct socket **sockp) +{ + if (cfg->family == AF_INET) + return udp_sock_create4(net, cfg, sockp); + + if (cfg->family == AF_INET6) + return udp_sock_create6(net, cfg, sockp); + + return -EPFNOSUPPORT; +} + +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); + +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; + +/* Setup the given (UDP) sock to receive UDP encapsulated packets */ +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *sock_cfg); + +/* Transmit the skb using UDP encapsulation. */ +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck); +#endif + +void udp_tunnel_sock_release(struct socket *sock); + +#endif /* _WG_NET_UDP_TUNNEL_H */ diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel.c b/net/wireguard/compat/udp_tunnel/udp_tunnel.c new file mode 100644 index 0000000000000..ae435660931c8 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel.c @@ -0,0 +1,387 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) +#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk))) +#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr) +#endif + +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) + ,int unused_vulnerable_length_param +#endif + ) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + int err; + struct socket *sock = NULL; + struct sockaddr_in udp_addr; + + err = __sock_create(net, AF_INET, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->local_ip; + udp_addr.sin_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->peer_ip; + udp_addr.sin_port = cfg->peer_udp_port; + err = kernel_connect(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr), 0); + if (err < 0) + goto error; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + sock->sk->sk_no_check_tx = !cfg->use_udp_checksums; +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + inet_sk(sock->sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sock->sk, cfg->sk_user_data); + /* We force the cast in this awful way, due to various Android kernels + * backporting things stupidly. */ + *(void **)&sock->sk->sk_data_ready = (void *)__compat_sk_data_ready; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v4_check(int len, __be32 saddr, + __be32 daddr, __wsum base) +{ + return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_UDP, base); +} + +static void udp_set_csum(bool nocheck, struct sk_buff *skb, + __be32 saddr, __be32 daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_V4_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v4_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} + +#endif + +static void __compat_fake_destructor(struct sk_buff *skb) +{ +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static void __compat_iptunnel_xmit(struct rtable *rt, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 proto, + __u8 tos, __u8 ttl, __be16 df, bool xnet) +{ + struct iphdr *iph; + struct pcpu_tstats *tstats = this_cpu_ptr(skb->dev->tstats); + + skb_scrub_packet(skb, xnet); + + skb->rxhash = 0; + skb_dst_set(skb, &rt->dst); + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + + /* Push down and install the IP header. */ + skb_push(skb, sizeof(struct iphdr)); + skb_reset_network_header(skb); + + iph = ip_hdr(skb); + + iph->version = 4; + iph->ihl = sizeof(struct iphdr) >> 2; + iph->frag_off = df; + iph->protocol = proto; + iph->tos = tos; + iph->daddr = dst; + iph->saddr = src; + iph->ttl = ttl; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 53) + __ip_select_ident(iph, &rt->dst, (skb_shinfo(skb)->gso_segs ?: 1) - 1); +#else + __ip_select_ident(iph, skb_shinfo(skb)->gso_segs ?: 1); +#endif + + iptunnel_xmit(skb, skb->dev); + u64_stats_update_begin(&tstats->syncp); + tstats->tx_bytes -= 8; + u64_stats_update_end(&tstats->syncp); +} +#define iptunnel_xmit __compat_iptunnel_xmit +#endif + +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck) +{ + struct udphdr *uh; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + struct net_device *dev = skb->dev; + int ret; +#endif + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + + udp_set_csum(nocheck, skb, src, dst, skb->len); + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + ret = +#endif + iptunnel_xmit( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) + sk, +#endif + rt, skb, src, dst, IPPROTO_UDP, tos, ttl, df, xnet); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + if (ret) + iptunnel_xmit_stats(ret - 8, &dev->stats, dev->tstats); +#endif +} + +void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); +} + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + struct sockaddr_in6 udp6_addr; + int err; + struct socket *sock = NULL; + + err = __sock_create(net, AF_INET6, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + if (cfg->ipv6_v6only) { + int val = 1; + + err = kernel_setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + (char *) &val, sizeof(val)); + if (err < 0) + goto error; + } + + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->local_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->peer_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->peer_udp_port; + err = kernel_connect(sock, + (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr), 0); + } + if (err < 0) + goto error; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + udp_set_no_check6_tx(sock->sk, !cfg->use_udp6_tx_checksums); + udp_set_no_check6_rx(sock->sk, !cfg->use_udp6_rx_checksums); +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v6_check(int len, + const struct in6_addr *saddr, + const struct in6_addr *daddr, + __wsum base) +{ + return csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, base); +} +static void udp6_set_csum(bool nocheck, struct sk_buff *skb, + const struct in6_addr *saddr, + const struct in6_addr *daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_IPV6_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v6_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} +#endif + +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + + uh->len = htons(skb->len); + + skb_dst_set(skb, dst); + + udp6_set_csum(nocheck, skb, saddr, daddr, skb->len); + + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, label); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; + + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h new file mode 100644 index 0000000000000..0605896e902f7 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define udp_sock_create4 udp_sock_create +#define udp_sock_create6 udp_sock_create +#include +#include +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#endif +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} +static inline void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + struct sock *sk = sock->sk; + inet_sk(sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sk, cfg->sk_user_data); + sk->sk_data_ready = __compat_sk_data_ready; +} +static inline void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sk_release_kernel(sock->sk); +} +static inline int udp_tunnel_xmit_skb(struct socket *sock, struct rtable *rt, + struct sk_buff *skb, __be32 src, __be32 dst, + __u8 tos, __u8 ttl, __be16 df, __be16 src_port, + __be16 dst_port, bool xnet) +{ + struct udphdr *uh; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + udp_set_csum(sock->sk->sk_no_check_tx, skb, src, dst, skb->len); + return iptunnel_xmit(sock->sk, rt, skb, src, dst, IPPROTO_UDP, + tos, ttl, df, xnet); +} +#if IS_ENABLED(CONFIG_IPV6) +static inline int udp_tunnel6_xmit_skb(struct socket *sock, struct dst_entry *dst, + struct sk_buff *skb, struct net_device *dev, + struct in6_addr *saddr, struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be16 src_port, + __be16 dst_port) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + struct sock *sk = sock->sk; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED + | IPSKB_REROUTED); + skb_dst_set(skb, dst); + udp6_set_csum(udp_get_no_check6_tx(sk), skb, &inet6_sk(sk)->saddr, + &sk->sk_v6_daddr, skb->len); + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, htonl(0)); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +#include +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; ret__ = udp_tunnel_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, i, j, k); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, j, k); +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0) +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +#endif +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); ret__ = udp_tunnel_xmit_skb(a, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); udp_tunnel6_xmit_skb(a, c, d, e, f, g, h, j, k, l); } while(0) +#endif +#else + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); iptunnel_xmit_stats(ret__, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && IS_ENABLED(CONFIG_IPV6) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, j, k, l) +#endif + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +struct __compat_udp_port_cfg { + u8 family; + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, use_udp6_tx_checksums:1, use_udp6_rx_checksums:1, ipv6_v6only:1; +}; +static inline int __maybe_unused __compat_udp_sock_create(struct net *net, struct __compat_udp_port_cfg *cfg, struct socket **sockp) +{ + struct udp_port_cfg old_cfg = { + .family = cfg->family, + .local_ip = cfg->local_ip, +#if IS_ENABLED(CONFIG_IPV6) + .local_ip6 = cfg->local_ip6, +#endif + .peer_ip = cfg->peer_ip, +#if IS_ENABLED(CONFIG_IPV6) + .peer_ip6 = cfg->peer_ip6, +#endif + .local_udp_port = cfg->local_udp_port, + .peer_udp_port = cfg->peer_udp_port, + .use_udp_checksums = cfg->use_udp_checksums, + .use_udp6_tx_checksums = cfg->use_udp6_tx_checksums, + .use_udp6_rx_checksums = cfg->use_udp6_rx_checksums + }; + if (cfg->family == AF_INET) + return udp_sock_create4(net, &old_cfg, sockp); + +#if IS_ENABLED(CONFIG_IPV6) + if (cfg->family == AF_INET6) { + int ret; + int old_bindv6only; + struct net *nobns; + + if (cfg->ipv6_v6only) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) + nobns = &init_net; +#else + nobns = net; +#endif + /* Since udp_port_cfg only learned of ipv6_v6only in 4.3, we do this horrible + * hack here and set the sysctl variable temporarily to something that will + * set the right option for us in sock_create. It's super racey! */ + old_bindv6only = nobns->ipv6.sysctl.bindv6only; + nobns->ipv6.sysctl.bindv6only = 1; + } + ret = udp_sock_create6(net, &old_cfg, sockp); + if (cfg->ipv6_v6only) + nobns->ipv6.sysctl.bindv6only = old_bindv6only; + return ret; + } +#endif + return -EPFNOSUPPORT; +} +#define udp_port_cfg __compat_udp_port_cfg +#define udp_sock_create(a, b, c) __compat_udp_sock_create(a, b, c) +#endif diff --git a/net/wireguard/cookie.c b/net/wireguard/cookie.c new file mode 100644 index 0000000000000..8b7d1fe0cdf4b --- /dev/null +++ b/net/wireguard/cookie.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "cookie.h" +#include "peer.h" +#include "device.h" +#include "messages.h" +#include "ratelimiter.h" +#include "timers.h" + +#include +#include + +#include +#include + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg) +{ + init_rwsem(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + checker->device = wg; +} + +enum { COOKIE_KEY_LABEL_LEN = 8 }; +static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----"; +static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--"; + +static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 pubkey[NOISE_PUBLIC_KEY_LEN], + const u8 label[COOKIE_KEY_LABEL_LEN]) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); + blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); + blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN); + blake2s_final(&blake, key); +} + +/* Must hold peer->handshake.static_identity->lock */ +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker) +{ + if (likely(checker->device->static_identity.has_identity)) { + precompute_key(checker->cookie_encryption_key, + checker->device->static_identity.static_public, + cookie_key_label); + precompute_key(checker->message_mac1_key, + checker->device->static_identity.static_public, + mac1_key_label); + } else { + memset(checker->cookie_encryption_key, 0, + NOISE_SYMMETRIC_KEY_LEN); + memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN); + } +} + +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer) +{ + precompute_key(peer->latest_cookie.cookie_decryption_key, + peer->handshake.remote_static, cookie_key_label); + precompute_key(peer->latest_cookie.message_mac1_key, + peer->handshake.remote_static, mac1_key_label); +} + +void wg_cookie_init(struct cookie *cookie) +{ + memset(cookie, 0, sizeof(*cookie)); + init_rwsem(&cookie->lock); +} + +static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, + const u8 key[NOISE_SYMMETRIC_KEY_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac1); + blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); +} + +static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, + const u8 cookie[COOKIE_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac2); + blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); +} + +static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, + struct cookie_checker *checker) +{ + struct blake2s_state state; + + if (wg_birthdate_has_expired(checker->secret_birthdate, + COOKIE_SECRET_MAX_AGE)) { + down_write(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + up_write(&checker->secret_lock); + } + + down_read(&checker->secret_lock); + + blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); + if (skb->protocol == htons(ETH_P_IP)) + blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, + sizeof(struct in_addr)); + else if (skb->protocol == htons(ETH_P_IPV6)) + blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, + sizeof(struct in6_addr)); + blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); + blake2s_final(&state, cookie); + + up_read(&checker->secret_lock); +} + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie) +{ + struct message_macs *macs = (struct message_macs *) + (skb->data + skb->len - sizeof(*macs)); + enum cookie_mac_state ret; + u8 computed_mac[COOKIE_LEN]; + u8 cookie[COOKIE_LEN]; + + ret = INVALID_MAC; + compute_mac1(computed_mac, skb->data, skb->len, + checker->message_mac1_key); + if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_BUT_NO_COOKIE; + + if (!check_cookie) + goto out; + + make_cookie(cookie, skb, checker); + + compute_mac2(computed_mac, skb->data, skb->len, cookie); + if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; + if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev))) + goto out; + + ret = VALID_MAC_WITH_COOKIE; + +out: + return ret; +} + +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)message + len - sizeof(*macs)); + + down_write(&peer->latest_cookie.lock); + compute_mac1(macs->mac1, message, len, + peer->latest_cookie.message_mac1_key); + memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN); + peer->latest_cookie.have_sent_mac1 = true; + up_write(&peer->latest_cookie.lock); + + down_read(&peer->latest_cookie.lock); + if (peer->latest_cookie.is_valid && + !wg_birthdate_has_expired(peer->latest_cookie.birthdate, + COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) + compute_mac2(macs->mac2, message, len, + peer->latest_cookie.cookie); + else + memset(macs->mac2, 0, COOKIE_LEN); + up_read(&peer->latest_cookie.lock); +} + +void wg_cookie_message_create(struct message_handshake_cookie *dst, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)skb->data + skb->len - sizeof(*macs)); + u8 cookie[COOKIE_LEN]; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE); + dst->receiver_index = index; + get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN); + + make_cookie(cookie, skb, checker); + xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN, + macs->mac1, COOKIE_LEN, dst->nonce, + checker->cookie_encryption_key); +} + +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL; + u8 cookie[COOKIE_LEN]; + bool ret; + + if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable, + INDEX_HASHTABLE_HANDSHAKE | + INDEX_HASHTABLE_KEYPAIR, + src->receiver_index, &peer))) + return; + + down_read(&peer->latest_cookie.lock); + if (unlikely(!peer->latest_cookie.have_sent_mac1)) { + up_read(&peer->latest_cookie.lock); + goto out; + } + ret = xchacha20poly1305_decrypt( + cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie), + peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce, + peer->latest_cookie.cookie_decryption_key); + up_read(&peer->latest_cookie.lock); + + if (ret) { + down_write(&peer->latest_cookie.lock); + memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN); + peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns(); + peer->latest_cookie.is_valid = true; + peer->latest_cookie.have_sent_mac1 = false; + up_write(&peer->latest_cookie.lock); + } else { + net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n", + wg->dev->name); + } + +out: + wg_peer_put(peer); +} diff --git a/net/wireguard/cookie.h b/net/wireguard/cookie.h new file mode 100644 index 0000000000000..c4bd61ca03f24 --- /dev/null +++ b/net/wireguard/cookie.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COOKIE_H +#define _WG_COOKIE_H + +#include "messages.h" +#include + +struct wg_peer; + +struct cookie_checker { + u8 secret[NOISE_HASH_LEN]; + u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + u64 secret_birthdate; + struct rw_semaphore secret_lock; + struct wg_device *device; +}; + +struct cookie { + u64 birthdate; + bool is_valid; + u8 cookie[COOKIE_LEN]; + bool have_sent_mac1; + u8 last_mac1_sent[COOKIE_LEN]; + u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + struct rw_semaphore lock; +}; + +enum cookie_mac_state { + INVALID_MAC, + VALID_MAC_BUT_NO_COOKIE, + VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, + VALID_MAC_WITH_COOKIE +}; + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg); +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer); +void wg_cookie_init(struct cookie *cookie); + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie); +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer); + +void wg_cookie_message_create(struct message_handshake_cookie *src, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker); +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg); + +#endif /* _WG_COOKIE_H */ diff --git a/net/wireguard/crypto/Makefile.include b/net/wireguard/crypto/Makefile.include new file mode 100644 index 0000000000000..f2a312e96d889 --- /dev/null +++ b/net/wireguard/crypto/Makefile.include @@ -0,0 +1,57 @@ +ifeq ($(CONFIG_X86_64)$(if $(CONFIG_UML),y,n),yn) +CONFIG_ZINC_ARCH_X86_64 := y +endif +ifeq ($(CONFIG_ARM)$(if $(CONFIG_CPU_32v3),y,n),yn) +CONFIG_ZINC_ARCH_ARM := y +endif +ifeq ($(CONFIG_ARM64),y) +CONFIG_ZINC_ARCH_ARM64 := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy) +CONFIG_ZINC_ARCH_MIPS := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy) +CONFIG_ZINC_ARCH_MIPS64 := y +endif + +zinc-y += chacha20/chacha20.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o +AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots + +zinc-y += poly1305/poly1305.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o +AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots +zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o + +zinc-y += chacha20poly1305.o + +zinc-y += blake2s/blake2s.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o + +zinc-y += curve25519/curve25519.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) +targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) + +# Old kernels don't set this, which causes trouble. +.SECONDARY: + +wireguard-y += $(addprefix crypto/zinc/,$(zinc-y)) +ccflags-y += -I$(kbuild-dir)/crypto/include +ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64 +ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM +ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64 +ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS +ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64 +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST diff --git a/net/wireguard/crypto/include/zinc/blake2s.h b/net/wireguard/crypto/include/zinc/blake2s.h new file mode 100644 index 0000000000000..2ca0bc30750d9 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/blake2s.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_BLAKE2S_H +#define _ZINC_BLAKE2S_H + +#include +#include +#include + +enum blake2s_lengths { + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32 +}; + +struct blake2s_state { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +void blake2s_init(struct blake2s_state *state, const size_t outlen); +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen); +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen); +void blake2s_final(struct blake2s_state *state, u8 *out); + +static inline void blake2s(u8 *out, const u8 *in, const u8 *key, + const size_t outlen, const size_t inlen, + const size_t keylen) +{ + struct blake2s_state state; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE || + (!key && keylen))); + + if (keylen) + blake2s_init_key(&state, outlen, key, keylen); + else + blake2s_init(&state, outlen); + + blake2s_update(&state, in, inlen); + blake2s_final(&state, out); +} + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen); + +#endif /* _ZINC_BLAKE2S_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20.h b/net/wireguard/crypto/include/zinc/chacha20.h new file mode 100644 index 0000000000000..1b0083d871fb6 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20_H +#define _ZINC_CHACHA20_H + +#include +#include +#include +#include + +enum chacha20_lengths { + CHACHA20_NONCE_SIZE = 16, + CHACHA20_KEY_SIZE = 32, + CHACHA20_KEY_WORDS = CHACHA20_KEY_SIZE / sizeof(u32), + CHACHA20_BLOCK_SIZE = 64, + CHACHA20_BLOCK_WORDS = CHACHA20_BLOCK_SIZE / sizeof(u32), + HCHACHA20_NONCE_SIZE = CHACHA20_NONCE_SIZE, + HCHACHA20_KEY_SIZE = CHACHA20_KEY_SIZE +}; + +enum chacha20_constants { /* expand 32-byte k */ + CHACHA20_CONSTANT_EXPA = 0x61707865U, + CHACHA20_CONSTANT_ND_3 = 0x3320646eU, + CHACHA20_CONSTANT_2_BY = 0x79622d32U, + CHACHA20_CONSTANT_TE_K = 0x6b206574U +}; + +struct chacha20_ctx { + union { + u32 state[16]; + struct { + u32 constant[4]; + u32 key[8]; + u32 counter[4]; + }; + }; +}; + +static inline void chacha20_init(struct chacha20_ctx *ctx, + const u8 key[CHACHA20_KEY_SIZE], + const u64 nonce) +{ + ctx->constant[0] = CHACHA20_CONSTANT_EXPA; + ctx->constant[1] = CHACHA20_CONSTANT_ND_3; + ctx->constant[2] = CHACHA20_CONSTANT_2_BY; + ctx->constant[3] = CHACHA20_CONSTANT_TE_K; + ctx->key[0] = get_unaligned_le32(key + 0); + ctx->key[1] = get_unaligned_le32(key + 4); + ctx->key[2] = get_unaligned_le32(key + 8); + ctx->key[3] = get_unaligned_le32(key + 12); + ctx->key[4] = get_unaligned_le32(key + 16); + ctx->key[5] = get_unaligned_le32(key + 20); + ctx->key[6] = get_unaligned_le32(key + 24); + ctx->key[7] = get_unaligned_le32(key + 28); + ctx->counter[0] = 0; + ctx->counter[1] = 0; + ctx->counter[2] = nonce & U32_MAX; + ctx->counter[3] = nonce >> 32; +} +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context); + +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context); + +#endif /* _ZINC_CHACHA20_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20poly1305.h b/net/wireguard/crypto/include/zinc/chacha20poly1305.h new file mode 100644 index 0000000000000..e3339f02996e2 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20poly1305.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20POLY1305_H +#define _ZINC_CHACHA20POLY1305_H + +#include +#include + +struct scatterlist; + +enum chacha20poly1305_lengths { + XCHACHA20POLY1305_NONCE_SIZE = 24, + CHACHA20POLY1305_KEY_SIZE = 32, + CHACHA20POLY1305_AUTHTAG_SIZE = 16 +}; + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_encrypt_sg_inplace( + struct scatterlist *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +bool __must_check +chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_decrypt_sg_inplace( + struct scatterlist *src, size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check xchacha20poly1305_decrypt( + u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +#endif /* _ZINC_CHACHA20POLY1305_H */ diff --git a/net/wireguard/crypto/include/zinc/curve25519.h b/net/wireguard/crypto/include/zinc/curve25519.h new file mode 100644 index 0000000000000..127d8a3a1c828 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/curve25519.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CURVE25519_H +#define _ZINC_CURVE25519_H + +#include + +enum curve25519_lengths { + CURVE25519_KEY_SIZE = 32 +}; + +bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]); +bool __must_check curve25519_generate_public( + u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]); + +static inline void curve25519_clamp_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + secret[0] &= 248; + secret[31] = (secret[31] & 127) | 64; +} + +#endif /* _ZINC_CURVE25519_H */ diff --git a/net/wireguard/crypto/include/zinc/poly1305.h b/net/wireguard/crypto/include/zinc/poly1305.h new file mode 100644 index 0000000000000..8a16d19f81776 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/poly1305.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_POLY1305_H +#define _ZINC_POLY1305_H + +#include +#include + +enum poly1305_lengths { + POLY1305_BLOCK_SIZE = 16, + POLY1305_KEY_SIZE = 32, + POLY1305_MAC_SIZE = 16 +}; + +struct poly1305_ctx { + u8 opaque[24 * sizeof(u64)]; + u32 nonce[4]; + u8 data[POLY1305_BLOCK_SIZE]; + size_t num; +} __aligned(8); + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context); +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context); + +#endif /* _ZINC_POLY1305_H */ diff --git a/net/wireguard/crypto/zinc.h b/net/wireguard/crypto/zinc.h new file mode 100644 index 0000000000000..9aa1e8d59bf5e --- /dev/null +++ b/net/wireguard/crypto/zinc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ZINC_H +#define _WG_ZINC_H + +int chacha20_mod_init(void); +int poly1305_mod_init(void); +int chacha20poly1305_mod_init(void); +int blake2s_mod_init(void); +int curve25519_mod_init(void); + +#endif diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c new file mode 100644 index 0000000000000..f8cda59bf297b --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); + +static bool blake2s_use_ssse3 __ro_after_init; +static bool blake2s_use_avx512 __ro_after_init; +static bool *const blake2s_nobs[] __initconst = { &blake2s_use_ssse3, + &blake2s_use_avx512 }; + +static void __init blake2s_fpu_init(void) +{ + blake2s_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); +#ifndef COMPAT_CANNOT_USE_AVX512 + blake2s_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + simd_context_t simd_context; + bool used_arch = false; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE / BLAKE2S_BLOCK_SIZE < 8); + + simd_get(&simd_context); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !blake2s_use_ssse3 || + !simd_use(&simd_context)) + goto out; + used_arch = true; + + for (;;) { + const size_t blocks = min_t(size_t, nblocks, + PAGE_SIZE / BLAKE2S_BLOCK_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && blake2s_use_avx512) + blake2s_compress_avx512(state, block, blocks, inc); + else + blake2s_compress_ssse3(state, block, blocks, inc); + + nblocks -= blocks; + if (!nblocks) + break; + block += blocks * BLAKE2S_BLOCK_SIZE; + simd_relax(&simd_context); + } +out: + simd_put(&simd_context); + return used_arch; +} diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S new file mode 100644 index 0000000000000..24910b766bdda --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. + */ + +#include + +.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32 +.align 32 +IV: .octa 0xA54FF53A3C6EF372BB67AE856A09E667 + .octa 0x5BE0CD191F83D9AB9B05688C510E527F +.section .rodata.cst16.ROT16, "aM", @progbits, 16 +.align 16 +ROT16: .octa 0x0D0C0F0E09080B0A0504070601000302 +.section .rodata.cst16.ROR328, "aM", @progbits, 16 +.align 16 +ROR328: .octa 0x0C0F0E0D080B0A090407060500030201 +.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160 +.align 64 +SIGMA: +.byte 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.byte 14, 4, 9, 13, 10, 8, 15, 6, 5, 1, 0, 11, 3, 12, 2, 7 +.byte 11, 12, 5, 15, 8, 0, 2, 13, 9, 10, 3, 7, 4, 14, 6, 1 +.byte 7, 3, 13, 11, 9, 1, 12, 14, 15, 2, 5, 4, 8, 6, 10, 0 +.byte 9, 5, 2, 10, 0, 7, 4, 15, 3, 14, 11, 6, 13, 1, 12, 8 +.byte 2, 6, 0, 8, 12, 10, 11, 3, 1, 4, 7, 15, 9, 13, 5, 14 +.byte 12, 1, 14, 4, 5, 15, 13, 10, 8, 0, 6, 9, 11, 7, 3, 2 +.byte 13, 7, 12, 3, 11, 14, 1, 9, 2, 5, 15, 8, 10, 0, 4, 6 +.byte 6, 14, 11, 0, 15, 9, 3, 8, 10, 12, 13, 1, 5, 2, 7, 4 +.byte 10, 8, 7, 1, 2, 4, 6, 5, 13, 15, 9, 3, 0, 11, 14, 12 +#ifdef CONFIG_AS_AVX512 +.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640 +.align 64 +SIGMA2: +.long 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.long 8, 2, 13, 15, 10, 9, 12, 3, 6, 4, 0, 14, 5, 11, 1, 7 +.long 11, 13, 8, 6, 5, 10, 14, 3, 2, 4, 12, 15, 1, 0, 7, 9 +.long 11, 10, 7, 0, 8, 15, 1, 13, 3, 6, 2, 12, 4, 14, 9, 5 +.long 4, 10, 9, 14, 15, 0, 11, 8, 1, 7, 3, 13, 2, 5, 6, 12 +.long 2, 11, 4, 15, 14, 3, 10, 8, 13, 6, 5, 7, 0, 12, 1, 9 +.long 4, 8, 15, 9, 14, 11, 13, 5, 3, 2, 1, 12, 6, 10, 7, 0 +.long 6, 13, 0, 14, 12, 2, 1, 11, 15, 4, 5, 8, 7, 9, 3, 10 +.long 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 +.long 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 +#endif /* CONFIG_AS_AVX512 */ + +.text +#ifdef CONFIG_AS_SSSE3 +SYM_FUNC_START(blake2s_compress_ssse3) + testq %rdx,%rdx + je .Lendofloop + movdqu (%rdi),%xmm0 + movdqu 0x10(%rdi),%xmm1 + movdqa ROT16(%rip),%xmm12 + movdqa ROR328(%rip),%xmm13 + movdqu 0x20(%rdi),%xmm14 + movq %rcx,%xmm15 + leaq SIGMA+0xa0(%rip),%r8 + jmp .Lbeginofloop + .align 32 +.Lbeginofloop: + movdqa %xmm0,%xmm10 + movdqa %xmm1,%xmm11 + paddq %xmm15,%xmm14 + movdqa IV(%rip),%xmm2 + movdqa %xmm14,%xmm3 + pxor IV+0x10(%rip),%xmm3 + leaq SIGMA(%rip),%rcx +.Lroundloop: + movzbl (%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0x1(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x2(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x3(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + punpckldq %xmm5,%xmm4 + punpckldq %xmm7,%xmm6 + punpcklqdq %xmm6,%xmm4 + paddd %xmm4,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0x4(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x5(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x6(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0x7(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + punpckldq %xmm6,%xmm5 + punpckldq %xmm4,%xmm7 + punpcklqdq %xmm7,%xmm5 + paddd %xmm5,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x93,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x39,%xmm2,%xmm2 + movzbl 0x8(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x9(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xa(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xb(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + punpckldq %xmm7,%xmm6 + punpckldq %xmm5,%xmm4 + punpcklqdq %xmm4,%xmm6 + paddd %xmm6,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0xc(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xd(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xe(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0xf(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + punpckldq %xmm4,%xmm7 + punpckldq %xmm6,%xmm5 + punpcklqdq %xmm5,%xmm7 + paddd %xmm7,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x39,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x93,%xmm2,%xmm2 + addq $0x10,%rcx + cmpq %r8,%rcx + jnz .Lroundloop + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm1 + pxor %xmm10,%xmm0 + pxor %xmm11,%xmm1 + addq $0x40,%rsi + decq %rdx + jnz .Lbeginofloop + movdqu %xmm0,(%rdi) + movdqu %xmm1,0x10(%rdi) + movdqu %xmm14,0x20(%rdi) +.Lendofloop: + ret +SYM_FUNC_END(blake2s_compress_ssse3) +#endif /* CONFIG_AS_SSSE3 */ + +#ifdef CONFIG_AS_AVX512 +SYM_FUNC_START(blake2s_compress_avx512) + vmovdqu (%rdi),%xmm0 + vmovdqu 0x10(%rdi),%xmm1 + vmovdqu 0x20(%rdi),%xmm4 + vmovq %rcx,%xmm5 + vmovdqa IV(%rip),%xmm14 + vmovdqa IV+16(%rip),%xmm15 + jmp .Lblake2s_compress_avx512_mainloop +.align 32 +.Lblake2s_compress_avx512_mainloop: + vmovdqa %xmm0,%xmm10 + vmovdqa %xmm1,%xmm11 + vpaddq %xmm5,%xmm4,%xmm4 + vmovdqa %xmm14,%xmm2 + vpxor %xmm15,%xmm4,%xmm3 + vmovdqu (%rsi),%ymm6 + vmovdqu 0x20(%rsi),%ymm7 + addq $0x40,%rsi + leaq SIGMA2(%rip),%rax + movb $0xa,%cl +.Lblake2s_compress_avx512_roundloop: + addq $0x40,%rax + vmovdqa -0x40(%rax),%ymm8 + vmovdqa -0x20(%rax),%ymm9 + vpermi2d %ymm7,%ymm6,%ymm8 + vpermi2d %ymm7,%ymm6,%ymm9 + vmovdqa %ymm8,%ymm6 + vmovdqa %ymm9,%ymm7 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm8,%xmm8 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x93,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x39,%xmm2,%xmm2 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm9,%xmm9 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x39,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x93,%xmm2,%xmm2 + decb %cl + jne .Lblake2s_compress_avx512_roundloop + vpxor %xmm10,%xmm0,%xmm0 + vpxor %xmm11,%xmm1,%xmm1 + vpxor %xmm2,%xmm0,%xmm0 + vpxor %xmm3,%xmm1,%xmm1 + decq %rdx + jne .Lblake2s_compress_avx512_mainloop + vmovdqu %xmm0,(%rdi) + vmovdqu %xmm1,0x10(%rdi) + vmovdqu %xmm4,0x20(%rdi) + vzeroupper + retq +SYM_FUNC_END(blake2s_compress_avx512) +#endif /* CONFIG_AS_AVX512 */ diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s.c b/net/wireguard/crypto/zinc/blake2s/blake2s.c new file mode 100644 index 0000000000000..e9e0bdaffc82a --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + * + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include + +static const u32 blake2s_iv[8] = { + 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL +}; + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_set_lastblock(struct blake2s_state *state) +{ + state->f[0] = -1; +} + +static inline void blake2s_increment_counter(struct blake2s_state *state, + const u32 inc) +{ + state->t[0] += inc; + state->t[1] += (state->t[0] < inc); +} + +static inline void blake2s_init_param(struct blake2s_state *state, + const u32 param) +{ + int i; + + memset(state, 0, sizeof(*state)); + for (i = 0; i < 8; ++i) + state->h[i] = blake2s_iv[i]; + state->h[0] ^= param; +} + +void blake2s_init(struct blake2s_state *state, const size_t outlen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE)); + blake2s_init_param(state, 0x01010000 | outlen); + state->outlen = outlen; +} +EXPORT_SYMBOL(blake2s_init); + +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen) +{ + u8 block[BLAKE2S_BLOCK_SIZE] = { 0 }; + + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE || + !key || !keylen || keylen > BLAKE2S_KEY_SIZE)); + blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen); + state->outlen = outlen; + memcpy(block, key, keylen); + blake2s_update(state, block, BLAKE2S_BLOCK_SIZE); + memzero_explicit(block, BLAKE2S_BLOCK_SIZE); +} +EXPORT_SYMBOL(blake2s_init_key); + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "blake2s-x86_64-glue.c" +#else +static bool *const blake2s_nobs[] __initconst = { }; +static void __init blake2s_fpu_init(void) +{ +} +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + return false; +} +#endif + +static inline void blake2s_compress(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + WARN_ON(IS_ENABLED(DEBUG) && + (nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE)); + + if (blake2s_compress_arch(state, block, nblocks, inc)) + return; + + while (nblocks > 0) { + blake2s_increment_counter(state, inc); + memcpy(m, block, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, state->h, 32); + v[ 8] = blake2s_iv[0]; + v[ 9] = blake2s_iv[1]; + v[10] = blake2s_iv[2]; + v[11] = blake2s_iv[3]; + v[12] = blake2s_iv[4] ^ state->t[0]; + v[13] = blake2s_iv[5] ^ state->t[1]; + v[14] = blake2s_iv[6] ^ state->f[0]; + v[15] = blake2s_iv[7] ^ state->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + state->h[i] ^= v[i] ^ v[i + 8]; + + block += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(state->buf + state->buflen, in, fill); + blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); + state->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + /* Hash one less (full) block than strictly possible */ + blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(state->buf + state->buflen, in, inlen); + state->buflen += inlen; +} +EXPORT_SYMBOL(blake2s_update); + +void blake2s_final(struct blake2s_state *state, u8 *out) +{ + WARN_ON(IS_ENABLED(DEBUG) && !out); + blake2s_set_lastblock(state); + memset(state->buf + state->buflen, 0, + BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ + blake2s_compress(state, state->buf, 1, state->buflen); + cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); + memcpy(out, state->h, state->outlen); + memzero_explicit(state, sizeof(*state)); +} +EXPORT_SYMBOL(blake2s_final); + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen) +{ + struct blake2s_state state; + u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; + u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); + int i; + + if (keylen > BLAKE2S_BLOCK_SIZE) { + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, key, keylen); + blake2s_final(&state, x_key); + } else + memcpy(x_key, key, keylen); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, in, inlen); + blake2s_final(&state, i_hash); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x5c ^ 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); + blake2s_final(&state, i_hash); + + memcpy(out, i_hash, outlen); + memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); + memzero_explicit(i_hash, BLAKE2S_HASH_SIZE); +} +EXPORT_SYMBOL(blake2s_hmac); + +#include "../selftest/blake2s.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init blake2s_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + blake2s_fpu_init(); + if (!selftest_run("blake2s", blake2s_selftest, blake2s_nobs, + ARRAY_SIZE(blake2s_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c new file mode 100644 index 0000000000000..41e2e79abb2bc --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#if defined(CONFIG_ZINC_ARCH_ARM) +#include +#include +#endif + +asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]); +asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_neon __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { &chacha20_use_neon }; +static void __init chacha20_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + chacha20_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A7: + case ARM_CPU_PART_CORTEX_A5: + /* The Cortex-A7 and Cortex-A5 do not perform well with the NEON + * implementation but do incredibly with the scalar one and use + * less power. + */ + break; + default: + chacha20_use_neon = elf_hwcap & HWCAP_NEON; + } +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + for (;;) { + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon && + len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + chacha20_neon(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } else { + chacha20_arm(dst, src, len, ctx->key, ctx->counter); + ctx->counter[0] += (len + 63) / 64; + break; + } + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM)) { + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + hchacha20_arm(x, derived_key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl new file mode 100644 index 0000000000000..6785383ab7bba --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl @@ -0,0 +1,1227 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# December 2014 +# +# ChaCha20 for ARMv4. +# +# September 2018 +# +# Improve scalar performance per Eric Biggers' suggestion to eliminate +# separate rotates. This requires b[0..3] and d[0..3] to be maintained +# pre-rotated, hence odd twists prior inner loop and when accumulating +# key material. Since amount of instructions is reduced as result, even +# NEON performance is improved somewhat, most notably by ~9% on low-end +# Cortex-A5/A7. Full unroll was shown to provide even better scalar +# performance on Cortex-A5/A7, naturally at the cost of manyfold size +# increase. We let it be. Oversized code works in benchmarks, but is not +# necessarily optimal in real life, when it's likely to be out-of-cache +# upon entry and evict significant part of cache upon completion. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.4 1xNEON 3xNEON+1xIALU +# +# Cortex-A5 14.2(*)/+160% 21.8 12.9(**) +# Cortex-A8 10.2(*)/+190% 13.9 6.10 +# Cortex-A9 10.8(*)/+150% 14.3 6.50 +# Cortex-A15 11.0/+40% 16.0 4.90 +# Snapdragon S4 13.9(***)/+90% 13.6 4.90 +# +# (*) most "favourable" result for aligned data on little-endian +# processor, result for misaligned data is 10-15% lower; +# (**) pure 4xNEON [with "vertical" layout] was shown to provide ~8% +# better performance on Cortex-A5/A7, but not on others; +# (***) it's 17% slower than original, trade-off is considered +# acceptable, because of improvement on others, specifically +# +36% on Cortex-A5/A7 and +20% on Cortex-A9; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x")); +my @t=map("r$_",(8..11)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my $odd = $d0&1; +my ($xc,$xc_) = (@t[0..1]); +my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]); +my @ret; + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' are permanently allocated in registers, @x[0..7], + # while 'c's and pair of 'd's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. If you observe 'd' column, you'll + # notice that 15 and 13 are reused in next pair of rounds. + # This is why these two are chosen for offloading to memory, + # to make loads count more. + push @ret,( + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#13')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#13')", + "&eor ($xd,@x[$a0],$xd,'ror#24')", + "&eor ($xd_,@x[$a1],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b0],$xc, @x[$b0],'ror#13')", + "&eor (@x[$b1],$xc_,@x[$b1],'ror#13')", + + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#20')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#20')", + "&eor ($xd,@x[$a0],$xd,'ror#16')", + "&eor ($xd_,@x[$a1],$xd_,'ror#16')" ); + push @ret,( + "&str ($xd,'[sp,#4*(16+$d0)]')" ) if ($odd); + push @ret,( + "&add ($xc,$xc,$xd,'ror#24')" ); + push @ret,( + "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd); + push @ret,( + "&str ($xd_,'[sp,#4*(16+$d1)]')" ) if (!$odd); + push @ret,( + "&add ($xc_,$xc_,$xd_,'ror#24')" ); + push @ret,( + "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd); + push @ret,( + "&str ($xc,'[sp,#4*(16+$c0)]')", + "&eor (@x[$b0],@x[$b0],$xc,'ror#12')", + "&str ($xc_,'[sp,#4*(16+$c1)]')", + "&eor (@x[$b1],@x[$b1],$xc_,'ror#12')" ); + + $xd=@x[$d2] if (!$odd); + $xd_=@x[$d3] if ($odd); + push @ret,( + "&ldr ($xc,'[sp,#4*(16+$c2)]')", + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#13')", + "&ldr ($xc_,'[sp,#4*(16+$c3)]')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#13')", + "&eor ($xd,@x[$a2],$xd,'ror#24')", + "&eor ($xd_,@x[$a3],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b2],$xc, @x[$b2],'ror#13')", + "&eor (@x[$b3],$xc_,@x[$b3],'ror#13')", + + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#20')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#20')", + "&eor ($xd,@x[$a2],$xd,'ror#16')", + "&eor ($xd_,@x[$a3],$xd_,'ror#16')", + + "&add ($xc,$xc,$xd,'ror#24')", + "&add ($xc_,$xc_,$xd_,'ror#24')", + "&eor (@x[$b2],@x[$b2],$xc,'ror#12')", + "&eor (@x[$b3],@x[$b3],$xc_,'ror#12')" ); + + @ret; +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define ChaCha20_ctr32 chacha20_arm_cryptogams +# define ChaCha20_neon chacha20_neon +#endif + +.text +#if defined(__thumb2__) || defined(__clang__) +.syntax unified +# define ldrhsb ldrbhs +#endif +#if defined(__thumb2__) +.thumb +#else +.code 32 +#endif + +.align 5 +.Lsigma: +.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral +.Lone: +.long 1,0,0,0 +.Lrot8: +.long 0x02010003,0x06050407 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.LChaCha20_ctr32 +#else +.word -1 +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: +.LChaCha20_ctr32: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +#if __ARM_ARCH__<7 && !defined(__thumb2__) + sub r14,pc,#16 @ ChaCha20_ctr32 +#else + adr r14,.LChaCha20_ctr32 +#endif + cmp r2,#0 @ len==0? +#ifdef __thumb2__ + itt eq +#endif + addeq sp,sp,#4*3 + beq .Lno_data +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + cmp r2,#192 @ test len + bls .Lshort + ldr r4,[r14,#-24] + ldr r4,[r14,r4] +# ifdef __APPLE__ + ldr r4,[r4] +# endif + tst r4,#ARMV7_NEON + bne .LChaCha20_neon +.Lshort: +#endif + ldmia r12,{r4-r7} @ load counter and nonce + sub sp,sp,#4*(16) @ off-load area + sub r14,r14,#64 @ .Lsigma + stmdb sp!,{r4-r7} @ copy counter and nonce + ldmia r3,{r4-r11} @ load key + ldmia r14,{r0-r3} @ load sigma + stmdb sp!,{r4-r11} @ copy key + stmdb sp!,{r0-r3} @ copy sigma + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + b .Loop_outer_enter + +.align 4 +.Loop_outer: + ldmia sp,{r0-r9} @ load key material + str @t[3],[sp,#4*(32+2)] @ save len + str r12, [sp,#4*(32+1)] @ save inp + str r14, [sp,#4*(32+0)] @ save out +.Loop_outer_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop + +.align 4 +.Loop: + subs @t[3],@t[3],#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + bne .Loop + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + cmp @t[3],#64 @ done yet? +#ifdef __thumb2__ + itete lo +#endif + addlo r12,sp,#4*(0) @ shortcut or ... + ldrhs r12,[sp,#4*(32+1)] @ ... load inp + addlo r14,sp,#4*(0) @ shortcut or ... + ldrhs r14,[sp,#4*(32+0)] @ ... load out + + ldr @t[0],[sp,#4*(0)] @ load key material + ldr @t[1],[sp,#4*(1)] + +#if __ARM_ARCH__>=6 || !defined(__ARMEB__) +# if __ARM_ARCH__<7 + orr @t[2],r12,r14 + tst @t[2],#3 @ are input and output aligned? + ldr @t[2],[sp,#4*(2)] + bne .Lunaligned + cmp @t[3],#64 @ restore flags +# else + ldr @t[2],[sp,#4*(2)] +# endif + ldr @t[3],[sp,#4*(3)] + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] @ xor with input + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(4) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[1],[r14,#-12] + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @x[5],@t[1],@x[5],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] + add @t[0],sp,#4*(8) + str @x[4],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(12) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + str @x[1],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @x[5],@t[1],@x[5],ror#24 +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[4],[r14],#16 @ store output + str @x[5],[r14,#-12] +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_outer + + beq .Ldone +# if __ARM_ARCH__<7 + b .Ltail + +.align 4 +.Lunaligned: @ unaligned endian-neutral path + cmp @t[3],#64 @ restore flags +# endif +#endif +#if __ARM_ARCH__<7 + ldr @t[3],[sp,#4*(3)] +___ +for ($i=0;$i<16;$i+=4) { +my $j=$i&0x7; +my $twist=""; +if ($i==4) { $twist = ",ror#13"; } +elsif ($i==12) { $twist = ",ror#24"; } + +$code.=<<___ if ($i==4); + add @x[0],sp,#4*(16+8) +___ +$code.=<<___ if ($i==8); + ldmia @x[0],{@x[0]-@x[7]} @ load second half +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" +___ +$code.=<<___; + add @x[$j+0],@t[0],@x[$j+0]$twist @ accumulate key material +___ +$code.=<<___ if ($i==12); +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +___ +$code.=<<___; + add @x[$j+1],@t[1],@x[$j+1]$twist + add @x[$j+2],@t[2],@x[$j+2]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[0],@t[0],@t[0] @ zero or ... + ldrhsb @t[0],[r12],#16 @ ... load input + eorlo @t[1],@t[1],@t[1] + ldrhsb @t[1],[r12,#-12] + + add @x[$j+3],@t[3],@x[$j+3]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[2],@t[2],@t[2] + ldrhsb @t[2],[r12,#-8] + eorlo @t[3],@t[3],@t[3] + ldrhsb @t[3],[r12,#-4] + + eor @x[$j+0],@t[0],@x[$j+0] @ xor with input (or zero) + eor @x[$j+1],@t[1],@x[$j+1] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-15] @ load more input + ldrhsb @t[1],[r12,#-11] + eor @x[$j+2],@t[2],@x[$j+2] + strb @x[$j+0],[r14],#16 @ store output + eor @x[$j+3],@t[3],@x[$j+3] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-7] + ldrhsb @t[3],[r12,#-3] + strb @x[$j+1],[r14,#-12] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-8] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-14] @ load more input + ldrhsb @t[1],[r12,#-10] + strb @x[$j+3],[r14,#-4] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-15] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-6] + ldrhsb @t[3],[r12,#-2] + strb @x[$j+1],[r14,#-11] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-7] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-13] @ load more input + ldrhsb @t[1],[r12,#-9] + strb @x[$j+3],[r14,#-3] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-14] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-5] + ldrhsb @t[3],[r12,#-1] + strb @x[$j+1],[r14,#-10] + strb @x[$j+2],[r14,#-6] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+3],[r14,#-2] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 + strb @x[$j+0],[r14,#-13] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+1],[r14,#-9] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 + strb @x[$j+2],[r14,#-5] + strb @x[$j+3],[r14,#-1] +___ +$code.=<<___ if ($i<12); + add @t[0],sp,#4*(4+$i) + ldmia @t[0],{@t[0]-@t[3]} @ load key material +___ +} +$code.=<<___; +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + bhi .Loop_outer + + beq .Ldone +#endif + +.Ltail: + ldr r12,[sp,#4*(32+1)] @ load inp + add @t[1],sp,#4*(0) + ldr r14,[sp,#4*(32+0)] @ load out + +.Loop_tail: + ldrb @t[2],[@t[1]],#1 @ read buffer on stack + ldrb @t[3],[r12],#1 @ read input + subs @t[0],@t[0],#1 + eor @t[3],@t[3],@t[2] + strb @t[3],[r14],#1 @ store output + bne .Loop_tail + +.Ldone: + add sp,sp,#4*(32+3) +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .long 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) = + map("q$_",(0..15)); + +# This can replace vshr-by-24+vsli-by-8. It gives ~3% improvement on +# Cortex-A5/A7, but hurts Cortex-A9 by 5% and Snapdragon S4 by 14%! +sub vperm() +{ my ($dst,$src,$tbl) = @_; + $code .= " vtbl.8 $dst#lo,{$src#lo},$tbl#lo\n"; + $code .= " vtbl.8 $dst#hi,{$src#hi},$tbl#lo\n"; +} + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&vadd_i32 ($a,$a,$b)", + "&veor ($d,$d,$a)", + "&vrev32_16 ($d,$d)", # vrot ($d,16) + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,20)", + "&vsli_32 ($b,$t,12)", + + "&vadd_i32 ($a,$a,$b)", + "&veor ($t,$d,$a)", + "&vshr_u32 ($d,$t,24)", + "&vsli_32 ($d,$t,8)", + #"&vperm ($d,$t,$t3)", + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,25)", + "&vsli_32 ($b,$t,7)", + + "&vext_8 ($a,$a,$a,$odd?4:12)", + "&vext_8 ($d,$d,$d,8)", + "&vext_8 ($c,$c,$c,$odd?12:4)" + ); +} + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.arch armv7-a +.fpu neon + +# ifdef __KERNEL__ +.globl ChaCha20_neon +@ For optimal performance it's appropriate for caller to enforce +@ minimum input length, 193 bytes is suggested. +# endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +.LChaCha20_neon: + adr r14,.Lsigma + vstmdb sp!,{d8-d15} @ ABI spec says so + stmdb sp!,{r0-r3} + + vld1.32 {$b0-$c0},[r3] @ load key + ldmia r3,{r4-r11} @ load key + + sub sp,sp,#4*(16+16) + vld1.32 {$d0},[r12] @ load counter and nonce + add r12,sp,#4*8 + ldmia r14,{r0-r3} @ load sigma + vld1.32 {$a0},[r14]! @ load sigma + vld1.32 {$t0},[r14]! @ one + @ vld1.32 {$t3#lo},[r14] @ rot8 + vst1.32 {$c0-$d0},[r12] @ copy 1/2key|counter|nonce + vst1.32 {$a0-$b0},[sp] @ copy sigma|1/2key + + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + vshl.i32 $t1#lo,$t0#lo,#1 @ two + vstr $t0#lo,[sp,#4*(16+0)] + vshl.i32 $t2#lo,$t0#lo,#2 @ four + vstr $t1#lo,[sp,#4*(16+2)] + vmov $a1,$a0 + vstr $t2#lo,[sp,#4*(16+4)] + vmov $a2,$a0 + @ vstr $t3#lo,[sp,#4*(16+6)] + vmov $b1,$b0 + vmov $b2,$b0 + b .Loop_neon_enter + +.align 4 +.Loop_neon_outer: + ldmia sp,{r0-r9} @ load key material + cmp @t[3],#64*2 @ if len<=64*2 + bls .Lbreak_neon @ switch to integer-only + @ vldr $t3#lo,[sp,#4*(16+6)] @ rot8 + vmov $a1,$a0 + str @t[3],[sp,#4*(32+2)] @ save len + vmov $a2,$a0 + str r12, [sp,#4*(32+1)] @ save inp + vmov $b1,$b0 + str r14, [sp,#4*(32+0)] @ save out + vmov $b2,$b0 +.Loop_neon_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + vadd.i32 $d1,$d0,$t0 @ counter+1 + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + vmov $c1,$c0 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + vmov $c2,$c0 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + vadd.i32 $d2,$d1,$t0 @ counter+2 + add @x[12],@x[12],#3 @ counter+3 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop_neon + +.align 4 +.Loop_neon: + subs @t[3],@t[3],#1 +___ + my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0); + my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0); + my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1); + @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1); + @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + bne .Loop_neon + + add @t[3],sp,#32 + vld1.32 {$t0-$t1},[sp] @ load key material + vld1.32 {$t2-$t3},[@t[3]] + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + ldr r12,[sp,#4*(32+1)] @ load inp + ldr r14,[sp,#4*(32+0)] @ load out + + vadd.i32 $a0,$a0,$t0 @ accumulate key material + vadd.i32 $a1,$a1,$t0 + vadd.i32 $a2,$a2,$t0 + vldr $t0#lo,[sp,#4*(16+0)] @ one + + vadd.i32 $b0,$b0,$t1 + vadd.i32 $b1,$b1,$t1 + vadd.i32 $b2,$b2,$t1 + vldr $t1#lo,[sp,#4*(16+2)] @ two + + vadd.i32 $c0,$c0,$t2 + vadd.i32 $c1,$c1,$t2 + vadd.i32 $c2,$c2,$t2 + vadd.i32 $d1#lo,$d1#lo,$t0#lo @ counter+1 + vadd.i32 $d2#lo,$d2#lo,$t1#lo @ counter+2 + + vadd.i32 $d0,$d0,$t3 + vadd.i32 $d1,$d1,$t3 + vadd.i32 $d2,$d2,$t3 + + cmp @t[3],#64*4 + blo .Ltail_neon + + vld1.8 {$t0-$t1},[r12]! @ load input + mov @t[3],sp + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 @ xor with input + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + vst1.8 {$a0-$b0},[r14]! @ store output + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vld1.32 {$a0-$b0},[@t[3]]! @ load for next iteration + veor $t0#hi,$t0#hi,$t0#hi + vldr $t0#lo,[sp,#4*(16+4)] @ four + veor $b2,$b2,$t1 + vld1.32 {$c0-$d0},[@t[3]] + veor $c2,$c2,$t2 + vst1.8 {$a1-$b1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$c1-$d1},[r14]! + + vadd.i32 $d0#lo,$d0#lo,$t0#lo @ next counter value + vldr $t0#lo,[sp,#4*(16+0)] @ one + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + vst1.8 {$a2-$b2},[r14]! + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] + vst1.8 {$c2-$d2},[r14]! + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] @ xor with input + add @t[0],sp,#4*(4) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[5],@t[1],@x[5],ror#13 + ldr @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + ldr @t[2],[r12,#-8] + add @x[7],@t[3],@x[7],ror#13 + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] + add @t[0],sp,#4*(8) + eor @x[5],@x[5],@t[1] + str @x[4],[r14],#16 @ store output + eor @x[6],@x[6],@t[2] + str @x[5],[r14,#-12] + eor @x[7],@x[7],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] +# ifdef __thumb2__ + it hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] +# ifdef __thumb2__ + it hi +# endif + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] + add @t[0],sp,#4*(12) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],@t[0],#4 @ next counter value + add @x[5],@t[1],@x[5],ror#24 + str @t[0],[sp,#4*(12)] @ save next counter value + ldr @t[0],[r12],#16 @ load input + add @x[6],@t[2],@x[6],ror#24 + add @x[4],@x[4],#3 @ counter+3 + ldr @t[1],[r12,#-12] + add @x[7],@t[3],@x[7],ror#24 + ldr @t[2],[r12,#-8] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] +# ifdef __thumb2__ + it hi +# endif + ldrhi @t[0],[sp,#4*(32+2)] @ re-load len + eor @x[5],@x[5],@t[1] + eor @x[6],@x[6],@t[2] + str @x[4],[r14],#16 @ store output + eor @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + sub @t[3],@t[0],#64*4 @ len-=64*4 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_neon_outer + + b .Ldone_neon + +.align 4 +.Lbreak_neon: + @ harmonize NEON and integer-only stack frames: load data + @ from NEON frame, but save to integer-only one; distance + @ between the two is 4*(32+4+16-32)=4*(20). + + str @t[3], [sp,#4*(20+32+2)] @ save len + add @t[3],sp,#4*(32+4) + str r12, [sp,#4*(20+32+1)] @ save inp + str r14, [sp,#4*(20+32+0)] @ save out + + ldr @x[12],[sp,#4*(16+10)] + ldr @x[14],[sp,#4*(16+11)] + vldmia @t[3],{d8-d15} @ fulfill ABI requirement + str @x[12],[sp,#4*(20+16+10)] @ copy "@x[10]" + str @x[14],[sp,#4*(20+16+11)] @ copy "@x[11]" + + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(20+16+15)] + add @t[3],sp,#4*(20) + vst1.32 {$a0-$b0},[@t[3]]! @ copy key + add sp,sp,#4*(20) @ switch frame + vst1.32 {$c0-$d0},[@t[3]] + mov @t[3],#10 + b .Loop @ go integer-only + +.align 4 +.Ltail_neon: + cmp @t[3],#64*3 + bhs .L192_or_more_neon + cmp @t[3],#64*2 + bhs .L128_or_more_neon + cmp @t[3],#64*1 + bhs .L64_or_more_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a0-$b0},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c0-$d0},[@t[0]] + b .Loop_tail_neon + +.align 4 +.L64_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vst1.8 {$a0-$b0},[r14]! + vst1.8 {$c0-$d0},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a1-$b1},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c1-$d1},[@t[0]] + sub @t[3],@t[3],#64*1 @ len-=64*1 + b .Loop_tail_neon + +.align 4 +.L128_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vst1.8 {$a0-$b0},[r14]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vst1.8 {$a1-$b1},[r14]! + vst1.8 {$c1-$d1},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a2-$b2},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c2-$d2},[@t[0]] + sub @t[3],@t[3],#64*2 @ len-=64*2 + b .Loop_tail_neon + +.align 4 +.L192_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$a0-$b0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vst1.8 {$c0-$d0},[r14]! + veor $b2,$b2,$t1 + vst1.8 {$a1-$b1},[r14]! + veor $c2,$c2,$t2 + vst1.8 {$c1-$d1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$a2-$b2},[r14]! + vst1.8 {$c2-$d2},[r14]! + + beq .Ldone_neon + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(4) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#13 + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 + ldmia @t[0],{@t[0]-@t[3]} @ load key material +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia sp,{@x[0]-@x[7]} + add @x[0],sp,#4*(16+8) + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(12) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#24 + add @x[4],@x[4],#3 @ counter+3 + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 + ldr @t[3],[sp,#4*(32+2)] @ re-load len +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia @t[0],{@x[0]-@x[7]} + add @t[2],sp,#4*(0) + sub @t[3],@t[3],#64*3 @ len-=64*3 + +.Loop_tail_neon: + ldrb @t[0],[@t[2]],#1 @ read buffer on stack + ldrb @t[1],[r12],#1 @ read input + subs @t[3],@t[3],#1 + eor @t[0],@t[0],@t[1] + strb @t[0],[r14],#1 @ store output + bne .Loop_tail_neon + +.Ldone_neon: + add sp,sp,#4*(32+4) + vldmia sp,{d8-d15} + add sp,sp,#4*(16+3) + ldmia sp!,{r4-r11,pc} +.size ChaCha20_neon,.-ChaCha20_neon +# ifndef __KERNEL__ +.comm OPENSSL_armcap_P,4,4 +# endif +#endif +___ +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl new file mode 100644 index 0000000000000..ac14a99241652 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl @@ -0,0 +1,1163 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# June 2015 +# +# ChaCha20 for ARMv8. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.9 3xNEON+1xIALU 6xNEON+2xIALU(*) +# +# Apple A7 5.50/+49% 3.33 1.70 +# Cortex-A53 8.40/+80% 4.72 4.72(**) +# Cortex-A57 8.06/+43% 4.90 4.43(***) +# Denver 4.50/+82% 2.63 2.67(**) +# X-Gene 9.50/+46% 8.82 8.89(**) +# Mongoose 8.00/+44% 3.64 3.25(***) +# Kryo 8.17/+50% 4.83 4.65(***) +# +# (*) since no non-Apple processor exhibits significantly better +# performance, the code path is #ifdef __APPLE__-ed; +# (**) it's expected that doubling interleave factor doesn't help +# all processors, only those with higher NEON latency and +# higher instruction issue rate; +# (***) expected improvement was actually higher; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4)); + +my @x=map("x$_",(5..17,19..21)); +my @d=map("x$_",(22..28,30)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); + + ( + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],16)", + "&ror_32 (@x[$d1],@x[$d1],16)", + "&ror_32 (@x[$d2],@x[$d2],16)", + "&ror_32 (@x[$d3],@x[$d3],16)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],20)", + "&ror_32 (@x[$b1],@x[$b1],20)", + "&ror_32 (@x[$b2],@x[$b2],20)", + "&ror_32 (@x[$b3],@x[$b3],20)", + + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],24)", + "&ror_32 (@x[$d1],@x[$d1],24)", + "&ror_32 (@x[$d2],@x[$d2],24)", + "&ror_32 (@x[$d3],@x[$d3],24)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],25)", + "&ror_32 (@x[$b1],@x[$b1],25)", + "&ror_32 (@x[$b2],@x[$b2],25)", + "&ror_32 (@x[$b3],@x[$b3],25)" + ); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define ChaCha20_ctr32 chacha20_arm +# define ChaCha20_neon chacha20_neon +#endif + +.text + +.align 5 +.Lsigma: +.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral +.Lone: +.long 1,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +# ifdef __ILP32__ +.long OPENSSL_armcap_P-. +# else +.quad OPENSSL_armcap_P-. +# endif +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + cbz $len,.Labort +#ifndef __KERNEL__ + adr @x[0],.LOPENSSL_armcap_P + cmp $len,#192 + b.lo .Lshort +# ifdef __ILP32__ + ldrsw @x[1],[@x[0]] +# else + ldr @x[1],[@x[0]] +# endif + ldr w17,[@x[1],@x[0]] + tst w17,#ARMV7_NEON + b.ne ChaCha20_neon + +.Lshort: +#endif + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ldp @d[6],@d[7],[$ctr] // load counter +#ifdef __AARCH64EB__ + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + +.Loop_outer: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#64 +.Loop: + sub $ctr,$ctr,#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + cbnz $ctr,.Loop + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + b.lo .Ltail + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + + b.hi .Loop_outer + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 +.Labort: + ret + +.align 4 +.Ltail: + add $len,$len,#64 +.Less_than_64: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + stp @x[0],@x[2],[sp,#0] + stp @x[4],@x[6],[sp,#16] + stp @x[8],@x[10],[sp,#32] + stp @x[12],@x[14],[sp,#48] + +.Loop_tail: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) = + map("v$_.4s",(0..7,16..23)); +my (@K)=map("v$_.4s",(24..30)); +my $ONE="v31.4s"; + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&add ('$a','$a','$b')", + "&eor ('$d','$d','$a')", + "&rev32_16 ('$d','$d')", # vrot ($d,16) + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',20)", + "&sli ('$b','$t',12)", + + "&add ('$a','$a','$b')", + "&eor ('$t','$d','$a')", + "&ushr ('$d','$t',24)", + "&sli ('$d','$t',8)", + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',25)", + "&sli ('$b','$t',7)", + + "&ext ('$a','$a','$a',$odd?4:12)", + "&ext ('$d','$d','$d',8)", + "&ext ('$c','$c','$c',$odd?12:4)" + ); +} + +$code.=<<___; +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl ChaCha20_neon +.type ChaCha20_neon,%function +#endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] +#ifdef __APPLE__ + cmp $len,#512 + b.hs .L512_or_more_neon +#endif + + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +#ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + add @K[3],@K[3],$ONE // += 1 + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + +.Loop_outer_neon: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov $A0,@K[0] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov $A1,@K[0] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov $A2,@K[0] + mov.32 @x[6],@d[3] + mov $B0,@K[1] + lsr @x[7],@d[3],#32 + mov $B1,@K[1] + mov.32 @x[8],@d[4] + mov $B2,@K[1] + lsr @x[9],@d[4],#32 + mov $D0,@K[3] + mov.32 @x[10],@d[5] + mov $D1,@K[4] + lsr @x[11],@d[5],#32 + mov $D2,@K[5] + mov.32 @x[12],@d[6] + mov $C0,@K[2] + lsr @x[13],@d[6],#32 + mov $C1,@K[2] + mov.32 @x[14],@d[7] + mov $C2,@K[2] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#256 +.Loop_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + cbnz $ctr,.Loop_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add $A0,$A0,@K[0] + add @x[1],@x[1],@d[0],lsr#32 + add $A1,$A1,@K[0] + add.32 @x[2],@x[2],@d[1] + add $A2,$A2,@K[0] + add @x[3],@x[3],@d[1],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[4],@x[4],@d[2] + add $C1,$C1,@K[2] + add @x[5],@x[5],@d[2],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[6],@x[6],@d[3] + add $D0,$D0,@K[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add $D1,$D1,@K[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add $D2,$D2,@K[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add $B0,$B0,@K[1] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add $B1,$B1,@K[1] + add @x[15],@x[15],@d[7],lsr#32 + add $B2,$B2,@K[1] + + b.lo .Ltail_neon + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + add @K[3],@K[3],$ONE // += 4 + stp @x[8],@x[10],[$out,#32] + add @K[4],@K[4],$ONE + stp @x[12],@x[14],[$out,#48] + add @K[5],@K[5],$ONE + add $out,$out,#64 + + st1.8 {$A0-$D0},[$out],#64 + ld1.8 {$A0-$D0},[$inp],#64 + + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + eor $A2,$A2,$A0 + eor $B2,$B2,$B0 + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + b.hi .Loop_outer_neon + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret + +.Ltail_neon: + add $len,$len,#256 + cmp $len,#64 + b.lo .Less_than_64 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_128 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A0,$A0,$T0 + eor $B0,$B0,$T1 + eor $C0,$C0,$T2 + eor $D0,$D0,$T3 + st1.8 {$A0-$D0},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_192 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + + st1.8 {$A2-$D2},[sp] + b .Last_neon + +.Less_than_128: + st1.8 {$A0-$D0},[sp] + b .Last_neon +.Less_than_192: + st1.8 {$A1-$D1},[sp] + b .Last_neon + +.align 4 +.Last_neon: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + +.Loop_tail_neon: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail_neon + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + +.Ldone_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_neon,.-ChaCha20_neon +___ +{ +my ($T0,$T1,$T2,$T3,$T4,$T5)=@K; +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2, + $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23)); + +$code.=<<___; +#ifdef __APPLE__ +.type ChaCha20_512_neon,%function +.align 5 +ChaCha20_512_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + +.L512_or_more_neon: + sub sp,sp,#128+64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +# ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +# endif + add @K[3],@K[3],$ONE // += 1 + stp @K[0],@K[1],[sp,#0] // off-load key block, invariant part + add @K[3],@K[3],$ONE // not typo + str @K[2],[sp,#32] + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + add @K[6],@K[5],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + + stp d8,d9,[sp,#128+0] // meet ABI requirements + stp d10,d11,[sp,#128+16] + stp d12,d13,[sp,#128+32] + stp d14,d15,[sp,#128+48] + + sub $len,$len,#512 // not typo + +.Loop_outer_512_neon: + mov $A0,@K[0] + mov $A1,@K[0] + mov $A2,@K[0] + mov $A3,@K[0] + mov $A4,@K[0] + mov $A5,@K[0] + mov $B0,@K[1] + mov.32 @x[0],@d[0] // unpack key block + mov $B1,@K[1] + lsr @x[1],@d[0],#32 + mov $B2,@K[1] + mov.32 @x[2],@d[1] + mov $B3,@K[1] + lsr @x[3],@d[1],#32 + mov $B4,@K[1] + mov.32 @x[4],@d[2] + mov $B5,@K[1] + lsr @x[5],@d[2],#32 + mov $D0,@K[3] + mov.32 @x[6],@d[3] + mov $D1,@K[4] + lsr @x[7],@d[3],#32 + mov $D2,@K[5] + mov.32 @x[8],@d[4] + mov $D3,@K[6] + lsr @x[9],@d[4],#32 + mov $C0,@K[2] + mov.32 @x[10],@d[5] + mov $C1,@K[2] + lsr @x[11],@d[5],#32 + add $D4,$D0,$ONE // +4 + mov.32 @x[12],@d[6] + add $D5,$D1,$ONE // +4 + lsr @x[13],@d[6],#32 + mov $C2,@K[2] + mov.32 @x[14],@d[7] + mov $C3,@K[2] + lsr @x[15],@d[7],#32 + mov $C4,@K[2] + stp @K[3],@K[4],[sp,#48] // off-load key block, variable part + mov $C5,@K[2] + str @K[5],[sp,#80] + + mov $ctr,#5 + subs $len,$len,#512 +.Loop_upper_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + my $diff = ($#thread0+1)*6 - $#thread67 - 1; + my $i = 0; + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_upper_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + stp @x[4],@x[6],[$out,#16] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + stp @x[8],@x[10],[$out,#32] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#5 +.Loop_lower_neon: + sub $ctr,$ctr,#1 +___ + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_lower_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + ldp @K[0],@K[1],[sp,#0] + add @x[1],@x[1],@d[0],lsr#32 + ldp @K[2],@K[3],[sp,#32] + add.32 @x[2],@x[2],@d[1] + ldp @K[4],@K[5],[sp,#64] + add @x[3],@x[3],@d[1],lsr#32 + add $A0,$A0,@K[0] + add.32 @x[4],@x[4],@d[2] + add $A1,$A1,@K[0] + add @x[5],@x[5],@d[2],lsr#32 + add $A2,$A2,@K[0] + add.32 @x[6],@x[6],@d[3] + add $A3,$A3,@K[0] + add @x[7],@x[7],@d[3],lsr#32 + add $A4,$A4,@K[0] + add.32 @x[8],@x[8],@d[4] + add $A5,$A5,@K[0] + add @x[9],@x[9],@d[4],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[10],@x[10],@d[5] + add $C1,$C1,@K[2] + add @x[11],@x[11],@d[5],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[12],@x[12],@d[6] + add $C3,$C3,@K[2] + add @x[13],@x[13],@d[6],lsr#32 + add $C4,$C4,@K[2] + add.32 @x[14],@x[14],@d[7] + add $C5,$C5,@K[2] + add @x[15],@x[15],@d[7],lsr#32 + add $D4,$D4,$ONE // +4 + add @x[0],@x[0],@x[1],lsl#32 // pack + add $D5,$D5,$ONE // +4 + add @x[2],@x[2],@x[3],lsl#32 + add $D0,$D0,@K[3] + ldp @x[1],@x[3],[$inp,#0] // load input + add $D1,$D1,@K[4] + add @x[4],@x[4],@x[5],lsl#32 + add $D2,$D2,@K[5] + add @x[6],@x[6],@x[7],lsl#32 + add $D3,$D3,@K[6] + ldp @x[5],@x[7],[$inp,#16] + add $D4,$D4,@K[3] + add @x[8],@x[8],@x[9],lsl#32 + add $D5,$D5,@K[4] + add @x[10],@x[10],@x[11],lsl#32 + add $B0,$B0,@K[1] + ldp @x[9],@x[11],[$inp,#32] + add $B1,$B1,@K[1] + add @x[12],@x[12],@x[13],lsl#32 + add $B2,$B2,@K[1] + add @x[14],@x[14],@x[15],lsl#32 + add $B3,$B3,@K[1] + ldp @x[13],@x[15],[$inp,#48] + add $B4,$B4,@K[1] + add $inp,$inp,#64 + add $B5,$B5,@K[1] + +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#7 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + st1.8 {$A0-$D0},[$out],#64 + + ld1.8 {$A0-$D0},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + ld1.8 {$A1-$D1},[$inp],#64 + eor $A2,$A2,$A0 + ldp @K[0],@K[1],[sp,#0] + eor $B2,$B2,$B0 + ldp @K[2],@K[3],[sp,#32] + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + ld1.8 {$A2-$D2},[$inp],#64 + eor $A3,$A3,$A1 + eor $B3,$B3,$B1 + eor $C3,$C3,$C1 + eor $D3,$D3,$D1 + st1.8 {$A3-$D3},[$out],#64 + + ld1.8 {$A3-$D3},[$inp],#64 + eor $A4,$A4,$A2 + eor $B4,$B4,$B2 + eor $C4,$C4,$C2 + eor $D4,$D4,$D2 + st1.8 {$A4-$D4},[$out],#64 + + shl $A0,$ONE,#1 // 4 -> 8 + eor $A5,$A5,$A3 + eor $B5,$B5,$B3 + eor $C5,$C5,$C3 + eor $D5,$D5,$D3 + st1.8 {$A5-$D5},[$out],#64 + + add @K[3],@K[3],$A0 // += 8 + add @K[4],@K[4],$A0 + add @K[5],@K[5],$A0 + add @K[6],@K[6],$A0 + + b.hs .Loop_outer_512_neon + + adds $len,$len,#512 + ushr $A0,$ONE,#2 // 4 -> 1 + + ldp d8,d9,[sp,#128+0] // meet ABI requirements + ldp d10,d11,[sp,#128+16] + ldp d12,d13,[sp,#128+32] + ldp d14,d15,[sp,#128+48] + + stp @K[0],$ONE,[sp,#0] // wipe off-load area + stp @K[0],$ONE,[sp,#32] + stp @K[0],$ONE,[sp,#64] + + b.eq .Ldone_512_neon + + cmp $len,#192 + sub @K[3],@K[3],$A0 // -= 1 + sub @K[4],@K[4],$A0 + sub @K[5],@K[5],$A0 + add sp,sp,#128 + b.hs .Loop_outer_neon + + eor @K[1],@K[1],@K[1] + eor @K[2],@K[2],@K[2] + eor @K[3],@K[3],@K[3] + eor @K[4],@K[4],@K[4] + eor @K[5],@K[5],@K[5] + eor @K[6],@K[6],@K[6] + b .Loop_outer + +.Ldone_512_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#128+64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_512_neon,.-ChaCha20_512_neon +#endif +#endif +___ +} +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or + (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1)) or + (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or + (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or + (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1)); + + print $_,"\n"; +} +close STDOUT; # flush diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c new file mode 100644 index 0000000000000..96ce01e2c1333 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void chacha20_mips(u32 state[16], u8 *out, const u8 *in, + const size_t len); +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + chacha20_mips(ctx->state, dst, src, len); + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S new file mode 100644 index 0000000000000..a81e02db95e73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst . All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#define MASK_U32 0x3c +#define CHACHA20_BLOCK_SIZE 64 +#define STACK_SIZE 32 + +#define X0 $t0 +#define X1 $t1 +#define X2 $t2 +#define X3 $t3 +#define X4 $t4 +#define X5 $t5 +#define X6 $t6 +#define X7 $t7 +#define X8 $t8 +#define X9 $t9 +#define X10 $v1 +#define X11 $s6 +#define X12 $s5 +#define X13 $s4 +#define X14 $s3 +#define X15 $s2 +/* Use regs which are overwritten on exit for Tx so we don't leak clear data. */ +#define T0 $s1 +#define T1 $s0 +#define T(n) T ## n +#define X(n) X ## n + +/* Input arguments */ +#define STATE $a0 +#define OUT $a1 +#define IN $a2 +#define BYTES $a3 + +/* Output argument */ +/* NONCE[0] is kept in a register and not in memory. + * We don't want to touch original value in memory. + * Must be incremented every loop iteration. + */ +#define NONCE_0 $v0 + +/* SAVED_X and SAVED_CA are set in the jump table. + * Use regs which are overwritten on exit else we don't leak clear data. + * They are used to handling the last bytes which are not multiple of 4. + */ +#define SAVED_X X15 +#define SAVED_CA $s7 + +#define IS_UNALIGNED $s7 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#define ROTx rotl +#define ROTR(n) rotr n, 24 +#define CPU_TO_LE32(n) \ + wsbh n; \ + rotr n, 16; +#else +#define MSB 3 +#define LSB 0 +#define ROTx rotr +#define CPU_TO_LE32(n) +#define ROTR(n) +#endif + +#define FOR_EACH_WORD(x) \ + x( 0); \ + x( 1); \ + x( 2); \ + x( 3); \ + x( 4); \ + x( 5); \ + x( 6); \ + x( 7); \ + x( 8); \ + x( 9); \ + x(10); \ + x(11); \ + x(12); \ + x(13); \ + x(14); \ + x(15); + +#define FOR_EACH_WORD_REV(x) \ + x(15); \ + x(14); \ + x(13); \ + x(12); \ + x(11); \ + x(10); \ + x( 9); \ + x( 8); \ + x( 7); \ + x( 6); \ + x( 5); \ + x( 4); \ + x( 3); \ + x( 2); \ + x( 1); \ + x( 0); + +#define PLUS_ONE_0 1 +#define PLUS_ONE_1 2 +#define PLUS_ONE_2 3 +#define PLUS_ONE_3 4 +#define PLUS_ONE_4 5 +#define PLUS_ONE_5 6 +#define PLUS_ONE_6 7 +#define PLUS_ONE_7 8 +#define PLUS_ONE_8 9 +#define PLUS_ONE_9 10 +#define PLUS_ONE_10 11 +#define PLUS_ONE_11 12 +#define PLUS_ONE_12 13 +#define PLUS_ONE_13 14 +#define PLUS_ONE_14 15 +#define PLUS_ONE_15 16 +#define PLUS_ONE(x) PLUS_ONE_ ## x +#define _CONCAT3(a,b,c) a ## b ## c +#define CONCAT3(a,b,c) _CONCAT3(a,b,c) + +#define STORE_UNALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lwl T1, (x*4)+MSB ## (IN); \ + lwr T1, (x*4)+LSB ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + swl X ## x, (x*4)+MSB ## (OUT); \ + swr X ## x, (x*4)+LSB ## (OUT); + +#define STORE_ALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lw T1, (x*4) ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + sw X ## x, (x*4) ## (OUT); + +/* Jump table macro. + * Used for setup and handling the last bytes, which are not multiple of 4. + * X15 is free to store Xn + * Every jumptable entry must be equal in size. + */ +#define JMPTBL_ALIGNED(x) \ +.Lchacha20_mips_jmptbl_aligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_aligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define JMPTBL_UNALIGNED(x) \ +.Lchacha20_mips_jmptbl_unaligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_unaligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define AXR(A, B, C, D, K, L, M, N, V, W, Y, Z, S) \ + addu X(A), X(K); \ + addu X(B), X(L); \ + addu X(C), X(M); \ + addu X(D), X(N); \ + xor X(V), X(A); \ + xor X(W), X(B); \ + xor X(Y), X(C); \ + xor X(Z), X(D); \ + rotl X(V), S; \ + rotl X(W), S; \ + rotl X(Y), S; \ + rotl X(Z), S; + +.text +.set reorder +.set noat +.globl chacha20_mips +.ent chacha20_mips +chacha20_mips: + .frame $sp, STACK_SIZE, $ra + + addiu $sp, -STACK_SIZE + + /* Return bytes = 0. */ + beqz BYTES, .Lchacha20_mips_end + + lw NONCE_0, 48(STATE) + + /* Save s0-s7 */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + sw $s6, 24($sp) + sw $s7, 28($sp) + + /* Test IN or OUT is unaligned. + * IS_UNALIGNED = ( IN | OUT ) & 0x00000003 + */ + or IS_UNALIGNED, IN, OUT + andi IS_UNALIGNED, 0x3 + + /* Set number of rounds */ + li $at, 20 + + b .Lchacha20_rounds_start + +.align 4 +.Loop_chacha20_rounds: + addiu IN, CHACHA20_BLOCK_SIZE + addiu OUT, CHACHA20_BLOCK_SIZE + addiu NONCE_0, 1 + +.Lchacha20_rounds_start: + lw X0, 0(STATE) + lw X1, 4(STATE) + lw X2, 8(STATE) + lw X3, 12(STATE) + + lw X4, 16(STATE) + lw X5, 20(STATE) + lw X6, 24(STATE) + lw X7, 28(STATE) + lw X8, 32(STATE) + lw X9, 36(STATE) + lw X10, 40(STATE) + lw X11, 44(STATE) + + move X12, NONCE_0 + lw X13, 52(STATE) + lw X14, 56(STATE) + lw X15, 60(STATE) + +.Loop_chacha20_xor_rounds: + addiu $at, -2 + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 16); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 12); + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 8); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 7); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 16); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 12); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 8); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 7); + bnez $at, .Loop_chacha20_xor_rounds + + addiu BYTES, -(CHACHA20_BLOCK_SIZE) + + /* Is data src/dst unaligned? Jump */ + bnez IS_UNALIGNED, .Loop_chacha20_unaligned + + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES < 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_aligned + + FOR_EACH_WORD_REV(STORE_ALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + + /* BYTES < 0? Handle last bytes */ + bltz BYTES, .Lchacha20_mips_xor_bytes + +.Lchacha20_mips_xor_done: + /* Restore used registers */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + lw $s6, 24($sp) + lw $s7, 28($sp) + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + +.Lchacha20_mips_end: + addiu $sp, STACK_SIZE + jr $ra + +.Lchacha20_mips_no_full_block_aligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_aligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_aligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_ALIGNED) + + +.Loop_chacha20_unaligned: + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES > 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_unaligned + + FOR_EACH_WORD_REV(STORE_UNALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + + .set noreorder + /* Fall through to byte handling */ + bgez BYTES, .Lchacha20_mips_xor_done +.Lchacha20_mips_xor_unaligned_0_b: +.Lchacha20_mips_xor_aligned_0_b: + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + .set reorder + +.Lchacha20_mips_xor_bytes: + addu IN, $at + addu OUT, $at + /* First byte */ + lbu T1, 0(IN) + addiu $at, BYTES, 1 + CPU_TO_LE32(SAVED_X) + ROTR(SAVED_X) + xor T1, SAVED_X + sb T1, 0(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Second byte */ + lbu T1, 1(IN) + addiu $at, BYTES, 2 + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 1(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Third byte */ + lbu T1, 2(IN) + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 2(OUT) + b .Lchacha20_mips_xor_done + +.Lchacha20_mips_no_full_block_unaligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_UNALIGNED) +.end chacha20_mips +.set at diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S new file mode 100644 index 0000000000000..8fb4bc2e7b5b2 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S @@ -0,0 +1,461 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Google, Inc. + */ + +#include +#include + +/* + * Design notes: + * + * 16 registers would be needed to hold the state matrix, but only 14 are + * available because 'sp' and 'pc' cannot be used. So we spill the elements + * (x8, x9) to the stack and swap them out with (x10, x11). This adds one + * 'ldrd' and one 'strd' instruction per round. + * + * All rotates are performed using the implicit rotate operand accepted by the + * 'add' and 'eor' instructions. This is faster than using explicit rotate + * instructions. To make this work, we allow the values in the second and last + * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the + * wrong rotation amount. The rotation amount is then fixed up just in time + * when the values are used. 'brot' is the number of bits the values in row 'b' + * need to be rotated right to arrive at the correct values, and 'drot' + * similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such + * that they end up as (25, 24) after every round. + */ + + // ChaCha state registers + X0 .req r0 + X1 .req r1 + X2 .req r2 + X3 .req r3 + X4 .req r4 + X5 .req r5 + X6 .req r6 + X7 .req r7 + X8_X10 .req r8 // shared by x8 and x10 + X9_X11 .req r9 // shared by x9 and x11 + X12 .req r10 + X13 .req r11 + X14 .req r12 + X15 .req r14 + +.Lexpand_32byte_k: + // "expand 32-byte k" + .word 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + +#ifdef __thumb2__ +# define adrl adr +#endif + +.macro __rev out, in, t0, t1, t2 +.if __LINUX_ARM_ARCH__ >= 6 + rev \out, \in +.else + lsl \t0, \in, #24 + and \t1, \in, #0xff00 + and \t2, \in, #0xff0000 + orr \out, \t0, \in, lsr #24 + orr \out, \out, \t1, lsl #8 + orr \out, \out, \t2, lsr #8 +.endif +.endm + +.macro _le32_bswap x, t0, t1, t2 +#ifdef __ARMEB__ + __rev \x, \x, \t0, \t1, \t2 +#endif +.endm + +.macro _le32_bswap_4x a, b, c, d, t0, t1, t2 + _le32_bswap \a, \t0, \t1, \t2 + _le32_bswap \b, \t0, \t1, \t2 + _le32_bswap \c, \t0, \t1, \t2 + _le32_bswap \d, \t0, \t1, \t2 +.endm + +.macro __ldrd a, b, src, offset +#if __LINUX_ARM_ARCH__ >= 6 + ldrd \a, \b, [\src, #\offset] +#else + ldr \a, [\src, #\offset] + ldr \b, [\src, #\offset + 4] +#endif +.endm + +.macro __strd a, b, dst, offset +#if __LINUX_ARM_ARCH__ >= 6 + strd \a, \b, [\dst, #\offset] +#else + str \a, [\dst, #\offset] + str \b, [\dst, #\offset + 4] +#endif +.endm + +.macro _halfround a1, b1, c1, d1, a2, b2, c2, d2 + + // a += b; d ^= a; d = rol(d, 16); + add \a1, \a1, \b1, ror #brot + add \a2, \a2, \b2, ror #brot + eor \d1, \a1, \d1, ror #drot + eor \d2, \a2, \d2, ror #drot + // drot == 32 - 16 == 16 + + // c += d; b ^= c; b = rol(b, 12); + add \c1, \c1, \d1, ror #16 + add \c2, \c2, \d2, ror #16 + eor \b1, \c1, \b1, ror #brot + eor \b2, \c2, \b2, ror #brot + // brot == 32 - 12 == 20 + + // a += b; d ^= a; d = rol(d, 8); + add \a1, \a1, \b1, ror #20 + add \a2, \a2, \b2, ror #20 + eor \d1, \a1, \d1, ror #16 + eor \d2, \a2, \d2, ror #16 + // drot == 32 - 8 == 24 + + // c += d; b ^= c; b = rol(b, 7); + add \c1, \c1, \d1, ror #24 + add \c2, \c2, \d2, ror #24 + eor \b1, \c1, \b1, ror #20 + eor \b2, \c2, \b2, ror #20 + // brot == 32 - 7 == 25 +.endm + +.macro _doubleround + + // column round + + // quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13) + _halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13 + + // save (x8, x9); restore (x10, x11) + __strd X8_X10, X9_X11, sp, 0 + __ldrd X8_X10, X9_X11, sp, 8 + + // quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15) + _halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15 + + .set brot, 25 + .set drot, 24 + + // diagonal round + + // quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12) + _halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12 + + // save (x10, x11); restore (x8, x9) + __strd X8_X10, X9_X11, sp, 8 + __ldrd X8_X10, X9_X11, sp, 0 + + // quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14) + _halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14 +.endm + +.macro _chacha_permute nrounds + .set brot, 0 + .set drot, 0 + .rept \nrounds / 2 + _doubleround + .endr +.endm + +.macro _chacha nrounds + +.Lnext_block\@: + // Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + + // Do the core ChaCha permutation to update x0-x15. + _chacha_permute \nrounds + + add sp, #8 + // Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15). + push {X8_X10, X9_X11, X12, X13, X14, X15} + + // Load (OUT, IN, LEN). + ldr r14, [sp, #96] + ldr r12, [sp, #100] + ldr r11, [sp, #104] + + orr r10, r14, r12 + + // Use slow path if fewer than 64 bytes remain. + cmp r11, #64 + blt .Lxor_slowpath\@ + + // Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on + // ARMv6+, since ldmia and stmia (used below) still require alignment. + tst r10, #3 + bne .Lxor_slowpath\@ + + // Fast path: XOR 64 bytes of aligned data. + + // Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // x0-x3 + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + ldmia r12!, {r8-r11} + eor X0, X0, r8 + eor X1, X1, r9 + eor X2, X2, r10 + eor X3, X3, r11 + stmia r14!, {X0-X3} + + // x4-x7 + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + ldmia r12!, {X0-X3} + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + eor X4, X4, X0 + eor X5, X5, X1 + eor X6, X6, X2 + eor X7, X7, X3 + stmia r14!, {X4-X7} + + // x8-x15 + pop {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + ldmia r12!, {r8-r11} + eor r0, r0, r8 // x8 + eor r1, r1, r9 // x9 + eor r6, r6, r10 // x10 + eor r7, r7, r11 // x11 + stmia r14!, {r0,r1,r6,r7} + ldmia r12!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + ldr r9, [sp, #72] // load LEN + eor r2, r2, r0 // x12 + eor r3, r3, r1 // x13 + eor r4, r4, r6 // x14 + eor r5, r5, r7 // x15 + subs r9, #64 // decrement and check LEN + stmia r14!, {r2-r5} + + beq .Ldone\@ + +.Lprepare_for_next_block\@: + + // Stack: x0-x15 OUT IN LEN + + // Increment block counter (x12) + add r8, #1 + + // Store updated (OUT, IN, LEN) + str r14, [sp, #64] + str r12, [sp, #68] + str r9, [sp, #72] + + mov r14, sp + + // Store updated block counter (x12) + str r8, [sp, #48] + + sub sp, #16 + + // Reload state and do next block + ldmia r14!, {r0-r11} // load x0-x11 + __strd r10, r11, sp, 8 // store x10-x11 before state + ldmia r14, {r10-r12,r14} // load x12-x15 + b .Lnext_block\@ + +.Lxor_slowpath\@: + // Slow path: < 64 bytes remaining, or unaligned input or output buffer. + // We handle it by storing the 64 bytes of keystream to the stack, then + // XOR-ing the needed portion with the data. + + // Allocate keystream buffer + sub sp, #64 + mov r14, sp + + // Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Save keystream for x0-x3 + __ldrd r8, r9, sp, 96 + __ldrd r10, r11, sp, 104 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + stmia r14!, {X0-X3} + + // Save keystream for x4-x7 + __ldrd r8, r9, sp, 112 + __ldrd r10, r11, sp, 120 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + add r8, sp, #64 + stmia r14!, {X4-X7} + + // Save keystream for x8-x15 + ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 128 + __ldrd r10, r11, sp, 136 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + stmia r14!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 144 + __ldrd r10, r11, sp, 152 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + stmia r14, {r2-r5} + + // Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN + // Registers: r8 is block counter, r12 is IN. + + ldr r9, [sp, #168] // LEN + ldr r14, [sp, #160] // OUT + cmp r9, #64 + mov r0, sp + movle r1, r9 + movgt r1, #64 + // r1 is number of bytes to XOR, in range [1, 64] + +.if __LINUX_ARM_ARCH__ < 6 + orr r2, r12, r14 + tst r2, #3 // IN or OUT misaligned? + bne .Lxor_next_byte\@ +.endif + + // XOR a word at a time +.rept 16 + subs r1, #4 + blt .Lxor_words_done\@ + ldr r2, [r12], #4 + ldr r3, [r0], #4 + eor r2, r2, r3 + str r2, [r14], #4 +.endr + b .Lxor_slowpath_done\@ +.Lxor_words_done\@: + ands r1, r1, #3 + beq .Lxor_slowpath_done\@ + + // XOR a byte at a time +.Lxor_next_byte\@: + ldrb r2, [r12], #1 + ldrb r3, [r0], #1 + eor r2, r2, r3 + strb r2, [r14], #1 + subs r1, #1 + bne .Lxor_next_byte\@ + +.Lxor_slowpath_done\@: + subs r9, #64 + add sp, #96 + bgt .Lprepare_for_next_block\@ + +.Ldone\@: +.endm // _chacha + +/* + * void chacha20_arm(u8 *out, const u8 *in, size_t len, const u32 key[8], + * const u32 iv[4]); + */ +SYM_FUNC_START(chacha20_arm) + cmp r2, #0 // len == 0? + reteq lr + + push {r0-r2,r4-r11,lr} + + // Push state x0-x15 onto stack. + // Also store an extra copy of x10-x11 just before the state. + + ldr r4, [sp, #48] // iv + mov r0, sp + sub sp, #80 + + // iv: x12-x15 + ldm r4, {X12,X13,X14,X15} + stmdb r0!, {X12,X13,X14,X15} + + // key: x4-x11 + __ldrd X8_X10, X9_X11, r3, 24 + __strd X8_X10, X9_X11, sp, 8 + stmdb r0!, {X8_X10, X9_X11} + ldm r3, {X4-X9_X11} + stmdb r0!, {X4-X9_X11} + + // constants: x0-x3 + adrl X3, .Lexpand_32byte_k + ldm X3, {X0-X3} + __strd X0, X1, sp, 16 + __strd X2, X3, sp, 24 + + _chacha 20 + + add sp, #76 + pop {r4-r11, pc} +SYM_FUNC_END(chacha20_arm) + +/* + * void hchacha20_arm(const u32 state[16], u32 out[8]); + */ +SYM_FUNC_START(hchacha20_arm) + push {r1,r4-r11,lr} + + mov r14, r0 + ldmia r14!, {r0-r11} // load x0-x11 + push {r10-r11} // store x10-x11 to stack + ldm r14, {r10-r12,r14} // load x12-x15 + sub sp, #8 + + _chacha_permute 20 + + // Skip over (unused0-unused1, x10-x11) + add sp, #16 + + // Fix up rotations of x12-x15 + ror X12, X12, #drot + ror X13, X13, #drot + pop {r4} // load 'out' + ror X14, X14, #drot + ror X15, X15, #drot + + // Store (x0-x3,x12-x15) to 'out' + stm r4, {X0,X1,X2,X3,X12,X13,X14,X15} + + pop {r4-r11,pc} +SYM_FUNC_END(hchacha20_arm) diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c new file mode 100644 index 0000000000000..5ac5f686a641a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void hchacha20_ssse3(u32 *derived_key, const u8 *nonce, + const u8 *key); +asmlinkage void chacha20_ssse3(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx2(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512vl(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_ssse3 __ro_after_init; +static bool chacha20_use_avx2 __ro_after_init; +static bool chacha20_use_avx512 __ro_after_init; +static bool chacha20_use_avx512vl __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { + &chacha20_use_ssse3, &chacha20_use_avx2, &chacha20_use_avx512, + &chacha20_use_avx512vl }; + +static void __init chacha20_fpu_init(void) +{ + chacha20_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); + chacha20_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + chacha20_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; + chacha20_use_avx512vl = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !chacha20_use_ssse3 || + len <= CHACHA20_BLOCK_SIZE || !simd_use(simd_context)) + return false; + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 && + len >= CHACHA20_BLOCK_SIZE * 8) + chacha20_avx512(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx512vl(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx2(dst, src, bytes, ctx->key, ctx->counter); + else + chacha20_ssse3(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_AS_SSSE3) && chacha20_use_ssse3 && + simd_use(simd_context)) { + hchacha20_ssse3(derived_key, nonce, key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl new file mode 100644 index 0000000000000..29906a66b8b73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl @@ -0,0 +1,4106 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# November 2014 +# +# ChaCha20 for x86_64. +# +# December 2016 +# +# Add AVX512F code path. +# +# December 2017 +# +# Add AVX512VL code path. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc 4.8(i) 1x/2xSSSE3(ii) 4xSSSE3 NxAVX(v) +# +# P4 9.48/+99% - - +# Core2 7.83/+55% 7.90/5.76 4.35 +# Westmere 7.19/+50% 5.60/4.50 3.00 +# Sandy Bridge 8.31/+42% 5.45/4.00 2.72 +# Ivy Bridge 6.71/+46% 5.40/? 2.41 +# Haswell 5.92/+43% 5.20/3.45 2.42 1.23 +# Skylake[-X] 5.87/+39% 4.70/3.22 2.31 1.19[0.80(vi)] +# Silvermont 12.0/+33% 7.75/6.90 7.03(iii) +# Knights L 11.7/- ? 9.60(iii) 0.80 +# Goldmont 10.6/+17% 5.10/3.52 3.28 +# Sledgehammer 7.28/+52% - - +# Bulldozer 9.66/+28% 9.85/5.35(iv) 3.06(iv) +# Ryzen 5.96/+50% 5.19/3.00 2.40 2.09 +# VIA Nano 10.5/+46% 6.72/6.88 6.05 +# +# (i) compared to older gcc 3.x one can observe >2x improvement on +# most platforms; +# (ii) 2xSSSE3 is code path optimized specifically for 128 bytes used +# by chacha20_poly1305_tls_cipher, results are EVP-free; +# (iii) this is not optimal result for Atom because of MSROM +# limitations, SSE2 can do better, but gain is considered too +# low to justify the [maintenance] effort; +# (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20 +# and 4.85 for 128-byte inputs; +# (v) 8xAVX2, 8xAVX512VL or 16xAVX512F, whichever best applicable; +# (vi) even though Skylake-X can execute AVX512F code and deliver 0.57 +# cpb in single thread, the corresponding capability is suppressed; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +# input parameter block +($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8"); + +$code.=<<___ if $kernel; +#include +___ + +sub declare_variable() { + my ($name, $size, $type, $payload) = @_; + if($kernel) { + $code.=".section .rodata.cst$size.L$name, \"aM\", \@progbits, $size\n"; + $code.=".align $size\n"; + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } else { + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +if(!$kernel) { + $code .= ".text\n"; +} +&declare_variable('zero', 16, 'long', '0,0,0,0'); +&declare_variable('one', 16, 'long', '1,0,0,0'); +&declare_variable('inc', 16, 'long', '0,1,2,3'); +&declare_variable('four', 16, 'long', '4,4,4,4'); +&declare_variable('incy', 32, 'long', '0,2,4,6,1,3,5,7'); +&declare_variable('eight', 32, 'long', '8,8,8,8,8,8,8,8'); +&declare_variable('rot16', 16, 'byte', '0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd'); +&declare_variable('rot24', 16, 'byte', '0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe'); +&declare_variable('twoy', 32, 'long', '2,0,0,0, 2,0,0,0'); +&declare_variable('zeroz', 64, 'long', '0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0'); +&declare_variable('fourz', 64, 'long', '4,0,0,0, 4,0,0,0, 4,0,0,0, 4,0,0,0'); +&declare_variable('incz', 64, 'long', '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15'); +&declare_variable('sixteen', 64, 'long', '16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16'); +&declare_variable('sigma', 16, 'ascii', '"expand 32-byte k"'); + +$code.=<<___ if !$kernel; +.asciz "ChaCha20 for x86_64, CRYPTOGAMS by " +___ +$code.=".text\n"; + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +@x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)), + "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15))); +@t=("%esi","%edi"); + +sub ROUND { # critical path is 24 cycles per round +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_)=map("\"$_\"",@t); +my @x=map("\"$_\"",@x); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + # Normally instructions would be interleaved to favour in-order + # execution. Generally out-of-order cores manage it gracefully, + # but not this time for some reason. As in-order execution + # cores are dying breed, old Atom is the only one around, + # instructions are left uninterleaved. Besides, Atom is better + # off executing 1xSSSE3 code anyway... + + ( + "&add (@x[$a0],@x[$b0])", # Q1 + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],16)", + "&add (@x[$a1],@x[$b1])", # Q2 + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],16)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],12)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],12)", + + "&add (@x[$a0],@x[$b0])", + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],8)", + "&add (@x[$a1],@x[$b1])", + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],8)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],7)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],7)", + + "&mov (\"4*$c0(%rsp)\",$xc)", # reload pair of 'c's + "&mov (\"4*$c1(%rsp)\",$xc_)", + "&mov ($xc,\"4*$c2(%rsp)\")", + "&mov ($xc_,\"4*$c3(%rsp)\")", + + "&add (@x[$a2],@x[$b2])", # Q3 + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],16)", + "&add (@x[$a3],@x[$b3])", # Q4 + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],16)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],12)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],12)", + + "&add (@x[$a2],@x[$b2])", + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],8)", + "&add (@x[$a3],@x[$b3])", + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],8)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],7)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],7)" + ); +} + +######################################################################## +# Generic code path that handles all lengths on pre-SSSE3 processors. +if(!$kernel) { +&declare_function("chacha20_ctr32", 64, 5); +$code.=<<___; +.cfi_startproc + cmp \$0,$len + je .Lno_data + mov OPENSSL_ia32cap_P+4(%rip),%r9 +___ +$code.=<<___ if ($avx>2); + bt \$48,%r9 # check for AVX512F + jc .Lchacha20_avx512 + test %r9,%r9 # check for AVX512VL + js .Lchacha20_avx512vl +___ +$code.=<<___; + test \$`1<<(41-32)`,%r9d + jnz .Lchacha20_ssse3 +___ +$code.=<<___; + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$64+24,%rsp +.cfi_adjust_cfa_offset 64+24 +.Lctr32_body: + + #movdqa .Lsigma(%rip),%xmm0 + movdqu ($key),%xmm1 + movdqu 16($key),%xmm2 + movdqu ($counter),%xmm3 + movdqa .Lone(%rip),%xmm4 + + #movdqa %xmm0,4*0(%rsp) # key[0] + movdqa %xmm1,4*4(%rsp) # key[1] + movdqa %xmm2,4*8(%rsp) # key[2] + movdqa %xmm3,4*12(%rsp) # key[3] + mov $len,%rbp # reassign $len + jmp .Loop_outer + +.align 32 +.Loop_outer: + mov \$0x61707865,@x[0] # 'expa' + mov \$0x3320646e,@x[1] # 'nd 3' + mov \$0x79622d32,@x[2] # '2-by' + mov \$0x6b206574,@x[3] # 'te k' + mov 4*4(%rsp),@x[4] + mov 4*5(%rsp),@x[5] + mov 4*6(%rsp),@x[6] + mov 4*7(%rsp),@x[7] + movd %xmm3,@x[12] + mov 4*13(%rsp),@x[13] + mov 4*14(%rsp),@x[14] + mov 4*15(%rsp),@x[15] + + mov %rbp,64+0(%rsp) # save len + mov \$10,%ebp + mov $inp,64+8(%rsp) # save inp + movq %xmm2,%rsi # "@x[8]" + mov $out,64+16(%rsp) # save out + mov %rsi,%rdi + shr \$32,%rdi # "@x[9]" + jmp .Loop + +.align 32 +.Loop: +___ + foreach (&ROUND (0, 4, 8,12)) { eval; } + foreach (&ROUND (0, 5,10,15)) { eval; } + &dec ("%ebp"); + &jnz (".Loop"); + +$code.=<<___; + mov @t[1],4*9(%rsp) # modulo-scheduled + mov @t[0],4*8(%rsp) + mov 64(%rsp),%rbp # load len + movdqa %xmm2,%xmm1 + mov 64+8(%rsp),$inp # load inp + paddd %xmm4,%xmm3 # increment counter + mov 64+16(%rsp),$out # load out + + add \$0x61707865,@x[0] # 'expa' + add \$0x3320646e,@x[1] # 'nd 3' + add \$0x79622d32,@x[2] # '2-by' + add \$0x6b206574,@x[3] # 'te k' + add 4*4(%rsp),@x[4] + add 4*5(%rsp),@x[5] + add 4*6(%rsp),@x[6] + add 4*7(%rsp),@x[7] + add 4*12(%rsp),@x[12] + add 4*13(%rsp),@x[13] + add 4*14(%rsp),@x[14] + add 4*15(%rsp),@x[15] + paddd 4*8(%rsp),%xmm1 + + cmp \$64,%rbp + jb .Ltail + + xor 4*0($inp),@x[0] # xor with input + xor 4*1($inp),@x[1] + xor 4*2($inp),@x[2] + xor 4*3($inp),@x[3] + xor 4*4($inp),@x[4] + xor 4*5($inp),@x[5] + xor 4*6($inp),@x[6] + xor 4*7($inp),@x[7] + movdqu 4*8($inp),%xmm0 + xor 4*12($inp),@x[12] + xor 4*13($inp),@x[13] + xor 4*14($inp),@x[14] + xor 4*15($inp),@x[15] + lea 4*16($inp),$inp # inp+=64 + pxor %xmm1,%xmm0 + + movdqa %xmm2,4*8(%rsp) + movd %xmm3,4*12(%rsp) + + mov @x[0],4*0($out) # write output + mov @x[1],4*1($out) + mov @x[2],4*2($out) + mov @x[3],4*3($out) + mov @x[4],4*4($out) + mov @x[5],4*5($out) + mov @x[6],4*6($out) + mov @x[7],4*7($out) + movdqu %xmm0,4*8($out) + mov @x[12],4*12($out) + mov @x[13],4*13($out) + mov @x[14],4*14($out) + mov @x[15],4*15($out) + lea 4*16($out),$out # out+=64 + + sub \$64,%rbp + jnz .Loop_outer + + jmp .Ldone + +.align 16 +.Ltail: + mov @x[0],4*0(%rsp) + mov @x[1],4*1(%rsp) + xor %rbx,%rbx + mov @x[2],4*2(%rsp) + mov @x[3],4*3(%rsp) + mov @x[4],4*4(%rsp) + mov @x[5],4*5(%rsp) + mov @x[6],4*6(%rsp) + mov @x[7],4*7(%rsp) + movdqa %xmm1,4*8(%rsp) + mov @x[12],4*12(%rsp) + mov @x[13],4*13(%rsp) + mov @x[14],4*14(%rsp) + mov @x[15],4*15(%rsp) + +.Loop_tail: + movzb ($inp,%rbx),%eax + movzb (%rsp,%rbx),%edx + lea 1(%rbx),%rbx + xor %edx,%eax + mov %al,-1($out,%rbx) + dec %rbp + jnz .Loop_tail + +.Ldone: + add \$64+24,%rsp +.cfi_adjust_cfa_offset -64-24 + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbp +.cfi_restore %rbp + pop %rbx +.cfi_restore %rbx +.Lno_data: + ret +.cfi_endproc +___ +&end_function("chacha20_ctr32"); +} + +######################################################################## +# SSSE3 code path that handles shorter lengths +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7)); + +sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot16); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,20); + &pslld ($t,12); + &por ($b,$t); + + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot24); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,25); + &pslld ($t,7); + &por ($b,$t); +} + +my $xframe = $win64 ? 32+8 : 8; + +if($kernel) { + $code .= "#ifdef CONFIG_AS_SSSE3\n"; +} + +if($kernel) { +&declare_function("hchacha20_ssse3", 32, 5); +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($len),$b + movdqu 16($len),$c + movdqu ($inp),$d + # This code is only used when targeting kernel. + # If targeting win64, xmm{6,7} preserving needs to be added. + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + mov \$10,$counter # reuse $counter + jmp 1f +.align 32 +1: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz ("1b"); + +$code.=<<___; + movdqu $a, ($out) + movdqu $d, 16($out) + ret +___ +&end_function("hchacha20_ssse3"); +} + +&declare_function("chacha20_ssse3", 32, 5); +$code.=<<___; +.cfi_startproc + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if ($avx && !$kernel); + test \$`1<<(43-32)`,%r10d + jnz .Lchacha20_4xop # XOP is fastest even if we use 1/4 +___ +$code.=<<___; + cmp \$128,$len # we might throw away some data, + je .Lchacha20_128 + ja .Lchacha20_4x # but overall it won't be slower + +.Ldo_ssse3_after_all: + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lssse3_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_ssse3 + +.align 32 +.Loop_outer_ssse3: + movdqa .Lone(%rip),$d + movdqa 0x00(%rsp),$a + movdqa 0x10(%rsp),$b + movdqa 0x20(%rsp),$c + paddd 0x30(%rsp),$d + mov \$10,$counter + movdqa $d,0x30(%rsp) + jmp .Loop_ssse3 + +.align 32 +.Loop_ssse3: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz (".Loop_ssse3"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + + cmp \$64,$len + jb .Ltail_ssse3 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + lea 0x40($inp),$inp # inp+=64 + pxor $t,$c + pxor $t1,$d + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + lea 0x40($out),$out # out+=64 + + sub \$64,$len + jnz .Loop_outer_ssse3 + + jmp .Ldone_ssse3 + +.align 16 +.Ltail_ssse3: + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + xor $counter,$counter + +.Loop_tail_ssse3: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_ssse3 + +.Ldone_ssse3: +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lssse3_epilogue: + ret +.cfi_endproc +___ +} +&end_function("chacha20_ssse3"); + +######################################################################## +# SSSE3 code path that handles 128-byte inputs +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(8,9,2..7)); +my ($a1,$b1,$c1,$d1)=map("%xmm$_",(10,11,0,1)); + +sub SSSE3ROUND_2x { + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot16); + &pshufb($d1,$rot16); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,20); + &movdqa($t1,$b1); + &pslld ($t,12); + &psrld ($b1,20); + &por ($b,$t); + &pslld ($t1,12); + &por ($b1,$t1); + + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot24); + &pshufb($d1,$rot24); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,25); + &movdqa($t1,$b1); + &pslld ($t,7); + &psrld ($b1,25); + &por ($b,$t); + &pslld ($t1,7); + &por ($b1,$t1); +} + +my $xframe = $win64 ? 0x68 : 8; + +$code.=<<___; +.type chacha20_128,\@function,5 +.align 32 +chacha20_128: +.cfi_startproc +.Lchacha20_128: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x70(%r10) + movaps %xmm7,-0x60(%r10) + movaps %xmm8,-0x50(%r10) + movaps %xmm9,-0x40(%r10) + movaps %xmm10,-0x30(%r10) + movaps %xmm11,-0x20(%r10) +.L128_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lone(%rip),$d1 + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,$a1 + movdqa $a,0x00(%rsp) + movdqa $b,$b1 + movdqa $b,0x10(%rsp) + movdqa $c,$c1 + movdqa $c,0x20(%rsp) + paddd $d,$d1 + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_128 + +.align 32 +.Loop_128: +___ + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &pshufd ($a1,$a1,0b10010011); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b00111001); + + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + &pshufd ($a1,$a1,0b00111001); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b10010011); + + &dec ($counter); + &jnz (".Loop_128"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + paddd .Lone(%rip),$d1 + paddd 0x00(%rsp),$a1 + paddd 0x10(%rsp),$b1 + paddd 0x20(%rsp),$c1 + paddd 0x30(%rsp),$d1 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + pxor $t,$c + movdqu 0x40($inp),$t + pxor $t1,$d + movdqu 0x50($inp),$t1 + pxor $t,$a1 + movdqu 0x60($inp),$t + pxor $t1,$b1 + movdqu 0x70($inp),$t1 + pxor $t,$c1 + pxor $t1,$d1 + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + movdqu $a1,0x40($out) + movdqu $b1,0x50($out) + movdqu $c1,0x60($out) + movdqu $d1,0x70($out) +___ +$code.=<<___ if ($win64); + movaps -0x70(%r10),%xmm6 + movaps -0x60(%r10),%xmm7 + movaps -0x50(%r10),%xmm8 + movaps -0x40(%r10),%xmm9 + movaps -0x30(%r10),%xmm10 + movaps -0x20(%r10),%xmm11 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L128_epilogue: + ret +.cfi_endproc +.size chacha20_128,.-chacha20_128 +___ +} + +######################################################################## +# SSSE3 code path that handles longer messages. +{ +# assign variables to favor Atom front-end +my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3, + $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub SSSE3_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&paddd (@x[$a0],@x[$b0])", # Q1 + "&paddd (@x[$a1],@x[$b1])", # Q2 + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t1)", + "&pshufb (@x[$d1],$t1)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t0,@x[$b0])", + "&pslld (@x[$b0],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b1])", + "&pslld (@x[$b1],12)", + "&por (@x[$b0],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b1],$t1)", + + "&paddd (@x[$a0],@x[$b0])", + "&paddd (@x[$a1],@x[$b1])", + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t0)", + "&pshufb (@x[$d1],$t0)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t1,@x[$b0])", + "&pslld (@x[$b0],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b1])", + "&pslld (@x[$b1],7)", + "&por (@x[$b0],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b1],$t0)", + + "&movdqa (\"`16*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&movdqa (\"`16*($c1-8)`(%rsp)\",$xc_)", + "&movdqa ($xc,\"`16*($c2-8)`(%rsp)\")", + "&movdqa ($xc_,\"`16*($c3-8)`(%rsp)\")", + + "&paddd (@x[$a2],@x[$b2])", # Q3 + "&paddd (@x[$a3],@x[$b3])", # Q4 + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t1)", + "&pshufb (@x[$d3],$t1)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t0,@x[$b2])", + "&pslld (@x[$b2],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b3])", + "&pslld (@x[$b3],12)", + "&por (@x[$b2],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b3],$t1)", + + "&paddd (@x[$a2],@x[$b2])", + "&paddd (@x[$a3],@x[$b3])", + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t0)", + "&pshufb (@x[$d3],$t0)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t1,@x[$b2])", + "&pslld (@x[$b2],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b3])", + "&pslld (@x[$b3],7)", + "&por (@x[$b2],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b3],$t0)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_4x,\@function,5 +.align 32 +chacha20_4x: +.cfi_startproc +.Lchacha20_4x: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if (!$kernel); + mov %r9,%r11 +___ +$code.=<<___ if ($avx>1 && !$kernel); + shr \$32,%r9 # OPENSSL_ia32cap_P+8 + test \$`1<<5`,%r9 # test AVX2 + jnz .Lchacha20_8x +___ +$code.=<<___; + cmp \$192,$len + ja .Lproceed4x +___ +$code.=<<___ if (!$kernel); + and \$`1<<26|1<<22`,%r11 # isolate XSAVE+MOVBE + cmp \$`1<<22`,%r11 # check for MOVBE without XSAVE + je .Ldo_ssse3_after_all # to detect Atom +___ +$code.=<<___; +.Lproceed4x: + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4x_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$xa3 # key[0] + movdqu ($key),$xb3 # key[1] + movdqu 16($key),$xt3 # key[2] + movdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + pshufd \$0x00,$xa3,$xa0 # smash key by lanes... + pshufd \$0x55,$xa3,$xa1 + movdqa $xa0,0x40(%rsp) # ... and offload + pshufd \$0xaa,$xa3,$xa2 + movdqa $xa1,0x50(%rsp) + pshufd \$0xff,$xa3,$xa3 + movdqa $xa2,0x60(%rsp) + movdqa $xa3,0x70(%rsp) + + pshufd \$0x00,$xb3,$xb0 + pshufd \$0x55,$xb3,$xb1 + movdqa $xb0,0x80-0x100(%rcx) + pshufd \$0xaa,$xb3,$xb2 + movdqa $xb1,0x90-0x100(%rcx) + pshufd \$0xff,$xb3,$xb3 + movdqa $xb2,0xa0-0x100(%rcx) + movdqa $xb3,0xb0-0x100(%rcx) + + pshufd \$0x00,$xt3,$xt0 # "$xc0" + pshufd \$0x55,$xt3,$xt1 # "$xc1" + movdqa $xt0,0xc0-0x100(%rcx) + pshufd \$0xaa,$xt3,$xt2 # "$xc2" + movdqa $xt1,0xd0-0x100(%rcx) + pshufd \$0xff,$xt3,$xt3 # "$xc3" + movdqa $xt2,0xe0-0x100(%rcx) + movdqa $xt3,0xf0-0x100(%rcx) + + pshufd \$0x00,$xd3,$xd0 + pshufd \$0x55,$xd3,$xd1 + paddd .Linc(%rip),$xd0 # don't save counters yet + pshufd \$0xaa,$xd3,$xd2 + movdqa $xd1,0x110-0x100(%rcx) + pshufd \$0xff,$xd3,$xd3 + movdqa $xd2,0x120-0x100(%rcx) + movdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4x + +.align 32 +.Loop_outer4x: + movdqa 0x40(%rsp),$xa0 # re-load smashed key + movdqa 0x50(%rsp),$xa1 + movdqa 0x60(%rsp),$xa2 + movdqa 0x70(%rsp),$xa3 + movdqa 0x80-0x100(%rcx),$xb0 + movdqa 0x90-0x100(%rcx),$xb1 + movdqa 0xa0-0x100(%rcx),$xb2 + movdqa 0xb0-0x100(%rcx),$xb3 + movdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + movdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + movdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + movdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + movdqa 0x100-0x100(%rcx),$xd0 + movdqa 0x110-0x100(%rcx),$xd1 + movdqa 0x120-0x100(%rcx),$xd2 + movdqa 0x130-0x100(%rcx),$xd3 + paddd .Lfour(%rip),$xd0 # next SIMD counters + +.Loop_enter4x: + movdqa $xt2,0x20(%rsp) # SIMD equivalent of "@x[10]" + movdqa $xt3,0x30(%rsp) # SIMD equivalent of "@x[11]" + movdqa (%r9),$xt3 # .Lrot16(%rip) + mov \$10,%eax + movdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4x + +.align 32 +.Loop4x: +___ + foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4x + + paddd 0x40(%rsp),$xa0 # accumulate key material + paddd 0x50(%rsp),$xa1 + paddd 0x60(%rsp),$xa2 + paddd 0x70(%rsp),$xa3 + + movdqa $xa0,$xt2 # "de-interlace" data + punpckldq $xa1,$xa0 + movdqa $xa2,$xt3 + punpckldq $xa3,$xa2 + punpckhdq $xa1,$xt2 + punpckhdq $xa3,$xt3 + movdqa $xa0,$xa1 + punpcklqdq $xa2,$xa0 # "a0" + movdqa $xt2,$xa3 + punpcklqdq $xt3,$xt2 # "a2" + punpckhqdq $xa2,$xa1 # "a1" + punpckhqdq $xt3,$xa3 # "a3" +___ + ($xa2,$xt2)=($xt2,$xa2); +$code.=<<___; + paddd 0x80-0x100(%rcx),$xb0 + paddd 0x90-0x100(%rcx),$xb1 + paddd 0xa0-0x100(%rcx),$xb2 + paddd 0xb0-0x100(%rcx),$xb3 + + movdqa $xa0,0x00(%rsp) # offload $xaN + movdqa $xa1,0x10(%rsp) + movdqa 0x20(%rsp),$xa0 # "xc2" + movdqa 0x30(%rsp),$xa1 # "xc3" + + movdqa $xb0,$xt2 + punpckldq $xb1,$xb0 + movdqa $xb2,$xt3 + punpckldq $xb3,$xb2 + punpckhdq $xb1,$xt2 + punpckhdq $xb3,$xt3 + movdqa $xb0,$xb1 + punpcklqdq $xb2,$xb0 # "b0" + movdqa $xt2,$xb3 + punpcklqdq $xt3,$xt2 # "b2" + punpckhqdq $xb2,$xb1 # "b1" + punpckhqdq $xt3,$xb3 # "b3" +___ + ($xb2,$xt2)=($xt2,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + paddd 0xc0-0x100(%rcx),$xc0 + paddd 0xd0-0x100(%rcx),$xc1 + paddd 0xe0-0x100(%rcx),$xc2 + paddd 0xf0-0x100(%rcx),$xc3 + + movdqa $xa2,0x20(%rsp) # keep offloading $xaN + movdqa $xa3,0x30(%rsp) + + movdqa $xc0,$xt2 + punpckldq $xc1,$xc0 + movdqa $xc2,$xt3 + punpckldq $xc3,$xc2 + punpckhdq $xc1,$xt2 + punpckhdq $xc3,$xt3 + movdqa $xc0,$xc1 + punpcklqdq $xc2,$xc0 # "c0" + movdqa $xt2,$xc3 + punpcklqdq $xt3,$xt2 # "c2" + punpckhqdq $xc2,$xc1 # "c1" + punpckhqdq $xt3,$xc3 # "c3" +___ + ($xc2,$xt2)=($xt2,$xc2); + ($xt0,$xt1)=($xa2,$xa3); # use $xaN as temporary +$code.=<<___; + paddd 0x100-0x100(%rcx),$xd0 + paddd 0x110-0x100(%rcx),$xd1 + paddd 0x120-0x100(%rcx),$xd2 + paddd 0x130-0x100(%rcx),$xd3 + + movdqa $xd0,$xt2 + punpckldq $xd1,$xd0 + movdqa $xd2,$xt3 + punpckldq $xd3,$xd2 + punpckhdq $xd1,$xt2 + punpckhdq $xd3,$xt3 + movdqa $xd0,$xd1 + punpcklqdq $xd2,$xd0 # "d0" + movdqa $xt2,$xd3 + punpcklqdq $xt3,$xt2 # "d2" + punpckhqdq $xd2,$xd1 # "d1" + punpckhqdq $xt3,$xd3 # "d3" +___ + ($xd2,$xt2)=($xt2,$xd2); +$code.=<<___; + cmp \$64*4,$len + jb .Ltail4x + + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # inp+=64*4 + pxor 0x30(%rsp),$xt0 + pxor $xb3,$xt1 + pxor $xc3,$xt2 + pxor $xd3,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4x + + jmp .Ldone4x + +.Ltail4x: + cmp \$192,$len + jae .L192_or_more4x + cmp \$128,$len + jae .L128_or_more4x + cmp \$64,$len + jae .L64_or_more4x + + #movdqa 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + xor %r9,%r9 + #movdqa $xt0,0x00(%rsp) + movdqa $xb0,0x10(%rsp) + movdqa $xc0,0x20(%rsp) + movdqa $xd0,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L64_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaxN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x10(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + movdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + movdqa $xd1,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L128_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + je .Ldone4x + + movdqa 0x20(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + movdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + movdqa $xd2,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L192_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x30(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*3 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + movdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + movdqa $xd3,0x30(%rsp) + +.Loop_tail4x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4x + +.Ldone4x: +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4x_epilogue: + ret +.cfi_endproc +.size chacha20_4x,.-chacha20_4x +___ +} +if($kernel) { + $code .= "#endif\n"; +} + +######################################################################## +# XOP code path that handles all lengths. +if ($avx && !$kernel) { +# There is some "anomaly" observed depending on instructions' size or +# alignment. If you look closely at below code you'll notice that +# sometimes argument order varies. The order affects instruction +# encoding by making it larger, and such fiddling gives 5% performance +# improvement. This is on FX-4100... + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3); + +sub XOP_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],16)", + "&vprotd (@x[$d1],@x[$d1],16)", + "&vprotd (@x[$d2],@x[$d2],16)", + "&vprotd (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],12)", + "&vprotd (@x[$b1],@x[$b1],12)", + "&vprotd (@x[$b2],@x[$b2],12)", + "&vprotd (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$b0],@x[$a0])", # flip + "&vpaddd (@x[$a1],@x[$b1],@x[$a1])", # flip + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],8)", + "&vprotd (@x[$d1],@x[$d1],8)", + "&vprotd (@x[$d2],@x[$d2],8)", + "&vprotd (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],7)", + "&vprotd (@x[$b1],@x[$b1],7)", + "&vprotd (@x[$b2],@x[$b2],7)", + "&vprotd (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_xop", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_4xop: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4xop_body: +___ +$code.=<<___; + vzeroupper + + vmovdqa .Lsigma(%rip),$xa3 # key[0] + vmovdqu ($key),$xb3 # key[1] + vmovdqu 16($key),$xt3 # key[2] + vmovdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x40(%rsp) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0x50(%rsp) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0x60(%rsp) + vmovdqa $xa3,0x70(%rsp) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x80-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x90-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0xa0-0x100(%rcx) + vmovdqa $xb3,0xb0-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "$xc0" + vpshufd \$0x55,$xt3,$xt1 # "$xc1" + vmovdqa $xt0,0xc0-0x100(%rcx) + vpshufd \$0xaa,$xt3,$xt2 # "$xc2" + vmovdqa $xt1,0xd0-0x100(%rcx) + vpshufd \$0xff,$xt3,$xt3 # "$xc3" + vmovdqa $xt2,0xe0-0x100(%rcx) + vmovdqa $xt3,0xf0-0x100(%rcx) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Linc(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x110-0x100(%rcx) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x120-0x100(%rcx) + vmovdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4xop + +.align 32 +.Loop_outer4xop: + vmovdqa 0x40(%rsp),$xa0 # re-load smashed key + vmovdqa 0x50(%rsp),$xa1 + vmovdqa 0x60(%rsp),$xa2 + vmovdqa 0x70(%rsp),$xa3 + vmovdqa 0x80-0x100(%rcx),$xb0 + vmovdqa 0x90-0x100(%rcx),$xb1 + vmovdqa 0xa0-0x100(%rcx),$xb2 + vmovdqa 0xb0-0x100(%rcx),$xb3 + vmovdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + vmovdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + vmovdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + vmovdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + vmovdqa 0x100-0x100(%rcx),$xd0 + vmovdqa 0x110-0x100(%rcx),$xd1 + vmovdqa 0x120-0x100(%rcx),$xd2 + vmovdqa 0x130-0x100(%rcx),$xd3 + vpaddd .Lfour(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter4xop: + mov \$10,%eax + vmovdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4xop + +.align 32 +.Loop4xop: +___ + foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4xop + + vpaddd 0x40(%rsp),$xa0,$xa0 # accumulate key material + vpaddd 0x50(%rsp),$xa1,$xa1 + vpaddd 0x60(%rsp),$xa2,$xa2 + vpaddd 0x70(%rsp),$xa3,$xa3 + + vmovdqa $xt2,0x20(%rsp) # offload $xc2,3 + vmovdqa $xt3,0x30(%rsp) + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x80-0x100(%rcx),$xb0,$xb0 + vpaddd 0x90-0x100(%rcx),$xb1,$xb1 + vpaddd 0xa0-0x100(%rcx),$xb2,$xb2 + vpaddd 0xb0-0x100(%rcx),$xb3,$xb3 + + vmovdqa $xa0,0x00(%rsp) # offload $xa0,1 + vmovdqa $xa1,0x10(%rsp) + vmovdqa 0x20(%rsp),$xa0 # "xc2" + vmovdqa 0x30(%rsp),$xa1 # "xc3" + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vpaddd 0xc0-0x100(%rcx),$xc0,$xc0 + vpaddd 0xd0-0x100(%rcx),$xc1,$xc1 + vpaddd 0xe0-0x100(%rcx),$xc2,$xc2 + vpaddd 0xf0-0x100(%rcx),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xd0,$xd0 + vpaddd 0x110-0x100(%rcx),$xd1,$xd1 + vpaddd 0x120-0x100(%rcx),$xd2,$xd2 + vpaddd 0x130-0x100(%rcx),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # restore $xa0,1 + vmovdqa 0x10(%rsp),$xa1 + + cmp \$64*4,$len + jb .Ltail4xop + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + vpxor 0x40($inp),$xa3,$xa3 + vpxor 0x50($inp),$xb3,$xb3 + vpxor 0x60($inp),$xc3,$xc3 + vpxor 0x70($inp),$xd3,$xd3 + lea 0x80($inp),$inp # inp+=64*4 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + vmovdqu $xa3,0x40($out) + vmovdqu $xb3,0x50($out) + vmovdqu $xc3,0x60($out) + vmovdqu $xd3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4xop + + jmp .Ldone4xop + +.align 32 +.Ltail4xop: + cmp \$192,$len + jae .L192_or_more4xop + cmp \$128,$len + jae .L128_or_more4xop + cmp \$64,$len + jae .L64_or_more4xop + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x10(%rsp) + vmovdqa $xc0,0x20(%rsp) + vmovdqa $xd0,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L64_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*1 + vmovdqa $xa1,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + vmovdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + vmovdqa $xd1,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L128_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + je .Ldone4xop + + lea 0x80($inp),$inp # inp+=64*2 + vmovdqa $xa2,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + vmovdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + vmovdqa $xd2,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L192_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*3 + vmovdqa $xa3,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + vmovdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + vmovdqa $xd3,0x30(%rsp) + +.Loop_tail4xop: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4xop + +.Ldone4xop: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4xop_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_xop"); +} + +######################################################################## +# AVX2 code path +if ($avx>1) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub AVX2_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t1)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t1)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t0,@x[$b0],12)", + "&vpsrld (@x[$b0],@x[$b0],20)", + "&vpor (@x[$b0],$t0,@x[$b0])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t1,@x[$b1],12)", + "&vpsrld (@x[$b1],@x[$b1],20)", + "&vpor (@x[$b1],$t1,@x[$b1])", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t0)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t0)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t1,@x[$b0],7)", + "&vpsrld (@x[$b0],@x[$b0],25)", + "&vpor (@x[$b0],$t1,@x[$b0])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t0,@x[$b1],7)", + "&vpsrld (@x[$b1],@x[$b1],25)", + "&vpor (@x[$b1],$t0,@x[$b1])", + + "&vmovdqa (\"`32*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&vmovdqa (\"`32*($c1-8)`(%rsp)\",$xc_)", + "&vmovdqa ($xc,\"`32*($c2-8)`(%rsp)\")", + "&vmovdqa ($xc_,\"`32*($c3-8)`(%rsp)\")", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t1)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t1)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t0,@x[$b2],12)", + "&vpsrld (@x[$b2],@x[$b2],20)", + "&vpor (@x[$b2],$t0,@x[$b2])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t1,@x[$b3],12)", + "&vpsrld (@x[$b3],@x[$b3],20)", + "&vpor (@x[$b3],$t1,@x[$b3])", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t0)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t0)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t1,@x[$b2],7)", + "&vpsrld (@x[$b2],@x[$b2],25)", + "&vpor (@x[$b2],$t1,@x[$b2])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t0,@x[$b3],7)", + "&vpsrld (@x[$b3],@x[$b3],25)", + "&vpor (@x[$b3],$t0,@x[$b3])" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_avx2", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_8x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$0x280+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8x_body: +___ +$code.=<<___; + vzeroupper + + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x80 constant copy of key[0-2] smashed by lanes + # ... + # +0x200 SIMD counters (with nonce smashed by lanes) + # ... + # +0x280 + + vbroadcasti128 .Lsigma(%rip),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xt3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea 0x200(%rsp),%rax # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x80-0x100(%rcx) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0xa0-0x100(%rcx) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0xc0-0x100(%rcx) + vmovdqa $xa3,0xe0-0x100(%rcx) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x100-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x120-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0x140-0x100(%rcx) + vmovdqa $xb3,0x160-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "xc0" + vpshufd \$0x55,$xt3,$xt1 # "xc1" + vmovdqa $xt0,0x180-0x200(%rax) + vpshufd \$0xaa,$xt3,$xt2 # "xc2" + vmovdqa $xt1,0x1a0-0x200(%rax) + vpshufd \$0xff,$xt3,$xt3 # "xc3" + vmovdqa $xt2,0x1c0-0x200(%rax) + vmovdqa $xt3,0x1e0-0x200(%rax) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x220-0x200(%rax) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x240-0x200(%rax) + vmovdqa $xd3,0x260-0x200(%rax) + + jmp .Loop_enter8x + +.align 32 +.Loop_outer8x: + vmovdqa 0x80-0x100(%rcx),$xa0 # re-load smashed key + vmovdqa 0xa0-0x100(%rcx),$xa1 + vmovdqa 0xc0-0x100(%rcx),$xa2 + vmovdqa 0xe0-0x100(%rcx),$xa3 + vmovdqa 0x100-0x100(%rcx),$xb0 + vmovdqa 0x120-0x100(%rcx),$xb1 + vmovdqa 0x140-0x100(%rcx),$xb2 + vmovdqa 0x160-0x100(%rcx),$xb3 + vmovdqa 0x180-0x200(%rax),$xt0 # "xc0" + vmovdqa 0x1a0-0x200(%rax),$xt1 # "xc1" + vmovdqa 0x1c0-0x200(%rax),$xt2 # "xc2" + vmovdqa 0x1e0-0x200(%rax),$xt3 # "xc3" + vmovdqa 0x200-0x200(%rax),$xd0 + vmovdqa 0x220-0x200(%rax),$xd1 + vmovdqa 0x240-0x200(%rax),$xd2 + vmovdqa 0x260-0x200(%rax),$xd3 + vpaddd .Leight(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter8x: + vmovdqa $xt2,0x40(%rsp) # SIMD equivalent of "@x[10]" + vmovdqa $xt3,0x60(%rsp) # SIMD equivalent of "@x[11]" + vbroadcasti128 (%r9),$xt3 + vmovdqa $xd0,0x200-0x200(%rax) # save SIMD counters + mov \$10,%eax + jmp .Loop8x + +.align 32 +.Loop8x: +___ + foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8x + + lea 0x200(%rsp),%rax # size optimization + vpaddd 0x80-0x100(%rcx),$xa0,$xa0 # accumulate key + vpaddd 0xa0-0x100(%rcx),$xa1,$xa1 + vpaddd 0xc0-0x100(%rcx),$xa2,$xa2 + vpaddd 0xe0-0x100(%rcx),$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xb0,$xb0 + vpaddd 0x120-0x100(%rcx),$xb1,$xb1 + vpaddd 0x140-0x100(%rcx),$xb2,$xb2 + vpaddd 0x160-0x100(%rcx),$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vperm2i128 \$0x20,$xb0,$xa0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xb0,$xa0,$xb0 + vperm2i128 \$0x20,$xb1,$xa1,$xa0 + vperm2i128 \$0x31,$xb1,$xa1,$xb1 + vperm2i128 \$0x20,$xb2,$xa2,$xa1 + vperm2i128 \$0x31,$xb2,$xa2,$xb2 + vperm2i128 \$0x20,$xb3,$xa3,$xa2 + vperm2i128 \$0x31,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vmovdqa $xa0,0x00(%rsp) # offload $xaN + vmovdqa $xa1,0x20(%rsp) + vmovdqa 0x40(%rsp),$xc2 # $xa0 + vmovdqa 0x60(%rsp),$xc3 # $xa1 + + vpaddd 0x180-0x200(%rax),$xc0,$xc0 + vpaddd 0x1a0-0x200(%rax),$xc1,$xc1 + vpaddd 0x1c0-0x200(%rax),$xc2,$xc2 + vpaddd 0x1e0-0x200(%rax),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x200-0x200(%rax),$xd0,$xd0 + vpaddd 0x220-0x200(%rax),$xd1,$xd1 + vpaddd 0x240-0x200(%rax),$xd2,$xd2 + vpaddd 0x260-0x200(%rax),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # $xaN was offloaded, remember? + vmovdqa 0x20(%rsp),$xa1 + + cmp \$64*8,$len + jb .Ltail8x + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea 0x80($out),$out # size optimization + + sub \$64*8,$len + jnz .Loop_outer8x + + jmp .Ldone8x + +.Ltail8x: + cmp \$448,$len + jae .L448_or_more8x + cmp \$384,$len + jae .L384_or_more8x + cmp \$320,$len + jae .L320_or_more8x + cmp \$256,$len + jae .L256_or_more8x + cmp \$192,$len + jae .L192_or_more8x + cmp \$128,$len + jae .L128_or_more8x + cmp \$64,$len + jae .L64_or_more8x + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L64_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + je .Ldone8x + + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + vmovdqa $xc0,0x00(%rsp) + lea 0x40($out),$out # out+=64*1 + sub \$64,$len # len-=64*1 + vmovdqa $xd0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L128_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + je .Ldone8x + + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + vmovdqa $xa1,0x00(%rsp) + lea 0x80($out),$out # out+=64*2 + sub \$128,$len # len-=64*2 + vmovdqa $xb1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L192_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + je .Ldone8x + + lea 0xc0($inp),$inp # inp+=64*3 + xor %r9,%r9 + vmovdqa $xc1,0x00(%rsp) + lea 0xc0($out),$out # out+=64*3 + sub \$192,$len # len-=64*3 + vmovdqa $xd1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L256_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + je .Ldone8x + + lea 0x100($inp),$inp # inp+=64*4 + xor %r9,%r9 + vmovdqa $xa2,0x00(%rsp) + lea 0x100($out),$out # out+=64*4 + sub \$256,$len # len-=64*4 + vmovdqa $xb2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L320_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + je .Ldone8x + + lea 0x140($inp),$inp # inp+=64*5 + xor %r9,%r9 + vmovdqa $xc2,0x00(%rsp) + lea 0x140($out),$out # out+=64*5 + sub \$320,$len # len-=64*5 + vmovdqa $xd2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L384_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + je .Ldone8x + + lea 0x180($inp),$inp # inp+=64*6 + xor %r9,%r9 + vmovdqa $xa3,0x00(%rsp) + lea 0x180($out),$out # out+=64*6 + sub \$384,$len # len-=64*6 + vmovdqa $xb3,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L448_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vpxor 0x180($inp),$xa3,$xa3 + vpxor 0x1a0($inp),$xb3,$xb3 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + vmovdqu $xa3,0x180($out) + vmovdqu $xb3,0x1a0($out) + je .Ldone8x + + lea 0x1c0($inp),$inp # inp+=64*7 + xor %r9,%r9 + vmovdqa $xc3,0x00(%rsp) + lea 0x1c0($out),$out # out+=64*7 + sub \$448,$len # len-=64*7 + vmovdqa $xd3,0x20(%rsp) + +.Loop_tail8x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8x + +.Ldone8x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8x_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx2"); +if($kernel) { + $code .= "#endif\n"; +} +} + +######################################################################## +# AVX512 code paths +if ($avx>2) { +# This one handles shorter inputs... +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20)); +my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +sub vpxord() # size optimization +{ my $opcode = "vpxor"; # adhere to vpxor when possible + + foreach (@_) { + if (/%([zy])mm([0-9]+)/ && ($1 eq "z" || $2>=16)) { + $opcode = "vpxord"; + last; + } + } + + $code .= "\t$opcode\t".join(',',reverse @_)."\n"; +} + +sub AVX512ROUND { # critical path is 14 "SIMD ticks" per round + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,16); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,12); + + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,8); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,7); +} + +my $xframe = $win64 ? 32+8 : 8; + +&declare_function("chacha20_avx512", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$512,$len + ja .Lchacha20_16x + + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512_body: +___ +$code.=<<___; + vbroadcasti32x4 .Lsigma(%rip),$a + vbroadcasti32x4 ($key),$b + vbroadcasti32x4 16($key),$c + vbroadcasti32x4 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Lfourz(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 16 +.Loop_outer_avx512: + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512 + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$1,$a,$t0 + vextracti32x4 \$1,$b,$t1 + vextracti32x4 \$1,$c,$t2 + vextracti32x4 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$2,$a,$t0 + vextracti32x4 \$2,$b,$t1 + vextracti32x4 \$2,$c,$t2 + vextracti32x4 \$2,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$3,$a,$t0 + vextracti32x4 \$3,$b,$t1 + vextracti32x4 \$3,$c,$t2 + vextracti32x4 \$3,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jnz .Loop_outer_avx512 + + jmp .Ldone_avx512 + +.align 16 +.Ltail64_avx512: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512 + +.align 16 +.Ltail_avx512: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512 + + vmovdqu32 $a_,0x00(%rsp) + +.Ldone_avx512: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512"); + +map(s/%z/%y/, $a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz); + +&declare_function("chacha20_avx512vl", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512vl: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$128,$len + ja .Lchacha20_8xvl + + sub \$64+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512vl_body: +___ +$code.=<<___; + vbroadcasti128 .Lsigma(%rip),$a + vbroadcasti128 ($key),$b + vbroadcasti128 16($key),$c + vbroadcasti128 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Ltwoy(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 16 +.Loop_outer_avx512vl: + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 32 +.Loop_avx512vl: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512vl"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512vl + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512vl + + vextracti128 \$1,$a,$t0 + vextracti128 \$1,$b,$t1 + vextracti128 \$1,$c,$t2 + vextracti128 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512vl + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + jnz .Loop_outer_avx512vl + + jmp .Ldone_avx512vl + +.align 16 +.Ltail64_avx512vl: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512vl + +.align 16 +.Ltail_avx512vl: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512vl: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512vl + + vmovdqu32 $a_,0x00(%rsp) + vmovdqu32 $a_,0x20(%rsp) + +.Ldone_avx512vl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512vl_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512vl"); + +# This one handles longer inputs... + +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%zmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +my @key=map("%zmm$_",(16..31)); +my ($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +sub AVX512_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],16)", + "&vprold (@x[$d1],@x[$d1],16)", + "&vprold (@x[$d2],@x[$d2],16)", + "&vprold (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],12)", + "&vprold (@x[$b1],@x[$b1],12)", + "&vprold (@x[$b2],@x[$b2],12)", + "&vprold (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],8)", + "&vprold (@x[$d1],@x[$d1],8)", + "&vprold (@x[$d2],@x[$d2],8)", + "&vprold (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],7)", + "&vprold (@x[$b1],@x[$b1],7)", + "&vprold (@x[$b2],@x[$b2],7)", + "&vprold (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_16x,\@function,5 +.align 32 +chacha20_16x: +.cfi_startproc +.Lchacha20_16x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L16x_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti32x4 (%r9),$xa3 # key[0] + vbroadcasti32x4 ($key),$xb3 # key[1] + vbroadcasti32x4 16($key),$xc3 # key[2] + vbroadcasti32x4 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincz(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop_outer16x: + vpbroadcastd 0(%r9),$xa0 # reload key + vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Lsixteen(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop16x: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop16x + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0x44,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xb0,$xa0,$xb0 + vshufi32x4 \$0x44,$xb1,$xa1,$xa0 + vshufi32x4 \$0xee,$xb1,$xa1,$xb1 + vshufi32x4 \$0x44,$xb2,$xa2,$xa1 + vshufi32x4 \$0xee,$xb2,$xa2,$xb2 + vshufi32x4 \$0x44,$xb3,$xa3,$xa2 + vshufi32x4 \$0xee,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vshufi32x4 \$0x44,$xd0,$xc0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xd0,$xc0,$xd0 + vshufi32x4 \$0x44,$xd1,$xc1,$xc0 + vshufi32x4 \$0xee,$xd1,$xc1,$xd1 + vshufi32x4 \$0x44,$xd2,$xc2,$xc1 + vshufi32x4 \$0xee,$xd2,$xc2,$xd2 + vshufi32x4 \$0x44,$xd3,$xc3,$xc2 + vshufi32x4 \$0xee,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); +$code.=<<___; + vshufi32x4 \$0x88,$xc0,$xa0,$xt0 # "de-interlace" further + vshufi32x4 \$0xdd,$xc0,$xa0,$xa0 + vshufi32x4 \$0x88,$xd0,$xb0,$xc0 + vshufi32x4 \$0xdd,$xd0,$xb0,$xd0 + vshufi32x4 \$0x88,$xc1,$xa1,$xt1 + vshufi32x4 \$0xdd,$xc1,$xa1,$xa1 + vshufi32x4 \$0x88,$xd1,$xb1,$xc1 + vshufi32x4 \$0xdd,$xd1,$xb1,$xd1 + vshufi32x4 \$0x88,$xc2,$xa2,$xt2 + vshufi32x4 \$0xdd,$xc2,$xa2,$xa2 + vshufi32x4 \$0x88,$xd2,$xb2,$xc2 + vshufi32x4 \$0xdd,$xd2,$xb2,$xd2 + vshufi32x4 \$0x88,$xc3,$xa3,$xt3 + vshufi32x4 \$0xdd,$xc3,$xa3,$xa3 + vshufi32x4 \$0x88,$xd3,$xb3,$xc3 + vshufi32x4 \$0xdd,$xd3,$xb3,$xd3 +___ + ($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3)= + ($xt0,$xt1,$xt2,$xt3,$xa0,$xa1,$xa2,$xa3); + + ($xa0,$xb0,$xc0,$xd0, $xa1,$xb1,$xc1,$xd1, + $xa2,$xb2,$xc2,$xd2, $xa3,$xb3,$xc3,$xd3) = + ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +$code.=<<___; + cmp \$64*16,$len + jb .Ltail16x + + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxord 0x40($inp),$xb0,$xb0 + vpxord 0x80($inp),$xc0,$xc0 + vpxord 0xc0($inp),$xd0,$xd0 + vmovdqu32 $xa0,0x00($out) + vmovdqu32 $xb0,0x40($out) + vmovdqu32 $xc0,0x80($out) + vmovdqu32 $xd0,0xc0($out) + + vpxord 0x100($inp),$xa1,$xa1 + vpxord 0x140($inp),$xb1,$xb1 + vpxord 0x180($inp),$xc1,$xc1 + vpxord 0x1c0($inp),$xd1,$xd1 + vmovdqu32 $xa1,0x100($out) + vmovdqu32 $xb1,0x140($out) + vmovdqu32 $xc1,0x180($out) + vmovdqu32 $xd1,0x1c0($out) + + vpxord 0x200($inp),$xa2,$xa2 + vpxord 0x240($inp),$xb2,$xb2 + vpxord 0x280($inp),$xc2,$xc2 + vpxord 0x2c0($inp),$xd2,$xd2 + vmovdqu32 $xa2,0x200($out) + vmovdqu32 $xb2,0x240($out) + vmovdqu32 $xc2,0x280($out) + vmovdqu32 $xd2,0x2c0($out) + + vpxord 0x300($inp),$xa3,$xa3 + vpxord 0x340($inp),$xb3,$xb3 + vpxord 0x380($inp),$xc3,$xc3 + vpxord 0x3c0($inp),$xd3,$xd3 + lea 0x400($inp),$inp + vmovdqu32 $xa3,0x300($out) + vmovdqu32 $xb3,0x340($out) + vmovdqu32 $xc3,0x380($out) + vmovdqu32 $xd3,0x3c0($out) + lea 0x400($out),$out + + sub \$64*16,$len + jnz .Loop_outer16x + + jmp .Ldone16x + +.align 32 +.Ltail16x: + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_16x + vpxord ($inp),$xa0,$xa0 # xor with input + vmovdqu32 $xa0,($out,$inp) + je .Ldone16x + vmovdqa32 $xb0,$xa0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_16x + vpxord ($inp),$xb0,$xb0 + vmovdqu32 $xb0,($out,$inp) + je .Ldone16x + vmovdqa32 $xc0,$xa0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_16x + vpxord ($inp),$xc0,$xc0 + vmovdqu32 $xc0,($out,$inp) + je .Ldone16x + vmovdqa32 $xd0,$xa0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_16x + vpxord ($inp),$xd0,$xd0 + vmovdqu32 $xd0,($out,$inp) + je .Ldone16x + vmovdqa32 $xa1,$xa0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_16x + vpxord ($inp),$xa1,$xa1 + vmovdqu32 $xa1,($out,$inp) + je .Ldone16x + vmovdqa32 $xb1,$xa0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_16x + vpxord ($inp),$xb1,$xb1 + vmovdqu32 $xb1,($out,$inp) + je .Ldone16x + vmovdqa32 $xc1,$xa0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_16x + vpxord ($inp),$xc1,$xc1 + vmovdqu32 $xc1,($out,$inp) + je .Ldone16x + vmovdqa32 $xd1,$xa0 + lea 64($inp),$inp + + cmp \$64*8,$len + jb .Less_than_64_16x + vpxord ($inp),$xd1,$xd1 + vmovdqu32 $xd1,($out,$inp) + je .Ldone16x + vmovdqa32 $xa2,$xa0 + lea 64($inp),$inp + + cmp \$64*9,$len + jb .Less_than_64_16x + vpxord ($inp),$xa2,$xa2 + vmovdqu32 $xa2,($out,$inp) + je .Ldone16x + vmovdqa32 $xb2,$xa0 + lea 64($inp),$inp + + cmp \$64*10,$len + jb .Less_than_64_16x + vpxord ($inp),$xb2,$xb2 + vmovdqu32 $xb2,($out,$inp) + je .Ldone16x + vmovdqa32 $xc2,$xa0 + lea 64($inp),$inp + + cmp \$64*11,$len + jb .Less_than_64_16x + vpxord ($inp),$xc2,$xc2 + vmovdqu32 $xc2,($out,$inp) + je .Ldone16x + vmovdqa32 $xd2,$xa0 + lea 64($inp),$inp + + cmp \$64*12,$len + jb .Less_than_64_16x + vpxord ($inp),$xd2,$xd2 + vmovdqu32 $xd2,($out,$inp) + je .Ldone16x + vmovdqa32 $xa3,$xa0 + lea 64($inp),$inp + + cmp \$64*13,$len + jb .Less_than_64_16x + vpxord ($inp),$xa3,$xa3 + vmovdqu32 $xa3,($out,$inp) + je .Ldone16x + vmovdqa32 $xb3,$xa0 + lea 64($inp),$inp + + cmp \$64*14,$len + jb .Less_than_64_16x + vpxord ($inp),$xb3,$xb3 + vmovdqu32 $xb3,($out,$inp) + je .Ldone16x + vmovdqa32 $xc3,$xa0 + lea 64($inp),$inp + + cmp \$64*15,$len + jb .Less_than_64_16x + vpxord ($inp),$xc3,$xc3 + vmovdqu32 $xc3,($out,$inp) + je .Ldone16x + vmovdqa32 $xd3,$xa0 + lea 64($inp),$inp + +.Less_than_64_16x: + vmovdqa32 $xa0,0x00(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail16x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail16x + + vpxord $xa0,$xa0,$xa0 + vmovdqa32 $xa0,0(%rsp) + +.Ldone16x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L16x_epilogue: + ret +.cfi_endproc +.size chacha20_16x,.-chacha20_16x +___ + +# switch to %ymm domain +($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%ymm$_",(0..15)); +@xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +@key=map("%ymm$_",(16..31)); +($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +$code.=<<___; +.type chacha20_8xvl,\@function,5 +.align 32 +chacha20_8xvl: +.cfi_startproc +.Lchacha20_8xvl: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8xvl_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti128 (%r9),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xc3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop_outer8xvl: + #vpbroadcastd 0(%r9),$xa0 # reload key + #vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Leight(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop8xvl: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8xvl + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$3,$xb0,$xa0,$xb0 + vshufi32x4 \$0,$xb1,$xa1,$xa0 + vshufi32x4 \$3,$xb1,$xa1,$xb1 + vshufi32x4 \$0,$xb2,$xa2,$xa1 + vshufi32x4 \$3,$xb2,$xa2,$xb2 + vshufi32x4 \$0,$xb3,$xa3,$xa2 + vshufi32x4 \$3,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); +$code.=<<___; + cmp \$64*8,$len + jb .Ltail8xvl + + mov \$0x80,%eax # size optimization + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea ($out,%rax),$out # size optimization + + vpbroadcastd 0(%r9),%ymm0 # reload key + vpbroadcastd 4(%r9),%ymm1 + + sub \$64*8,$len + jnz .Loop_outer8xvl + + jmp .Ldone8xvl + +.align 32 +.Ltail8xvl: + vmovdqa64 $xa0,%ymm8 # size optimization +___ +$xa0 = "%ymm8"; +$code.=<<___; + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out,$inp) + vmovdqu $xb0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc0,$xa0 + vmovdqa $xd0,$xb0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc0,$xc0 + vpxor 0x20($inp),$xd0,$xd0 + vmovdqu $xc0,0x00($out,$inp) + vmovdqu $xd0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa1,$xa0 + vmovdqa $xb1,$xb0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vmovdqu $xa1,0x00($out,$inp) + vmovdqu $xb1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc1,$xa0 + vmovdqa $xd1,$xb0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc1,$xc1 + vpxor 0x20($inp),$xd1,$xd1 + vmovdqu $xc1,0x00($out,$inp) + vmovdqu $xd1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa32 $xa2,$xa0 + vmovdqa $xb2,$xb0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_8xvl + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vmovdqu32 $xa2,0x00($out,$inp) + vmovdqu $xb2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc2,$xa0 + vmovdqa $xd2,$xb0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc2,$xc2 + vpxor 0x20($inp),$xd2,$xd2 + vmovdqu $xc2,0x00($out,$inp) + vmovdqu $xd2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa3,$xa0 + vmovdqa $xb3,$xb0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vmovdqu $xa3,0x00($out,$inp) + vmovdqu $xb3,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc3,$xa0 + vmovdqa $xd3,$xb0 + lea 64($inp),$inp + +.Less_than_64_8xvl: + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail8xvl: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8xvl + + vpxor $xa0,$xa0,$xa0 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xa0,0x20(%rsp) + +.Ldone8xvl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8xvl_epilogue: + ret +.cfi_endproc +.size chacha20_8xvl,.-chacha20_8xvl +___ +if($kernel) { + $code .= "#endif\n"; +} +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + lea .Lctr32_body(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea .Lno_data(%rip),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 64+24+48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.type simd_handler,\@abi-omnipotent +.align 16 +simd_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipR9 + + mov 4(%r11),%r10d # HandlerData[1] + mov 8(%r11),%ecx # HandlerData[2] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + neg %rcx + lea -8(%rax,%rcx),%rsi + lea 512($context),%rdi # &context.Xmm6 + neg %ecx + shr \$3,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lcommon_seh_tail +.size simd_handler,.-simd_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_chacha20_ctr32 + .rva .LSEH_end_chacha20_ctr32 + .rva .LSEH_info_chacha20_ctr32 + + .rva .LSEH_begin_chacha20_ssse3 + .rva .LSEH_end_chacha20_ssse3 + .rva .LSEH_info_chacha20_ssse3 + + .rva .LSEH_begin_chacha20_128 + .rva .LSEH_end_chacha20_128 + .rva .LSEH_info_chacha20_128 + + .rva .LSEH_begin_chacha20_4x + .rva .LSEH_end_chacha20_4x + .rva .LSEH_info_chacha20_4x +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_chacha20_xop + .rva .LSEH_end_chacha20_xop + .rva .LSEH_info_chacha20_xop +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_chacha20_avx2 + .rva .LSEH_end_chacha20_avx2 + .rva .LSEH_info_chacha20_avx2 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_chacha20_avx512 + .rva .LSEH_end_chacha20_avx512 + .rva .LSEH_info_chacha20_avx512 + + .rva .LSEH_begin_chacha20_avx512vl + .rva .LSEH_end_chacha20_avx512vl + .rva .LSEH_info_chacha20_avx512vl + + .rva .LSEH_begin_chacha20_16x + .rva .LSEH_end_chacha20_16x + .rva .LSEH_info_chacha20_16x + + .rva .LSEH_begin_chacha20_8xvl + .rva .LSEH_end_chacha20_8xvl + .rva .LSEH_info_chacha20_8xvl +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_chacha20_ctr32: + .byte 9,0,0,0 + .rva se_handler + +.LSEH_info_chacha20_ssse3: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lssse3_body,.Lssse3_epilogue + .long 0x20,0 + +.LSEH_info_chacha20_128: + .byte 9,0,0,0 + .rva simd_handler + .rva .L128_body,.L128_epilogue + .long 0x60,0 + +.LSEH_info_chacha20_4x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4x_body,.L4x_epilogue + .long 0xa0,0 +___ +$code.=<<___ if ($avx); +.LSEH_info_chacha20_xop: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4xop_body,.L4xop_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>1); +.LSEH_info_chacha20_avx2: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8x_body,.L8x_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>2); +.LSEH_info_chacha20_avx512: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512_body,.Lavx512_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_avx512vl: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512vl_body,.Lavx512vl_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_16x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L16x_body,.L16x_epilogue # HandlerData[] + .long 0xa0,0 + +.LSEH_info_chacha20_8xvl: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8xvl_body,.L8xvl_epilogue # HandlerData[] + .long 0xa0,0 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/%x#%[yz]/%x/g; # "down-shift" + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} + +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20.c b/net/wireguard/crypto/zinc/chacha20/chacha20.c new file mode 100644 index 0000000000000..b4763c81e0826 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the ChaCha20 stream cipher. + * + * Information: https://cr.yp.to/chacha.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include // For crypto_xor_cpy. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "chacha20-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "chacha20-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) +#include "chacha20-mips-glue.c" +#else +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + return false; +} +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} +#endif + +#define QUARTER_ROUND(x, a, b, c, d) ( \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 16), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 12), \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 8), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 7) \ +) + +#define C(i, j) (i * 4 + j) + +#define DOUBLE_ROUND(x) ( \ + /* Column Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 0), C(2, 0), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 1), C(2, 1), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 2), C(2, 2), C(3, 2)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 3), C(2, 3), C(3, 3)), \ + /* Diagonal Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 1), C(2, 2), C(3, 3)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 2), C(2, 3), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 3), C(2, 0), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 0), C(2, 1), C(3, 2)) \ +) + +#define TWENTY_ROUNDS(x) ( \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x) \ +) + +static void chacha20_block_generic(struct chacha20_ctx *ctx, __le32 *stream) +{ + u32 x[CHACHA20_BLOCK_WORDS]; + int i; + + for (i = 0; i < ARRAY_SIZE(x); ++i) + x[i] = ctx->state[i]; + + TWENTY_ROUNDS(x); + + for (i = 0; i < ARRAY_SIZE(x); ++i) + stream[i] = cpu_to_le32(x[i] + ctx->state[i]); + + ctx->counter[0] += 1; +} + +static void chacha20_generic(struct chacha20_ctx *ctx, u8 *out, const u8 *in, + u32 len) +{ + __le32 buf[CHACHA20_BLOCK_WORDS]; + + while (len >= CHACHA20_BLOCK_SIZE) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, CHACHA20_BLOCK_SIZE); + len -= CHACHA20_BLOCK_SIZE; + out += CHACHA20_BLOCK_SIZE; + in += CHACHA20_BLOCK_SIZE; + } + if (len) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, len); + } +} + +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context) +{ + if (!chacha20_arch(ctx, dst, src, len, simd_context)) + chacha20_generic(ctx, dst, src, len); +} +EXPORT_SYMBOL(chacha20); + +static void hchacha20_generic(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE]) +{ + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + + TWENTY_ROUNDS(x); + + memcpy(derived_key + 0, x + 0, sizeof(u32) * 4); + memcpy(derived_key + 4, x + 12, sizeof(u32) * 4); +} + +/* Derived key should be 32-bit aligned */ +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context) +{ + if (!hchacha20_arch(derived_key, nonce, key, simd_context)) + hchacha20_generic(derived_key, nonce, key); +} +EXPORT_SYMBOL(hchacha20); + +#include "../selftest/chacha20.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + chacha20_fpu_init(); + if (!selftest_run("chacha20", chacha20_selftest, chacha20_nobs, + ARRAY_SIZE(chacha20_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20poly1305.c b/net/wireguard/crypto/zinc/chacha20poly1305.c new file mode 100644 index 0000000000000..571a64e95a83a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20poly1305.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the ChaCha20Poly1305 AEAD construction. + * + * Information: https://tools.ietf.org/html/rfc8439 + */ + +#include +#include +#include +#include "selftest/run.h" + +#include +#include +#include +#include +#include // For blkcipher_walk. + +static const u8 pad0[CHACHA20_BLOCK_SIZE] = { 0 }; + +static inline void +__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + chacha20(&chacha20_state, dst, src, src_len, simd_context); + + poly1305_update(&poly1305_state, dst, src_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, dst + src_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + + simd_get(&simd_context); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key, + &simd_context); + simd_put(&simd_context); +} +EXPORT_SYMBOL(chacha20poly1305_encrypt); + +bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, + const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + poly1305_update(&poly1305_state, miter.addr, + min_t(size_t, sl, miter.length), simd_context); + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) + poly1305_final(&poly1305_state, miter.addr + miter.length + sl, + simd_context); + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.mac, simd_context); + scatterwalk_map_and_copy(b.mac, src, src_len, sizeof(b.mac), 1); + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return true; +} +EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); + +static inline bool +__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + int ret; + size_t dst_len; + union { + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return false; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + dst_len = src_len - POLY1305_MAC_SIZE; + poly1305_update(&poly1305_state, src, dst_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - dst_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(dst_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, b.mac, simd_context); + + ret = crypto_memneq(b.mac, src + dst_len, POLY1305_MAC_SIZE); + if (likely(!ret)) + chacha20(&chacha20_state, dst, src, dst_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + + return !ret; +} + +bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context, ret; + + simd_get(&simd_context); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce, + key, &simd_context); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt); + +bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, + size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + struct { + u8 read_mac[POLY1305_MAC_SIZE]; + u8 computed_mac[POLY1305_MAC_SIZE]; + }; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + bool ret = false; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return ret; + src_len -= POLY1305_MAC_SIZE; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + poly1305_update(&poly1305_state, addr, length, simd_context); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + ret = !crypto_memneq(b.computed_mac, + miter.addr + miter.length + sl, + POLY1305_MAC_SIZE); + } + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + scatterwalk_map_and_copy(b.read_mac, src, src_len, + sizeof(b.read_mac), 0); + ret = !crypto_memneq(b.read_mac, b.computed_mac, + POLY1305_MAC_SIZE); + + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); +} +EXPORT_SYMBOL(xchacha20poly1305_encrypt); + +bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + bool ret; + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(xchacha20poly1305_decrypt); + +#include "selftest/chacha20poly1305.c" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!selftest_run("chacha20poly1305", chacha20poly1305_selftest, + NULL, 0)) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c new file mode 100644 index 0000000000000..e0c5a5d297c01 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); + +static bool curve25519_use_neon __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { &curve25519_use_neon }; +static void __init curve25519_fpu_init(void) +{ + curve25519_use_neon = elf_hwcap & HWCAP_NEON; +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + simd_context_t simd_context; + bool used_arch = false; + + simd_get(&simd_context); + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && curve25519_use_neon && + simd_use(&simd_context)) { + curve25519_neon(mypublic, secret, basepoint); + used_arch = true; + } + simd_put(&simd_context); + return used_arch; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S new file mode 100644 index 0000000000000..8eca8a11ef28f --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S @@ -0,0 +1,2064 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This + * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been + * manually reworked for use in kernel space. + */ + +#if defined(CONFIG_KERNEL_MODE_NEON) && !defined(__ARMEB__) +#include + +.text +.fpu neon +.arch armv7-a +.align 4 + +SYM_FUNC_START(curve25519_neon) + push {r4-r11, lr} + mov ip, sp + sub r3, sp, #704 + and r3, r3, #0xfffffff0 + mov sp, r3 + movw r4, #0 + movw r5, #254 + vmov.i32 q0, #1 + vshr.u64 q1, q0, #7 + vshr.u64 q0, q0, #8 + vmov.i32 d4, #19 + vmov.i32 d5, #38 + add r6, sp, #480 + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128] + add r6, r3, #0 + vmov.i32 q2, #0 + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 d4, [r6, : 64] + add r6, r3, #0 + movw r7, #960 + sub r7, r7, #2 + neg r7, r7 + sub r7, r7, r7, LSL #7 + str r7, [r6] + add r6, sp, #672 + vld1.8 {d4-d5}, [r1]! + vld1.8 {d6-d7}, [r1] + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d6-d7}, [r6, : 128] + sub r1, r6, #16 + ldrb r6, [r1] + and r6, r6, #248 + strb r6, [r1] + ldrb r6, [r1, #31] + and r6, r6, #127 + orr r6, r6, #64 + strb r6, [r1, #31] + vmov.i64 q2, #0xffffffff + vshr.u64 q3, q2, #7 + vshr.u64 q2, q2, #6 + vld1.8 {d8}, [r2] + vld1.8 {d10}, [r2] + add r2, r2, #6 + vld1.8 {d12}, [r2] + vld1.8 {d14}, [r2] + add r2, r2, #6 + vld1.8 {d16}, [r2] + add r2, r2, #4 + vld1.8 {d18}, [r2] + vld1.8 {d20}, [r2] + add r2, r2, #6 + vld1.8 {d22}, [r2] + add r2, r2, #2 + vld1.8 {d24}, [r2] + vld1.8 {d26}, [r2] + vshr.u64 q5, q5, #26 + vshr.u64 q6, q6, #3 + vshr.u64 q7, q7, #29 + vshr.u64 q8, q8, #6 + vshr.u64 q10, q10, #25 + vshr.u64 q11, q11, #3 + vshr.u64 q12, q12, #12 + vshr.u64 q13, q13, #38 + vand q4, q4, q2 + vand q6, q6, q2 + vand q8, q8, q2 + vand q10, q10, q2 + vand q2, q12, q2 + vand q5, q5, q3 + vand q7, q7, q3 + vand q9, q9, q3 + vand q11, q11, q3 + vand q3, q13, q3 + add r2, r3, #48 + vadd.i64 q12, q4, q1 + vadd.i64 q13, q10, q1 + vshr.s64 q12, q12, #26 + vshr.s64 q13, q13, #26 + vadd.i64 q5, q5, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q14, q5, q0 + vadd.i64 q11, q11, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q11, q0 + vsub.i64 q4, q4, q12 + vshr.s64 q12, q14, #25 + vsub.i64 q10, q10, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q14, q6, q1 + vadd.i64 q2, q2, q13 + vsub.i64 q5, q5, q12 + vshr.s64 q12, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q1 + vadd.i64 q7, q7, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q15, q7, q0 + vsub.i64 q11, q11, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q12 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q3, q0 + vadd.i64 q8, q8, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q15, q8, q1 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q7, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q9, q9, q12 + vtrn.32 d12, d14 + vshl.i64 q12, q12, #26 + vtrn.32 d13, d15 + vadd.i64 q0, q9, q0 + vadd.i64 q4, q4, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q6, q13, #4 + vsub.i64 q7, q8, q12 + vshr.s64 q0, q0, #25 + vadd.i64 q4, q4, q6 + vadd.i64 q6, q10, q0 + vshl.i64 q0, q0, #25 + vadd.i64 q8, q6, q1 + vadd.i64 q4, q4, q13 + vshl.i64 q10, q13, #25 + vadd.i64 q1, q4, q1 + vsub.i64 q0, q9, q0 + vshr.s64 q8, q8, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d14, d0 + vshr.s64 q1, q1, #26 + vtrn.32 d15, d1 + vadd.i64 q0, q11, q8 + vst1.8 d14, [r2, : 64] + vshl.i64 q7, q8, #26 + vadd.i64 q5, q5, q1 + vtrn.32 d4, d6 + vshl.i64 q1, q1, #26 + vtrn.32 d5, d7 + vsub.i64 q3, q6, q7 + add r2, r2, #16 + vsub.i64 q1, q4, q1 + vst1.8 d4, [r2, : 64] + vtrn.32 d6, d0 + vtrn.32 d7, d1 + sub r2, r2, #8 + vtrn.32 d2, d10 + vtrn.32 d3, d11 + vst1.8 d6, [r2, : 64] + sub r2, r2, #24 + vst1.8 d2, [r2, : 64] + add r2, r3, #96 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #144 + vmov.i32 q0, #0 + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #240 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #48 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d4, [r6, : 64] +.Lmainloop: + mov r2, r5, LSR #3 + and r6, r5, #7 + ldrb r2, [r1, r2] + mov r2, r2, LSR r6 + and r2, r2, #1 + str r5, [sp, #456] + eor r4, r4, r2 + str r2, [sp, #460] + neg r2, r4 + add r4, r3, #96 + add r5, r3, #192 + add r6, r3, #144 + vld1.8 {d8-d9}, [r4, : 128]! + add r7, r3, #240 + vld1.8 {d10-d11}, [r5, : 128]! + veor q6, q4, q5 + vld1.8 {d14-d15}, [r6, : 128]! + vdup.i32 q8, r2 + vld1.8 {d18-d19}, [r7, : 128]! + veor q10, q7, q9 + vld1.8 {d22-d23}, [r4, : 128]! + vand q6, q6, q8 + vld1.8 {d24-d25}, [r5, : 128]! + vand q10, q10, q8 + vld1.8 {d26-d27}, [r6, : 128]! + veor q4, q4, q6 + vld1.8 {d28-d29}, [r7, : 128]! + veor q5, q5, q6 + vld1.8 {d0}, [r4, : 64] + veor q6, q7, q10 + vld1.8 {d2}, [r5, : 64] + veor q7, q9, q10 + vld1.8 {d4}, [r6, : 64] + veor q9, q11, q12 + vld1.8 {d6}, [r7, : 64] + veor q10, q0, q1 + sub r2, r4, #32 + vand q9, q9, q8 + sub r4, r5, #32 + vand q10, q10, q8 + sub r5, r6, #32 + veor q11, q11, q9 + sub r6, r7, #32 + veor q0, q0, q10 + veor q9, q12, q9 + veor q1, q1, q10 + veor q10, q13, q14 + veor q12, q2, q3 + vand q10, q10, q8 + vand q8, q12, q8 + veor q12, q13, q10 + veor q2, q2, q8 + veor q10, q14, q10 + veor q3, q3, q8 + vadd.i32 q8, q4, q6 + vsub.i32 q4, q4, q6 + vst1.8 {d16-d17}, [r2, : 128]! + vadd.i32 q6, q11, q12 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q4, q11, q12 + vst1.8 {d12-d13}, [r2, : 128]! + vadd.i32 q6, q0, q2 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q0, q0, q2 + vst1.8 d12, [r2, : 64] + vadd.i32 q2, q5, q7 + vst1.8 d0, [r5, : 64] + vsub.i32 q0, q5, q7 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q9, q10 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q9, q10 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q1, q3 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q1, q3 + vst1.8 d4, [r4, : 64] + vst1.8 d0, [r6, : 64] + add r2, sp, #512 + add r4, r3, #96 + add r5, r3, #144 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #288 + vshl.i64 q12, q12, #25 + add r4, r3, #336 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #240 + add r4, r3, #96 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #144 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #192 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128] + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #144 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, r3, #288 + add r4, r3, #336 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vsub.i32 q1, q1, q2 + add r5, r3, #240 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vsub.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #144 + add r4, r3, #96 + add r5, r3, #144 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q2, q0, q1 + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d6-d7}, [r4, : 128]! + vsub.i32 q4, q1, q3 + vadd.i32 q1, q1, q3 + vld1.8 {d6}, [r2, : 64] + vld1.8 {d10}, [r4, : 64] + vsub.i32 q6, q3, q5 + vadd.i32 q3, q3, q5 + vst1.8 {d4-d5}, [r5, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d8-d9}, [r5, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d12, [r5, : 64] + vst1.8 d6, [r6, : 64] + add r2, r3, #0 + add r4, r3, #240 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #336 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #288 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #288 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, sp, #512 + add r4, r3, #144 + add r5, r3, #192 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #144 + vshl.i64 q12, q12, #25 + add r4, r3, #192 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #336 + add r4, r3, #288 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q1, q1, q2 + add r5, r3, #288 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vadd.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #48 + add r4, r3, #144 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #288 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #240 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #240 + vshl.i64 q7, q7, #25 + add r4, r3, #144 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + ldr r2, [sp, #456] + ldr r4, [sp, #460] + subs r5, r2, #1 + bge .Lmainloop + add r1, r3, #144 + add r2, r3, #336 + vld1.8 {d0-d1}, [r1, : 128]! + vld1.8 {d2-d3}, [r1, : 128]! + vld1.8 {d4}, [r1, : 64] + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 d4, [r2, : 64] + movw r1, #0 +.Linvertloop: + add r2, r3, #144 + movw r4, #0 + movw r5, #2 + cmp r1, #1 + moveq r5, #1 + addeq r2, r3, #336 + addeq r4, r3, #48 + cmp r1, #2 + moveq r5, #1 + addeq r2, r3, #48 + cmp r1, #3 + moveq r5, #5 + addeq r4, r3, #336 + cmp r1, #4 + moveq r5, #10 + cmp r1, #5 + moveq r5, #20 + cmp r1, #6 + moveq r5, #10 + addeq r2, r3, #336 + addeq r4, r3, #336 + cmp r1, #7 + moveq r5, #50 + cmp r1, #8 + moveq r5, #100 + cmp r1, #9 + moveq r5, #50 + addeq r2, r3, #336 + cmp r1, #10 + moveq r5, #5 + addeq r2, r3, #48 + cmp r1, #11 + moveq r5, #0 + addeq r2, r3, #96 + add r6, r3, #144 + add r7, r3, #288 + vld1.8 {d0-d1}, [r6, : 128]! + vld1.8 {d2-d3}, [r6, : 128]! + vld1.8 {d4}, [r6, : 64] + vst1.8 {d0-d1}, [r7, : 128]! + vst1.8 {d2-d3}, [r7, : 128]! + vst1.8 d4, [r7, : 64] + cmp r5, #0 + beq .Lskipsquaringloop +.Lsquaringloop: + add r6, r3, #288 + add r7, r3, #288 + add r8, r3, #288 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r7, : 128]! + vld1.8 {d6-d7}, [r7, : 128]! + vld1.8 {d9}, [r7, : 64] + vld1.8 {d10-d11}, [r6, : 128]! + add r7, sp, #384 + vld1.8 {d12-d13}, [r6, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r6, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r7, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r7, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r7, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r7, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r7, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r6, r7, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r6, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r6, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r6, r6, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r6, : 64] + sub r6, r6, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r7, sp, #480 + vld1.8 {d14-d15}, [r7, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r6, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r6, : 64]! + add r6, sp, #496 + vld1.8 {d20-d21}, [r6, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r6, r8, #8 + vst1.8 d0, [r6, : 64] + vsub.i64 d19, d19, d9 + add r6, r6, #16 + vst1.8 d16, [r6, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r6, r6, #8 + vst1.8 d6, [r6, : 64] + add r6, r6, #16 + vst1.8 d18, [r6, : 64] + vzip.i32 q2, q5 + sub r6, r6, #32 + vst1.8 d4, [r6, : 64] + subs r5, r5, #1 + bhi .Lsquaringloop +.Lskipsquaringloop: + mov r2, r2 + add r5, r3, #288 + add r6, r3, #144 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r5, : 128]! + vld1.8 {d6-d7}, [r5, : 128]! + vld1.8 {d9}, [r5, : 64] + vld1.8 {d10-d11}, [r2, : 128]! + add r5, sp, #384 + vld1.8 {d12-d13}, [r2, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r2, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r5, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r5, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r5, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r5, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r5, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r2, r5, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r2, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r2, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r2, r2, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r2, : 64] + sub r2, r2, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r5, sp, #480 + vld1.8 {d14-d15}, [r5, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r2, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r2, : 64]! + add r2, sp, #496 + vld1.8 {d20-d21}, [r2, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r2, r6, #8 + vst1.8 d0, [r2, : 64] + vsub.i64 d19, d19, d9 + add r2, r2, #16 + vst1.8 d16, [r2, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r2, r2, #8 + vst1.8 d6, [r2, : 64] + add r2, r2, #16 + vst1.8 d18, [r2, : 64] + vzip.i32 q2, q5 + sub r2, r2, #32 + vst1.8 d4, [r2, : 64] + cmp r4, #0 + beq .Lskippostcopy + add r2, r3, #144 + mov r4, r4 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskippostcopy: + cmp r1, #1 + bne .Lskipfinalcopy + add r2, r3, #288 + add r4, r3, #144 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskipfinalcopy: + add r1, r1, #1 + cmp r1, #12 + blo .Linvertloop + add r1, r3, #144 + ldr r2, [r1], #4 + ldr r3, [r1], #4 + ldr r4, [r1], #4 + ldr r5, [r1], #4 + ldr r6, [r1], #4 + ldr r7, [r1], #4 + ldr r8, [r1], #4 + ldr r9, [r1], #4 + ldr r10, [r1], #4 + ldr r1, [r1] + add r11, r1, r1, LSL #4 + add r11, r11, r1, LSL #1 + add r11, r11, #16777216 + mov r11, r11, ASR #25 + add r11, r11, r2 + mov r11, r11, ASR #26 + add r11, r11, r3 + mov r11, r11, ASR #25 + add r11, r11, r4 + mov r11, r11, ASR #26 + add r11, r11, r5 + mov r11, r11, ASR #25 + add r11, r11, r6 + mov r11, r11, ASR #26 + add r11, r11, r7 + mov r11, r11, ASR #25 + add r11, r11, r8 + mov r11, r11, ASR #26 + add r11, r11, r9 + mov r11, r11, ASR #25 + add r11, r11, r10 + mov r11, r11, ASR #26 + add r11, r11, r1 + mov r11, r11, ASR #25 + add r2, r2, r11 + add r2, r2, r11, LSL #1 + add r2, r2, r11, LSL #4 + mov r11, r2, ASR #26 + add r3, r3, r11 + sub r2, r2, r11, LSL #26 + mov r11, r3, ASR #25 + add r4, r4, r11 + sub r3, r3, r11, LSL #25 + mov r11, r4, ASR #26 + add r5, r5, r11 + sub r4, r4, r11, LSL #26 + mov r11, r5, ASR #25 + add r6, r6, r11 + sub r5, r5, r11, LSL #25 + mov r11, r6, ASR #26 + add r7, r7, r11 + sub r6, r6, r11, LSL #26 + mov r11, r7, ASR #25 + add r8, r8, r11 + sub r7, r7, r11, LSL #25 + mov r11, r8, ASR #26 + add r9, r9, r11 + sub r8, r8, r11, LSL #26 + mov r11, r9, ASR #25 + add r10, r10, r11 + sub r9, r9, r11, LSL #25 + mov r11, r10, ASR #26 + add r1, r1, r11 + sub r10, r10, r11, LSL #26 + mov r11, r1, ASR #25 + sub r1, r1, r11, LSL #25 + add r2, r2, r3, LSL #26 + mov r3, r3, LSR #6 + add r3, r3, r4, LSL #19 + mov r4, r4, LSR #13 + add r4, r4, r5, LSL #13 + mov r5, r5, LSR #19 + add r5, r5, r6, LSL #6 + add r6, r7, r8, LSL #25 + mov r7, r8, LSR #7 + add r7, r7, r9, LSL #19 + mov r8, r9, LSR #13 + add r8, r8, r10, LSL #12 + mov r9, r10, LSR #20 + add r1, r9, r1, LSL #6 + str r2, [r0] + str r3, [r0, #4] + str r4, [r0, #8] + str r5, [r0, #12] + str r6, [r0, #16] + str r7, [r0, #20] + str r8, [r0, #24] + str r1, [r0, #28] + movw r0, #0 + mov sp, ip + pop {r4-r11, pc} +SYM_FUNC_END(curve25519_neon) +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c new file mode 100644 index 0000000000000..42cfb6c00f984 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2016 The fiat-crypto Authors. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally + * machine generated, it has been tweaked to be suitable for use in the kernel. + * It is optimized for 32-bit machines and machines that cannot work efficiently + * with 128-bit integer types. + */ + +/* fe means field element. Here the field is \Z/(2^255-19). An element t, + * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 + * t[3]+2^102 t[4]+...+2^230 t[9]. + * fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc. + * Multiplication and carrying produce fe from fe_loose. + */ +typedef struct fe { u32 v[10]; } fe; + +/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc + * Addition and subtraction produce fe_loose from (fe, fe). + */ +typedef struct fe_loose { u32 v[10]; } fe_loose; + +static __always_inline void fe_frombytes_impl(u32 h[10], const u8 *s) +{ + /* Ignores top bit of s. */ + u32 a0 = get_unaligned_le32(s); + u32 a1 = get_unaligned_le32(s+4); + u32 a2 = get_unaligned_le32(s+8); + u32 a3 = get_unaligned_le32(s+12); + u32 a4 = get_unaligned_le32(s+16); + u32 a5 = get_unaligned_le32(s+20); + u32 a6 = get_unaligned_le32(s+24); + u32 a7 = get_unaligned_le32(s+28); + h[0] = a0&((1<<26)-1); /* 26 used, 32-26 left. 26 */ + h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 = 6+19 = 25 */ + h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */ + h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) + 6 = 19+ 6 = 25 */ + h[4] = (a3>> 6); /* (32- 6) = 26 */ + h[5] = a4&((1<<25)-1); /* 25 */ + h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 = 7+19 = 26 */ + h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */ + h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) + 6 = 20+ 6 = 26 */ + h[9] = (a7>> 6)&((1<<25)-1); /* 25 */ +} + +static __always_inline void fe_frombytes(fe *h, const u8 *s) +{ + fe_frombytes_impl(h->v, s); +} + +static __always_inline u8 /*bool*/ +addcarryx_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of carry + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 25) - 1); + return (x >> 25) & 1; +} + +static __always_inline u8 /*bool*/ +addcarryx_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of carry + * (27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 26) - 1); + return (x >> 26) & 1; +} + +static __always_inline u8 /*bool*/ +subborrow_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of borrow + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 25) - 1); + return x >> 31; +} + +static __always_inline u8 /*bool*/ +subborrow_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of borrow + *(27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 26) - 1); + return x >> 31; +} + +static __always_inline u32 cmovznz32(u32 t, u32 z, u32 nz) +{ + t = -!!t; /* all set if nonzero, 0 if 0 */ + return (t&nz) | ((~t)&z); +} + +static __always_inline void fe_freeze(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u32 x20; u8/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20); + { u32 x23; u8/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23); + { u32 x26; u8/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26); + { u32 x29; u8/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29); + { u32 x32; u8/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32); + { u32 x35; u8/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35); + { u32 x38; u8/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38); + { u32 x41; u8/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41); + { u32 x44; u8/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44); + { u32 x47; u8/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47); + { u32 x49 = cmovznz32(x48, 0x0, 0xffffffff); + { u32 x50 = (x49 & 0x3ffffed); + { u32 x52; u8/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52); + { u32 x54 = (x49 & 0x1ffffff); + { u32 x56; u8/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56); + { u32 x58 = (x49 & 0x3ffffff); + { u32 x60; u8/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60); + { u32 x62 = (x49 & 0x1ffffff); + { u32 x64; u8/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64); + { u32 x66 = (x49 & 0x3ffffff); + { u32 x68; u8/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68); + { u32 x70 = (x49 & 0x1ffffff); + { u32 x72; u8/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72); + { u32 x74 = (x49 & 0x3ffffff); + { u32 x76; u8/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76); + { u32 x78 = (x49 & 0x1ffffff); + { u32 x80; u8/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80); + { u32 x82 = (x49 & 0x3ffffff); + { u32 x84; u8/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84); + { u32 x86 = (x49 & 0x1ffffff); + { u32 x88; addcarryx_u25(x85, x47, x86, &x88); + out[0] = x52; + out[1] = x56; + out[2] = x60; + out[3] = x64; + out[4] = x68; + out[5] = x72; + out[6] = x76; + out[7] = x80; + out[8] = x84; + out[9] = x88; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_tobytes(u8 s[32], const fe *f) +{ + u32 h[10]; + fe_freeze(h, f->v); + s[0] = h[0] >> 0; + s[1] = h[0] >> 8; + s[2] = h[0] >> 16; + s[3] = (h[0] >> 24) | (h[1] << 2); + s[4] = h[1] >> 6; + s[5] = h[1] >> 14; + s[6] = (h[1] >> 22) | (h[2] << 3); + s[7] = h[2] >> 5; + s[8] = h[2] >> 13; + s[9] = (h[2] >> 21) | (h[3] << 5); + s[10] = h[3] >> 3; + s[11] = h[3] >> 11; + s[12] = (h[3] >> 19) | (h[4] << 6); + s[13] = h[4] >> 2; + s[14] = h[4] >> 10; + s[15] = h[4] >> 18; + s[16] = h[5] >> 0; + s[17] = h[5] >> 8; + s[18] = h[5] >> 16; + s[19] = (h[5] >> 24) | (h[6] << 1); + s[20] = h[6] >> 7; + s[21] = h[6] >> 15; + s[22] = (h[6] >> 23) | (h[7] << 3); + s[23] = h[7] >> 5; + s[24] = h[7] >> 13; + s[25] = (h[7] >> 21) | (h[8] << 4); + s[26] = h[8] >> 4; + s[27] = h[8] >> 12; + s[28] = (h[8] >> 20) | (h[9] << 6); + s[29] = h[9] >> 2; + s[30] = h[9] >> 10; + s[31] = h[9] >> 18; +} + +/* h = f */ +static __always_inline void fe_copy(fe *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +static __always_inline void fe_copy_lt(fe_loose *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +/* h = 0 */ +static __always_inline void fe_0(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); +} + +/* h = 1 */ +static __always_inline void fe_1(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); + h->v[0] = 1; +} + +static void fe_add_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = (x5 + x23); + out[1] = (x7 + x25); + out[2] = (x9 + x27); + out[3] = (x11 + x29); + out[4] = (x13 + x31); + out[5] = (x15 + x33); + out[6] = (x17 + x35); + out[7] = (x19 + x37); + out[8] = (x21 + x39); + out[9] = (x20 + x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f + g + * Can overlap h with f or g. + */ +static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g) +{ + fe_add_impl(h->v, f->v, g->v); +} + +static void fe_sub_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = ((0x7ffffda + x5) - x23); + out[1] = ((0x3fffffe + x7) - x25); + out[2] = ((0x7fffffe + x9) - x27); + out[3] = ((0x3fffffe + x11) - x29); + out[4] = ((0x7fffffe + x13) - x31); + out[5] = ((0x3fffffe + x15) - x33); + out[6] = ((0x7fffffe + x17) - x35); + out[7] = ((0x3fffffe + x19) - x37); + out[8] = ((0x7fffffe + x21) - x39); + out[9] = ((0x3fffffe + x20) - x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f - g + * Can overlap h with f or g. + */ +static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g) +{ + fe_sub_impl(h->v, f->v, g->v); +} + +static void fe_mul_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void +fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static void fe_sqr_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u64 x19 = ((u64)x2 * x2); + { u64 x20 = ((u64)(0x2 * x2) * x4); + { u64 x21 = (0x2 * (((u64)x4 * x4) + ((u64)x2 * x6))); + { u64 x22 = (0x2 * (((u64)x4 * x6) + ((u64)x2 * x8))); + { u64 x23 = ((((u64)x6 * x6) + ((u64)(0x4 * x4) * x8)) + ((u64)(0x2 * x2) * x10)); + { u64 x24 = (0x2 * ((((u64)x6 * x8) + ((u64)x4 * x10)) + ((u64)x2 * x12))); + { u64 x25 = (0x2 * (((((u64)x8 * x8) + ((u64)x6 * x10)) + ((u64)x2 * x14)) + ((u64)(0x2 * x4) * x12))); + { u64 x26 = (0x2 * (((((u64)x8 * x10) + ((u64)x6 * x12)) + ((u64)x4 * x14)) + ((u64)x2 * x16))); + { u64 x27 = (((u64)x10 * x10) + (0x2 * ((((u64)x6 * x14) + ((u64)x2 * x18)) + (0x2 * (((u64)x4 * x16) + ((u64)x8 * x12)))))); + { u64 x28 = (0x2 * ((((((u64)x10 * x12) + ((u64)x8 * x14)) + ((u64)x6 * x16)) + ((u64)x4 * x18)) + ((u64)x2 * x17))); + { u64 x29 = (0x2 * (((((u64)x12 * x12) + ((u64)x10 * x14)) + ((u64)x6 * x18)) + (0x2 * (((u64)x8 * x16) + ((u64)x4 * x17))))); + { u64 x30 = (0x2 * (((((u64)x12 * x14) + ((u64)x10 * x16)) + ((u64)x8 * x18)) + ((u64)x6 * x17))); + { u64 x31 = (((u64)x14 * x14) + (0x2 * (((u64)x10 * x18) + (0x2 * (((u64)x12 * x16) + ((u64)x8 * x17)))))); + { u64 x32 = (0x2 * ((((u64)x14 * x16) + ((u64)x12 * x18)) + ((u64)x10 * x17))); + { u64 x33 = (0x2 * ((((u64)x16 * x16) + ((u64)x14 * x18)) + ((u64)(0x2 * x12) * x17))); + { u64 x34 = (0x2 * (((u64)x16 * x18) + ((u64)x14 * x17))); + { u64 x35 = (((u64)x18 * x18) + ((u64)(0x4 * x16) * x17)); + { u64 x36 = ((u64)(0x2 * x18) * x17); + { u64 x37 = ((u64)(0x2 * x17) * x17); + { u64 x38 = (x27 + (x37 << 0x4)); + { u64 x39 = (x38 + (x37 << 0x1)); + { u64 x40 = (x39 + x37); + { u64 x41 = (x26 + (x36 << 0x4)); + { u64 x42 = (x41 + (x36 << 0x1)); + { u64 x43 = (x42 + x36); + { u64 x44 = (x25 + (x35 << 0x4)); + { u64 x45 = (x44 + (x35 << 0x1)); + { u64 x46 = (x45 + x35); + { u64 x47 = (x24 + (x34 << 0x4)); + { u64 x48 = (x47 + (x34 << 0x1)); + { u64 x49 = (x48 + x34); + { u64 x50 = (x23 + (x33 << 0x4)); + { u64 x51 = (x50 + (x33 << 0x1)); + { u64 x52 = (x51 + x33); + { u64 x53 = (x22 + (x32 << 0x4)); + { u64 x54 = (x53 + (x32 << 0x1)); + { u64 x55 = (x54 + x32); + { u64 x56 = (x21 + (x31 << 0x4)); + { u64 x57 = (x56 + (x31 << 0x1)); + { u64 x58 = (x57 + x31); + { u64 x59 = (x20 + (x30 << 0x4)); + { u64 x60 = (x59 + (x30 << 0x1)); + { u64 x61 = (x60 + x30); + { u64 x62 = (x19 + (x29 << 0x4)); + { u64 x63 = (x62 + (x29 << 0x1)); + { u64 x64 = (x63 + x29); + { u64 x65 = (x64 >> 0x1a); + { u32 x66 = ((u32)x64 & 0x3ffffff); + { u64 x67 = (x65 + x61); + { u64 x68 = (x67 >> 0x19); + { u32 x69 = ((u32)x67 & 0x1ffffff); + { u64 x70 = (x68 + x58); + { u64 x71 = (x70 >> 0x1a); + { u32 x72 = ((u32)x70 & 0x3ffffff); + { u64 x73 = (x71 + x55); + { u64 x74 = (x73 >> 0x19); + { u32 x75 = ((u32)x73 & 0x1ffffff); + { u64 x76 = (x74 + x52); + { u64 x77 = (x76 >> 0x1a); + { u32 x78 = ((u32)x76 & 0x3ffffff); + { u64 x79 = (x77 + x49); + { u64 x80 = (x79 >> 0x19); + { u32 x81 = ((u32)x79 & 0x1ffffff); + { u64 x82 = (x80 + x46); + { u64 x83 = (x82 >> 0x1a); + { u32 x84 = ((u32)x82 & 0x3ffffff); + { u64 x85 = (x83 + x43); + { u64 x86 = (x85 >> 0x19); + { u32 x87 = ((u32)x85 & 0x1ffffff); + { u64 x88 = (x86 + x40); + { u64 x89 = (x88 >> 0x1a); + { u32 x90 = ((u32)x88 & 0x3ffffff); + { u64 x91 = (x89 + x28); + { u64 x92 = (x91 >> 0x19); + { u32 x93 = ((u32)x91 & 0x1ffffff); + { u64 x94 = (x66 + (0x13 * x92)); + { u32 x95 = (u32) (x94 >> 0x1a); + { u32 x96 = ((u32)x94 & 0x3ffffff); + { u32 x97 = (x95 + x69); + { u32 x98 = (x97 >> 0x19); + { u32 x99 = (x97 & 0x1ffffff); + out[0] = x96; + out[1] = x99; + out[2] = (x98 + x72); + out[3] = x75; + out[4] = x78; + out[5] = x81; + out[6] = x84; + out[7] = x87; + out[8] = x90; + out[9] = x93; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_sq_tl(fe *h, const fe_loose *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_sq_tt(fe *h, const fe *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_loose_invert(fe *out, const fe_loose *z) +{ + fe t0; + fe t1; + fe t2; + fe t3; + int i; + + fe_sq_tl(&t0, z); + fe_sq_tt(&t1, &t0); + for (i = 1; i < 2; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_tlt(&t1, z, &t1); + fe_mul_ttt(&t0, &t0, &t1); + fe_sq_tt(&t2, &t0); + fe_mul_ttt(&t1, &t1, &t2); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 20; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 100; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t1, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_ttt(out, &t1, &t0); +} + +static __always_inline void fe_invert(fe *out, const fe *z) +{ + fe_loose l; + fe_copy_lt(&l, z); + fe_loose_invert(out, &l); +} + +/* Replace (f,g) with (g,f) if b == 1; + * replace (f,g) with (f,g) if b == 0. + * + * Preconditions: b in {0,1} + */ +static __always_inline void fe_cswap(fe *f, fe *g, unsigned int b) +{ + unsigned i; + b = 0 - b; + for (i = 0; i < 10; i++) { + u32 x = f->v[i] ^ g->v[i]; + x &= b; + f->v[i] ^= x; + g->v[i] ^= x; + } +} + +/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/ +static __always_inline void fe_mul_121666_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = 0; + { const u32 x39 = 0; + { const u32 x37 = 0; + { const u32 x35 = 0; + { const u32 x33 = 0; + { const u32 x31 = 0; + { const u32 x29 = 0; + { const u32 x27 = 0; + { const u32 x25 = 0; + { const u32 x23 = 121666; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul121666(fe *h, const fe_loose *f) +{ + fe_mul_121666_impl(h->v, f->v); +} + +static void curve25519_generic(u8 out[CURVE25519_KEY_SIZE], + const u8 scalar[CURVE25519_KEY_SIZE], + const u8 point[CURVE25519_KEY_SIZE]) +{ + fe x1, x2, z2, x3, z3; + fe_loose x2l, z2l, x3l; + unsigned swap = 0; + int pos; + u8 e[32]; + + memcpy(e, scalar, 32); + curve25519_clamp_secret(e); + + /* The following implementation was transcribed to Coq and proven to + * correspond to unary scalar multiplication in affine coordinates given + * that x1 != 0 is the x coordinate of some point on the curve. It was + * also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives + * z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was + * quantified over the underlying field, so it applies to Curve25519 + * itself and the quadratic twist of Curve25519. It was not proven in + * Coq that prime-field arithmetic correctly simulates extension-field + * arithmetic on prime-field values. The decoding of the byte array + * representation of e was not considered. + * + * Specification of Montgomery curves in affine coordinates: + * + * + * Proof that these form a group that is isomorphic to a Weierstrass + * curve: + * + * + * Coq transcription and correctness proof of the loop + * (where scalarbits=255): + * + * + * preconditions: 0 <= e < 2^255 (not necessarily e < order), + * fe_invert(0) = 0 + */ + fe_frombytes(&x1, point); + fe_1(&x2); + fe_0(&z2); + fe_copy(&x3, &x1); + fe_1(&z3); + + for (pos = 254; pos >= 0; --pos) { + fe tmp0, tmp1; + fe_loose tmp0l, tmp1l; + /* loop invariant as of right before the test, for the case + * where x1 != 0: + * pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 + * is nonzero + * let r := e >> (pos+1) in the following equalities of + * projective points: + * to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + * to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + * x1 is the nonzero x coordinate of the nonzero + * point (r*P-(r+1)*P) + */ + unsigned b = 1 & (e[pos / 8] >> (pos & 7)); + swap ^= b; + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + swap = b; + /* Coq transcription of ladderstep formula (called from + * transcribed loop): + * + * + * x1 != 0 + * x1 = 0 + */ + fe_sub(&tmp0l, &x3, &z3); + fe_sub(&tmp1l, &x2, &z2); + fe_add(&x2l, &x2, &z2); + fe_add(&z2l, &x3, &z3); + fe_mul_tll(&z3, &tmp0l, &x2l); + fe_mul_tll(&z2, &z2l, &tmp1l); + fe_sq_tl(&tmp0, &tmp1l); + fe_sq_tl(&tmp1, &x2l); + fe_add(&x3l, &z3, &z2); + fe_sub(&z2l, &z3, &z2); + fe_mul_ttt(&x2, &tmp1, &tmp0); + fe_sub(&tmp1l, &tmp1, &tmp0); + fe_sq_tl(&z2, &z2l); + fe_mul121666(&z3, &tmp1l); + fe_sq_tl(&x3, &x3l); + fe_add(&tmp0l, &tmp0, &z3); + fe_mul_ttt(&z3, &x1, &z2); + fe_mul_tll(&z2, &tmp1l, &tmp0l); + } + /* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) + * else (x2, z2) + */ + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + + fe_invert(&z2, &z2); + fe_mul_ttt(&x2, &x2, &z2); + fe_tobytes(out, &x2); + + memzero_explicit(&x1, sizeof(x1)); + memzero_explicit(&x2, sizeof(x2)); + memzero_explicit(&z2, sizeof(z2)); + memzero_explicit(&x3, sizeof(x3)); + memzero_explicit(&z3, sizeof(z3)); + memzero_explicit(&x2l, sizeof(x2l)); + memzero_explicit(&z2l, sizeof(z2l)); + memzero_explicit(&x3l, sizeof(x3l)); + memzero_explicit(&e, sizeof(e)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c new file mode 100644 index 0000000000000..0f729ec74ece8 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2016-2017 INRIA and Microsoft Corporation. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally machine + * generated, it has been tweaked to be suitable for use in the kernel. It is + * optimized for 64-bit machines that can efficiently work with 128-bit + * integer types. + */ + +typedef __uint128_t u128; + +static __always_inline u64 u64_eq_mask(u64 a, u64 b) +{ + u64 x = a ^ b; + u64 minus_x = ~x + (u64)1U; + u64 x_or_minus_x = x | minus_x; + u64 xnx = x_or_minus_x >> (u32)63U; + u64 c = xnx - (u64)1U; + return c; +} + +static __always_inline u64 u64_gte_mask(u64 a, u64 b) +{ + u64 x = a; + u64 y = b; + u64 x_xor_y = x ^ y; + u64 x_sub_y = x - y; + u64 x_sub_y_xor_y = x_sub_y ^ y; + u64 q = x_xor_y | x_sub_y_xor_y; + u64 x_xor_q = x ^ q; + u64 x_xor_q_ = x_xor_q >> (u32)63U; + u64 c = x_xor_q_ - (u64)1U; + return c; +} + +static __always_inline void modulo_carry_top(u64 *b) +{ + u64 b4 = b[4]; + u64 b0 = b[0]; + u64 b4_ = b4 & 0x7ffffffffffffLLU; + u64 b0_ = b0 + 19 * (b4 >> 51); + b[4] = b4_; + b[0] = b0_; +} + +static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input) +{ + { + u128 xi = input[0]; + output[0] = ((u64)(xi)); + } + { + u128 xi = input[1]; + output[1] = ((u64)(xi)); + } + { + u128 xi = input[2]; + output[2] = ((u64)(xi)); + } + { + u128 xi = input[3]; + output[3] = ((u64)(xi)); + } + { + u128 xi = input[4]; + output[4] = ((u64)(xi)); + } +} + +static __always_inline void +fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s) +{ + output[0] += (u128)input[0] * s; + output[1] += (u128)input[1] * s; + output[2] += (u128)input[2] * s; + output[3] += (u128)input[3] * s; + output[4] += (u128)input[4] * s; +} + +static __always_inline void fproduct_carry_wide_(u128 *tmp) +{ + { + u32 ctr = 0; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 1; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + + { + u32 ctr = 2; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 3; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } +} + +static __always_inline void fmul_shift_reduce(u64 *output) +{ + u64 tmp = output[4]; + u64 b0; + { + u32 ctr = 5 - 0 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 1 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 2 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 3 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + output[0] = tmp; + b0 = output[0]; + output[0] = 19 * b0; +} + +static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input, + u64 *input21) +{ + u32 i; + u64 input2i; + { + u64 input2i = input21[0]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[1]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[2]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[3]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + i = 4; + input2i = input21[i]; + fproduct_sum_scalar_multiplication_(output, input, input2i); +} + +static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21) +{ + u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] }; + { + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + u128 t[5] = { 0 }; + fmul_mul_shift_reduce_(t, tmp, input21); + fproduct_carry_wide_(t); + b4 = t[4]; + b0 = t[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + t[4] = b4_; + t[0] = b0_; + fproduct_copy_from_wide_(output, t); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; + } +} + +static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output) +{ + u64 r0 = output[0]; + u64 r1 = output[1]; + u64 r2 = output[2]; + u64 r3 = output[3]; + u64 r4 = output[4]; + u64 d0 = r0 * 2; + u64 d1 = r1 * 2; + u64 d2 = r2 * 2 * 19; + u64 d419 = r4 * 19; + u64 d4 = d419 * 2; + u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) + + (((u128)(d2) * (r3)))); + u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) + + (((u128)(r3 * 19) * (r3)))); + u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) + + (((u128)(d4) * (r3)))); + u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) + + (((u128)(r4) * (d419)))); + u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) + + (((u128)(r2) * (r2)))); + tmp[0] = s0; + tmp[1] = s1; + tmp[2] = s2; + tmp[3] = s3; + tmp[4] = s4; +} + +static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output) +{ + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + fsquare_fsquare__(tmp, output); + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; +} + +static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp, + u32 count1) +{ + u32 i; + fsquare_fsquare_(tmp, output); + for (i = 1; i < count1; ++i) + fsquare_fsquare_(tmp, output); +} + +static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input, + u32 count1) +{ + u128 t[5]; + memcpy(output, input, 5 * sizeof(*input)); + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void fsquare_fsquare_times_inplace(u64 *output, + u32 count1) +{ + u128 t[5]; + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void crecip_crecip(u64 *out, u64 *z) +{ + u64 buf[20] = { 0 }; + u64 *a0 = buf; + u64 *t00 = buf + 5; + u64 *b0 = buf + 10; + u64 *t01; + u64 *b1; + u64 *c0; + u64 *a; + u64 *t0; + u64 *b; + u64 *c; + fsquare_fsquare_times(a0, z, 1); + fsquare_fsquare_times(t00, a0, 2); + fmul_fmul(b0, t00, z); + fmul_fmul(a0, b0, a0); + fsquare_fsquare_times(t00, a0, 1); + fmul_fmul(b0, t00, b0); + fsquare_fsquare_times(t00, b0, 5); + t01 = buf + 5; + b1 = buf + 10; + c0 = buf + 15; + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 10); + fmul_fmul(c0, t01, b1); + fsquare_fsquare_times(t01, c0, 20); + fmul_fmul(t01, t01, c0); + fsquare_fsquare_times_inplace(t01, 10); + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 50); + a = buf; + t0 = buf + 5; + b = buf + 10; + c = buf + 15; + fmul_fmul(c, t0, b); + fsquare_fsquare_times(t0, c, 100); + fmul_fmul(t0, t0, c); + fsquare_fsquare_times_inplace(t0, 50); + fmul_fmul(t0, t0, b); + fsquare_fsquare_times_inplace(t0, 5); + fmul_fmul(out, t0, a); +} + +static __always_inline void fsum(u64 *a, u64 *b) +{ + a[0] += b[0]; + a[1] += b[1]; + a[2] += b[2]; + a[3] += b[3]; + a[4] += b[4]; +} + +static __always_inline void fdifference(u64 *a, u64 *b) +{ + u64 tmp[5] = { 0 }; + u64 b0; + u64 b1; + u64 b2; + u64 b3; + u64 b4; + memcpy(tmp, b, 5 * sizeof(*b)); + b0 = tmp[0]; + b1 = tmp[1]; + b2 = tmp[2]; + b3 = tmp[3]; + b4 = tmp[4]; + tmp[0] = b0 + 0x3fffffffffff68LLU; + tmp[1] = b1 + 0x3ffffffffffff8LLU; + tmp[2] = b2 + 0x3ffffffffffff8LLU; + tmp[3] = b3 + 0x3ffffffffffff8LLU; + tmp[4] = b4 + 0x3ffffffffffff8LLU; + { + u64 xi = a[0]; + u64 yi = tmp[0]; + a[0] = yi - xi; + } + { + u64 xi = a[1]; + u64 yi = tmp[1]; + a[1] = yi - xi; + } + { + u64 xi = a[2]; + u64 yi = tmp[2]; + a[2] = yi - xi; + } + { + u64 xi = a[3]; + u64 yi = tmp[3]; + a[3] = yi - xi; + } + { + u64 xi = a[4]; + u64 yi = tmp[4]; + a[4] = yi - xi; + } +} + +static __always_inline void fscalar(u64 *output, u64 *b, u64 s) +{ + u128 tmp[5]; + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + { + u64 xi = b[0]; + tmp[0] = ((u128)(xi) * (s)); + } + { + u64 xi = b[1]; + tmp[1] = ((u128)(xi) * (s)); + } + { + u64 xi = b[2]; + tmp[2] = ((u128)(xi) * (s)); + } + { + u64 xi = b[3]; + tmp[3] = ((u128)(xi) * (s)); + } + { + u64 xi = b[4]; + tmp[4] = ((u128)(xi) * (s)); + } + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); +} + +static __always_inline void fmul(u64 *output, u64 *a, u64 *b) +{ + fmul_fmul(output, a, b); +} + +static __always_inline void crecip(u64 *output, u64 *input) +{ + crecip_crecip(output, input); +} + +static __always_inline void point_swap_conditional_step(u64 *a, u64 *b, + u64 swap1, u32 ctr) +{ + u32 i = ctr - 1; + u64 ai = a[i]; + u64 bi = b[i]; + u64 x = swap1 & (ai ^ bi); + u64 ai1 = ai ^ x; + u64 bi1 = bi ^ x; + a[i] = ai1; + b[i] = bi1; +} + +static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1) +{ + point_swap_conditional_step(a, b, swap1, 5); + point_swap_conditional_step(a, b, swap1, 4); + point_swap_conditional_step(a, b, swap1, 3); + point_swap_conditional_step(a, b, swap1, 2); + point_swap_conditional_step(a, b, swap1, 1); +} + +static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap) +{ + u64 swap1 = 0 - iswap; + point_swap_conditional5(a, b, swap1); + point_swap_conditional5(a + 5, b + 5, swap1); +} + +static __always_inline void point_copy(u64 *output, u64 *input) +{ + memcpy(output, input, 5 * sizeof(*input)); + memcpy(output + 5, input + 5, 5 * sizeof(*input)); +} + +static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p, + u64 *pq, u64 *qmqp) +{ + u64 *qx = qmqp; + u64 *x2 = pp; + u64 *z2 = pp + 5; + u64 *x3 = ppq; + u64 *z3 = ppq + 5; + u64 *x = p; + u64 *z = p + 5; + u64 *xprime = pq; + u64 *zprime = pq + 5; + u64 buf[40] = { 0 }; + u64 *origx = buf; + u64 *origxprime0 = buf + 5; + u64 *xxprime0; + u64 *zzprime0; + u64 *origxprime; + xxprime0 = buf + 25; + zzprime0 = buf + 30; + memcpy(origx, x, 5 * sizeof(*x)); + fsum(x, z); + fdifference(z, origx); + memcpy(origxprime0, xprime, 5 * sizeof(*xprime)); + fsum(xprime, zprime); + fdifference(zprime, origxprime0); + fmul(xxprime0, xprime, z); + fmul(zzprime0, x, zprime); + origxprime = buf + 5; + { + u64 *xx0; + u64 *zz0; + u64 *xxprime; + u64 *zzprime; + u64 *zzzprime; + xx0 = buf + 15; + zz0 = buf + 20; + xxprime = buf + 25; + zzprime = buf + 30; + zzzprime = buf + 35; + memcpy(origxprime, xxprime, 5 * sizeof(*xxprime)); + fsum(xxprime, zzprime); + fdifference(zzprime, origxprime); + fsquare_fsquare_times(x3, xxprime, 1); + fsquare_fsquare_times(zzzprime, zzprime, 1); + fmul(z3, zzzprime, qx); + fsquare_fsquare_times(xx0, x, 1); + fsquare_fsquare_times(zz0, z, 1); + { + u64 *zzz; + u64 *xx; + u64 *zz; + u64 scalar; + zzz = buf + 10; + xx = buf + 15; + zz = buf + 20; + fmul(x2, xx, zz); + fdifference(zz, xx); + scalar = 121665; + fscalar(zzz, zz, scalar); + fsum(zzz, xx); + fmul(z2, zzz, zz); + } + } +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt) +{ + u64 bit0 = (u64)(byt >> 7); + u64 bit; + point_swap_conditional(nq, nqpq, bit0); + addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q); + bit = (u64)(byt >> 7); + point_swap_conditional(nq2, nqpq2, bit); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, u8 byt) +{ + u8 byt1; + ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt); + byt1 = byt << 1; + ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt, u32 i) +{ + while (i--) { + ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2, + nqpq2, q, byt); + byt <<= 2; + } +} + +static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq, + u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, + u32 i) +{ + while (i--) { + u8 byte = n1[i]; + ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q, + byte, 4); + } +} + +static void ladder_cmult(u64 *result, u8 *n1, u64 *q) +{ + u64 point_buf[40] = { 0 }; + u64 *nq = point_buf; + u64 *nqpq = point_buf + 10; + u64 *nq2 = point_buf + 20; + u64 *nqpq2 = point_buf + 30; + point_copy(nqpq, q); + nq[0] = 1; + ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32); + point_copy(result, nq); +} + +static __always_inline void format_fexpand(u64 *output, const u8 *input) +{ + const u8 *x00 = input + 6; + const u8 *x01 = input + 12; + const u8 *x02 = input + 19; + const u8 *x0 = input + 24; + u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4; + i0 = get_unaligned_le64(input); + i1 = get_unaligned_le64(x00); + i2 = get_unaligned_le64(x01); + i3 = get_unaligned_le64(x02); + i4 = get_unaligned_le64(x0); + output0 = i0 & 0x7ffffffffffffLLU; + output1 = i1 >> 3 & 0x7ffffffffffffLLU; + output2 = i2 >> 6 & 0x7ffffffffffffLLU; + output3 = i3 >> 1 & 0x7ffffffffffffLLU; + output4 = i4 >> 12 & 0x7ffffffffffffLLU; + output[0] = output0; + output[1] = output1; + output[2] = output2; + output[3] = output3; + output[4] = output4; +} + +static __always_inline void format_fcontract_first_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_first_carry_full(u64 *input) +{ + format_fcontract_first_carry_pass(input); + modulo_carry_top(input); +} + +static __always_inline void format_fcontract_second_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_second_carry_full(u64 *input) +{ + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + format_fcontract_second_carry_pass(input); + modulo_carry_top(input); + i0 = input[0]; + i1 = input[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + input[0] = i0_; + input[1] = i1_; +} + +static __always_inline void format_fcontract_trim(u64 *input) +{ + u64 a0 = input[0]; + u64 a1 = input[1]; + u64 a2 = input[2]; + u64 a3 = input[3]; + u64 a4 = input[4]; + u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU); + u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU); + u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU); + u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU); + u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU); + u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4; + u64 a0_ = a0 - (0x7ffffffffffedLLU & mask); + u64 a1_ = a1 - (0x7ffffffffffffLLU & mask); + u64 a2_ = a2 - (0x7ffffffffffffLLU & mask); + u64 a3_ = a3 - (0x7ffffffffffffLLU & mask); + u64 a4_ = a4 - (0x7ffffffffffffLLU & mask); + input[0] = a0_; + input[1] = a1_; + input[2] = a2_; + input[3] = a3_; + input[4] = a4_; +} + +static __always_inline void format_fcontract_store(u8 *output, u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 o0 = t1 << 51 | t0; + u64 o1 = t2 << 38 | t1 >> 13; + u64 o2 = t3 << 25 | t2 >> 26; + u64 o3 = t4 << 12 | t3 >> 39; + u8 *b0 = output; + u8 *b1 = output + 8; + u8 *b2 = output + 16; + u8 *b3 = output + 24; + put_unaligned_le64(o0, b0); + put_unaligned_le64(o1, b1); + put_unaligned_le64(o2, b2); + put_unaligned_le64(o3, b3); +} + +static __always_inline void format_fcontract(u8 *output, u64 *input) +{ + format_fcontract_first_carry_full(input); + format_fcontract_second_carry_full(input); + format_fcontract_trim(input); + format_fcontract_store(output, input); +} + +static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point) +{ + u64 *x = point; + u64 *z = point + 5; + u64 buf[10] __aligned(32) = { 0 }; + u64 *zmone = buf; + u64 *sc = buf + 5; + crecip(zmone, z); + fmul(sc, x, zmone); + format_fcontract(scalar, sc); +} + +static void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + u64 buf0[10] __aligned(32) = { 0 }; + u64 *x0 = buf0; + u64 *z = buf0 + 5; + u64 *q; + format_fexpand(x0, basepoint); + z[0] = 1; + q = buf0; + { + u8 e[32] __aligned(32) = { 0 }; + u8 *scalar; + memcpy(e, secret, 32); + curve25519_clamp_secret(e); + scalar = e; + { + u64 buf[15] = { 0 }; + u64 *nq = buf; + u64 *x = nq; + x[0] = 1; + ladder_cmult(nq, scalar, q); + format_scalar_of_point(mypublic, nq); + memzero_explicit(buf, sizeof(buf)); + } + memzero_explicit(e, sizeof(e)); + } + memzero_explicit(buf0, sizeof(buf0)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c new file mode 100644 index 0000000000000..d62bd37b007f9 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +#include "curve25519-x86_64.c" + +static bool curve25519_use_bmi2 __ro_after_init; +static bool curve25519_use_adx __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { + &curve25519_use_bmi2, &curve25519_use_adx }; + +static void __init curve25519_fpu_init(void) +{ + curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) && + boot_cpu_has(X86_FEATURE_BMI2); + curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) && + boot_cpu_has(X86_FEATURE_BMI2) && + boot_cpu_has(X86_FEATURE_ADX); +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx(mypublic, secret, basepoint); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2(mypublic, secret, basepoint); + return true; + } + return false; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx_base(pub, secret); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2_base(pub, secret); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c new file mode 100644 index 0000000000000..e8af3e2e30f82 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c @@ -0,0 +1,2366 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright (c) 2017 Armando Faz . All Rights Reserved. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2018 Samuel Neves . All Rights Reserved. + */ + +enum { NUM_WORDS_ELTFP25519 = 4 }; +typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519]; +typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519]; + +#define mul_eltfp25519_1w_adx(c, a, b) do { \ + mul_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_1w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_1w_bmi2(c, a, b) do { \ + mul_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_1w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_adx(a) do { \ + sqr_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_1w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_bmi2(a) do { \ + sqr_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_1w_bmi2(a, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_adx(c, a, b) do { \ + mul2_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_2w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_bmi2(c, a, b) do { \ + mul2_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_2w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_adx(a) do { \ + sqr2_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_2w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_bmi2(a) do { \ + sqr2_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_2w_bmi2(a, m.buffer); \ +} while (0) + +#define sqrn_eltfp25519_1w_adx(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_adx(a); \ +} while (0) + +#define sqrn_eltfp25519_1w_bmi2(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_bmi2(a); \ +} while (0) + +#define copy_eltfp25519_1w(C, A) do { \ + (C)[0] = (A)[0]; \ + (C)[1] = (A)[1]; \ + (C)[2] = (A)[2]; \ + (C)[3] = (A)[3]; \ +} while (0) + +#define setzero_eltfp25519_1w(C) do { \ + (C)[0] = 0; \ + (C)[1] = 0; \ + (C)[2] = 0; \ + (C)[3] = 0; \ +} while (0) + +__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { + /* 1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL, + 0xffffffffffffffffUL, 0x5fffffffffffffffUL, + /* 2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL, + 0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL, + /* 3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL, + 0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL, + /* 4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL, + 0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL, + /* 5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL, + 0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL, + /* 6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL, + 0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL, + /* 7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL, + 0xc1c20d06231f7614UL, 0x2938218da274f972UL, + /* 8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL, + 0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL, + /* 9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL, + 0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL, + /* 10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL, + 0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL, + /* 11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL, + 0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL, + /* 12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL, + 0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL, + /* 13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL, + 0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL, + /* 14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL, + 0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL, + /* 15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL, + 0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL, + /* 16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL, + 0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL, + /* 17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL, + 0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL, + /* 18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL, + 0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL, + /* 19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL, + 0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL, + /* 20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL, + 0x9d4935467caaf22eUL, 0x5166408eee85ff49UL, + /* 21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL, + 0x5259729241159b1cUL, 0x6a621892d5b0ab33UL, + /* 22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL, + 0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL, + /* 23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL, + 0x23758739f630a257UL, 0x295a407a01a78580UL, + /* 24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL, + 0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL, + /* 25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL, + 0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL, + /* 26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL, + 0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL, + /* 27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL, + 0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL, + /* 28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL, + 0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL, + /* 29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL, + 0x74b4c4ceab102f64UL, 0x183abadd10139845UL, + /* 30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL, + 0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL, + /* 31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL, + 0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL, + /* 32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL, + 0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL, + /* 33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL, + 0xd88768e4904032d8UL, 0x131384427b3aaeecUL, + /* 34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL, + 0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL, + /* 35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL, + 0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL, + /* 36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL, + 0xa401760b882c797aUL, 0x1fc223e28dc88730UL, + /* 37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL, + 0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL, + /* 38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL, + 0xfdbf177988bbc586UL, 0x2959894fcad81df5UL, + /* 39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL, + 0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL, + /* 40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL, + 0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL, + /* 41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL, + 0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL, + /* 42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL, + 0x5c217736fa279374UL, 0x7dde05734afeb1faUL, + /* 43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL, + 0xe6053bf89595bf7aUL, 0x394faf38da245530UL, + /* 44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL, + 0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL, + /* 45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL, + 0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL, + /* 46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL, + 0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL, + /* 47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL, + 0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL, + /* 48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL, + 0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL, + /* 49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL, + 0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL, + /* 50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL, + 0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL, + /* 51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL, + 0xc189218075e91436UL, 0x6d9284169b3b8484UL, + /* 52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL, + 0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL, + /* 53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL, + 0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL, + /* 54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL, + 0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL, + /* 55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL, + 0x19346a65d3224a08UL, 0x0f5034e49b9af466UL, + /* 56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL, + 0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL, + /* 57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL, + 0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL, + /* 58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL, + 0xf826842130f5ad28UL, 0x3ea988f75301a441UL, + /* 59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL, + 0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL, + /* 60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL, + 0xd01469df811d644bUL, 0x77fea47d81a5d71fUL, + /* 61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL, + 0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL, + /* 62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL, + 0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL, + /* 63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL, + 0xbea450e1dbd885d5UL, 0x61b68649320f712cUL, + /* 64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL, + 0x25232973322dbef4UL, 0x445dc4758c17f770UL, + /* 65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL, + 0x1efebefdc053db34UL, 0x4adbe867c65daf99UL, + /* 66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL, + 0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL, + /* 67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL, + 0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL, + /* 68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL, + 0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL, + /* 69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL, + 0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL, + /* 70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL, + 0xbdaacb805831ca6fUL, 0x445b652dc916694fUL, + /* 71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL, + 0xa1823aafe04c314aUL, 0x790a2d94437cf586UL, + /* 72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL, + 0xbf70903b204f5169UL, 0x2f7a89891ba319feUL, + /* 73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL, + 0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL, + /* 74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL, + 0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL, + /* 75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL, + 0x674f1288f8e11217UL, 0x5682250f329f93d0UL, + /* 76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL, + 0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL, + /* 77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL, + 0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL, + /* 78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL, + 0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL, + /* 79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL, + 0x5deadacec9f04973UL, 0x29275b5d41d29b27UL, + /* 80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL, + 0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL, + /* 81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL, + 0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL, + /* 82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL, + 0x894d1d855ae52359UL, 0x68e122157b743d69UL, + /* 83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL, + 0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL, + /* 84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL, + 0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL, + /* 85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL, + 0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL, + /* 86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL, + 0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL, + /* 87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL, + 0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL, + /* 88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL, + 0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL, + /* 89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL, + 0x45adb16e76cefcf2UL, 0x01f768aead232999UL, + /* 90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL, + 0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL, + /* 91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL, + 0x5eefa966de2a701dUL, 0x23b20565de55e3efUL, + /* 92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL, + 0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL, + /* 93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL, + 0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL, + /* 94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL, + 0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL, + /* 95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL, + 0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL, + /* 96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL, + 0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL, + /* 97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL, + 0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL, + /* 98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL, + 0xb9886314844006b1UL, 0x40d2a72ab454cc60UL, + /* 99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL, + 0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL, + /* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL, + 0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL, + /* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL, + 0x40727064c416d74fUL, 0x6e15c6114b502ef0UL, + /* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL, + 0x4a497962066e6043UL, 0x705b3aab41355b44UL, + /* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL, + 0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL, + /* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL, + 0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL, + /* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL, + 0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL, + /* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL, + 0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL, + /* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL, + 0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL, + /* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL, + 0x2088ce1570033c68UL, 0x7fba1f495c837987UL, + /* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL, + 0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL, + /* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL, + 0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL, + /* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL, + 0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL, + /* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL, + 0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL, + /* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL, + 0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL, + /* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL, + 0x00f52e3f67280294UL, 0x566d4fc14730c509UL, + /* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL, + 0x216730fba68d6095UL, 0x22e8c3843f69cea7UL, + /* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL, + 0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL, + /* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL, + 0x508e862f121692fcUL, 0x3a81907fa093c291UL, + /* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL, + 0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL, + /* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL, + 0xbee595ce8a9df2e5UL, 0x25e496c722422236UL, + /* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL, + 0xe488de11d761e352UL, 0x0e878a01a085545cUL, + /* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL, + 0x9ea37a487ae80d67UL, 0x67a9958011e41794UL, + /* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL, + 0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL, + /* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL, + 0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL, + /* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL, + 0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL, + /* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL, + 0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL, + /* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL, + 0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL, + /* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL, + 0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL, + /* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL, + 0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL, + /* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL, + 0x97134556a9832d06UL, 0x269bb0360a84f8a0UL, + /* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL, + 0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL, + /* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL, + 0x904659bb686e3772UL, 0x7215c371746ba8c8UL, + /* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL, + 0x266fd5809208f294UL, 0x5c847085619a26b9UL, + /* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL, + 0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL, + /* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL, + 0x2b1641917b307614UL, 0x117c554fc4f45b7cUL, + /* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL, + 0xd7e803f4171b2827UL, 0x1015e87487d225eaUL, + /* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL, + 0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL, + /* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL, + 0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL, + /* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL, + 0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL, + /* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL, + 0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL, + /* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL, + 0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL, + /* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL, + 0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL, + /* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL, + 0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL, + /* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL, + 0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL, + /* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL, + 0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL, + /* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL, + 0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL, + /* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL, + 0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL, + /* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL, + 0xec219c48fbd21604UL, 0x1aaf1af517c36731UL, + /* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL, + 0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL, + /* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL, + 0x4dbbc207f531561aUL, 0x0253b7f082128a27UL, + /* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL, + 0x52d17436309d4253UL, 0x356f97e13efae576UL, + /* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL, + 0x0c776128bed92c98UL, 0x1d34ae93032885b8UL, + /* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL, + 0x66124c6f97bda770UL, 0x0f81a0290654124aUL, + /* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL, + 0xff08d03f93d8c20aUL, 0x52a148199faef26bUL, + /* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL, + 0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL, + /* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL, + 0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL, + /* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL, + 0x5da643cb4bf30035UL, 0x77db28d63940f721UL, + /* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL, + 0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL, + /* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL, + 0x140a69245ca575edUL, 0x0cf1c37134273a4cUL, + /* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL, + 0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL, + /* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL, + 0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL, + /* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL, + 0x497d723f802e88e1UL, 0x30684dea602f408dUL, + /* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL, + 0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL, + /* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL, + 0x7468c7423a543258UL, 0x4a7f11464eb5642fUL, + /* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL, + 0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL, + /* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL, + 0x026df551dbb85c20UL, 0x74fcd91047e21901UL, + /* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL, + 0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL, + /* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL, + 0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL, + /* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL, + 0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL, + /* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL, + 0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL, + /* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL, + 0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL, + /* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL, + 0x13033ac001f66697UL, 0x273b24fe3b367d75UL, + /* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL, + 0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL, + /* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL, + 0xacc63ca34b8ec145UL, 0x74621888fee66574UL, + /* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL, + 0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL, + /* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL, + 0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL, + /* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL, + 0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL, + /* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL, + 0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL, + /* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL, + 0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL, + /* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL, + 0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL, + /* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL, + 0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL, + /* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL, + 0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL, + /* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL, + 0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL, + /* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL, + 0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL, + /* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL, + 0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL, + /* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL, + 0x81004b71e33cc191UL, 0x44e6be345122803cUL, + /* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL, + 0x49c8c4281af60c29UL, 0x21edb518de701aeeUL, + /* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL, + 0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL, + /* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL, + 0x12bc8d6915783712UL, 0x498194c0fc620abbUL, + /* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL, + 0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL, + /* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL, + 0x1e60c24598c71fffUL, 0x59f2f014979983efUL, + /* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL, + 0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL, + /* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL, + 0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL, + /* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL, + 0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL, + /* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL, + 0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL, + /* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL, + 0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL, + /* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL, + 0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL, + /* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL, + 0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL, + /* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL, + 0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL, + /* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL, + 0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL, + /* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL, + 0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL, + /* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL, + 0x883ada83a6a1652cUL, 0x585f1974034d6c17UL, + /* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL, + 0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL, + /* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL, + 0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL, + /* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL, + 0x33979624f0e917beUL, 0x2c018dc527356b30UL, + /* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL, + 0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL, + /* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL, + 0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL, + /* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL, + 0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL, + /* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL, + 0x345ead5e972d091eUL, 0x18c8df11a83103baUL, + /* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL, + 0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL, + /* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL, + 0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL, + /* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL, + 0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL, + /* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL, + 0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL, + /* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL, + 0x79952a008221e738UL, 0x4322e1a7535cd2bbUL, + /* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL, + 0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL, + /* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL, + 0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL, + /* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL, + 0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL, + /* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL, + 0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL, + /* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL, + 0x1df4c0af01314a60UL, 0x09a62dab89289527UL, + /* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL, + 0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL, + /* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL, + 0x49b96853d7a7084aUL, 0x4980a319601420a8UL, + /* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL, + 0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL, + /* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL, + 0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL, + /* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL, + 0xddeb34a061615d99UL, 0x5129cecceb64b773UL, + /* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL, + 0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL, + /* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL, + 0x680bd77c73edad2eUL, 0x487c02354edd9041UL, + /* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL, + 0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL, + /* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL, + 0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL, + /* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL, + 0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL, + /* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL, + 0xe9834262d13921edUL, 0x27fedafaa54bb592UL, + /* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL, + 0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL, + /* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL, + 0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL, + /* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL, + 0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL, + /* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL, + 0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL, + /* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL, + 0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL, + /* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL, + 0x645b426f3d1d58acUL, 0x4804a82227a557bcUL, + /* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL, + 0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL, + /* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL, + 0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL, + /* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL, + 0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL, + /* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL, + 0xc26ccff352b37ec7UL, 0x056f68341d797b21UL, + /* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL, + 0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL, + /* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL, + 0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL, + /* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL, + 0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL, + /* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL, + 0x2c43ecea0107c1ddUL, 0x526028809372de35UL, + /* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL, + 0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL, + /* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL, + 0x899fc38fc4b5c515UL, 0x250386b124ffc207UL, + /* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL, + 0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL, + /* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL, + 0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL, + /* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL, + 0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL, + /* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL, + 0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL, + /* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL, + 0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL, + /* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL, + 0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL, + /* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL, + 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL +}; + +#ifdef CONFIG_AS_ADX +/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] + * a is two 256-bit integers: a0[0:3] and a1[4:7] + * b is two 256-bit integers: b0[0:3] and b1[4:7] + */ +static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "xorl %%r14d, %%r14d ;" + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "adox %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adox %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, 64(%0);" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "adox %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adox %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 72(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 80(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 88(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "movq %%r8, 64(%0) ;" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "addq %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adcq %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 72(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 80(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 88(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 40(%1), %%r8, %%r14 ;" /* B[1]*B[0] */ + "xorl %%r15d, %%r15d;" + "mulx 48(%1), %%r9, %%r10 ;" /* B[2]*B[0] */ + "adcx %%r14, %%r9 ;" + "mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */ + "adcx %%rax, %%r10 ;" + "movq 56(%1), %%rdx ;" /* B[3] */ + "mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */ + "adcx %%rax, %%rbx ;" + "movq 40(%1), %%rdx ;" /* B[1] */ + "adcx %%r15, %%r13 ;" + "mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq 32(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + "movq 40(%1), %%rdx ;" /* B[1] */ + "mulx 32(%1), %%r8, %%r9 ;" /* B[0]*B[1] */ + "mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */ + "mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */ + + "movq 48(%1), %%rdx ;" /* B[2] */ + "mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */ + "mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox 64(%1), %%r8 ;" + "mulx 104(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 72(%1), %%r9 ;" + "mulx 112(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 80(%1), %%r10 ;" + "mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 88(%1), %%r11 ;" + /****************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 104(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /****************************************/ + "adcq $0, %%rcx ;" + "addq 64(%1), %%r8 ;" + "adcq 72(%1), %%r9 ;" + "adcq 80(%1), %%r10 ;" + "adcq 88(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r9; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[0]*B[1] */ + "adox %%r9, %%r10 ;" + "movq %%r10, 8(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */ + "adox %%r11, %%r15 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */ + "adox %%r13, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 8(%0), %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 16(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 16(%0), %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 24(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 24(%0), %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 32(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq %%r14, 48(%0) ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + "movq %%rax, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static __always_inline void +add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "xorl %%ecx, %%ecx ;" + "movq (%2), %%r8 ;" + "adcx (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcx 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcx 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcx 24(%1), %%r11 ;" + "cmovc %%eax, %%ecx ;" + "xorl %%eax, %%eax ;" + "adcx %%rcx, %%r8 ;" + "adcx %%rax, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rax, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rax, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $38, %%ecx ;" + "cmovc %%ecx, %%eax ;" + "addq %%rax, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static __always_inline void +add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%2), %%r8 ;" + "addq (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcq 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcq 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcq 24(%1), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +static __always_inline void +sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%1), %%r8 ;" + "subq (%2), %%r8 ;" + "movq 8(%1), %%r9 ;" + "sbbq 8(%2), %%r9 ;" + "movq 16(%1), %%r10 ;" + "sbbq 16(%2), %%r10 ;" + "movq 24(%1), %%r11 ;" + "sbbq 24(%2), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "sbbq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "sbbq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "sbbq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} + +/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */ +static __always_inline void +mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) +{ + const u64 a24 = 121666; + asm volatile( + "movq %2, %%rdx ;" + "mulx (%1), %%r8, %%r10 ;" + "mulx 8(%1), %%r9, %%r11 ;" + "addq %%r10, %%r9 ;" + "mulx 16(%1), %%r10, %%rax ;" + "adcq %%r11, %%r10 ;" + "mulx 24(%1), %%r11, %%rcx ;" + "adcq %%rax, %%r11 ;" + /**************************/ + "adcq $0, %%rcx ;" + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/ + "imul %%rdx, %%rcx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(a24) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} + +#ifdef CONFIG_AS_ADX +static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[4]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_adx(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 2); + mul_eltfp25519_1w_adx(T[0], a, T[2]); + mul_eltfp25519_1w_adx(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 1); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 10); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 20); + mul_eltfp25519_1w_adx(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 10); + mul_eltfp25519_1w_adx(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_adx(T[0], 50); + mul_eltfp25519_1w_adx(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 100); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 50); + mul_eltfp25519_1w_adx(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[5]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_bmi2(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 2); + mul_eltfp25519_1w_bmi2(T[0], a, T[2]); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 1); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 10); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 20); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 10); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_bmi2(T[0], 50); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 100); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 50); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +/* Given c, a 256-bit number, fred_eltfp25519_1w updates c + * with a number such that 0 <= C < 2**255-19. + */ +static __always_inline void fred_eltfp25519_1w(u64 *const c) +{ + u64 tmp0 = 38, tmp1 = 19; + asm volatile( + "btrq $63, %3 ;" /* Put bit 255 in carry flag and clear */ + "cmovncl %k5, %k4 ;" /* c[255] ? 38 : 19 */ + + /* Add either 19 or 38 to c */ + "addq %4, %0 ;" + "adcq $0, %1 ;" + "adcq $0, %2 ;" + "adcq $0, %3 ;" + + /* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */ + "movl $0, %k4 ;" + "cmovnsl %k5, %k4 ;" /* c[255] ? 0 : 19 */ + "btrq $63, %3 ;" /* Clear bit 255 */ + + /* Subtract 19 if necessary */ + "subq %4, %0 ;" + "sbbq $0, %1 ;" + "sbbq $0, %2 ;" + "sbbq $0, %3 ;" + + : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0), + "+r"(tmp1) + : + : "memory", "cc"); +} + +static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py) +{ + u64 temp; + asm volatile( + "test %9, %9 ;" + "movq %0, %8 ;" + "cmovnzq %4, %0 ;" + "cmovnzq %8, %4 ;" + "movq %1, %8 ;" + "cmovnzq %5, %1 ;" + "cmovnzq %8, %5 ;" + "movq %2, %8 ;" + "cmovnzq %6, %2 ;" + "cmovnzq %8, %6 ;" + "movq %3, %8 ;" + "cmovnzq %7, %3 ;" + "cmovnzq %8, %7 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]), + "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]), + "=r"(temp) + : "r"(bit) + : "cc" + ); +} + +static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py) +{ + asm volatile( + "test %4, %4 ;" + "cmovnzq %5, %0 ;" + "cmovnzq %6, %1 ;" + "cmovnzq %7, %2 ;" + "cmovnzq %8, %3 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]) + : "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3]) + : "cc" + ); +} + +static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_adx(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_adx(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_adx(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_adx(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_adx(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_adx(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_adx(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_adx(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_adx(A, Qz); + mul_eltfp25519_1w_adx((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_adx(C, &P[4 * k], B); /* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_adx(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_adx(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_adx(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_adx(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_adx(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_adx(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_adx(A, Zr1); + mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_bmi2(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_bmi2(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_bmi2(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_bmi2(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_bmi2(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_bmi2(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_bmi2(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_bmi2(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_bmi2(A, Qz); + mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_bmi2(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_bmi2(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_bmi2(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_bmi2(A, Zr1); + mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519.c b/net/wireguard/crypto/zinc/curve25519/curve25519.c new file mode 100644 index 0000000000000..82c669e4123c4 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the Curve25519 ECDH algorithm, using either + * a 32-bit implementation or a 64-bit implementation with 128-bit integers, + * depending on what is supported by the target compiler. + * + * Information: https://cr.yp.to/ecdh.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include // For crypto_memneq. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "curve25519-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) +#include "curve25519-arm-glue.c" +#else +static bool *const curve25519_nobs[] __initconst = { }; +static void __init curve25519_fpu_init(void) +{ +} +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + return false; +} +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "curve25519-hacl64.c" +#else +#include "curve25519-fiat32.c" +#endif + +static const u8 null_point[CURVE25519_KEY_SIZE] = { 0 }; + +bool curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (!curve25519_arch(mypublic, secret, basepoint)) + curve25519_generic(mypublic, secret, basepoint); + return crypto_memneq(mypublic, null_point, CURVE25519_KEY_SIZE); +} +EXPORT_SYMBOL(curve25519); + +bool curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + static const u8 basepoint[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; + + if (unlikely(!crypto_memneq(secret, null_point, CURVE25519_KEY_SIZE))) + return false; + + if (curve25519_base_arch(pub, secret)) + return crypto_memneq(pub, null_point, CURVE25519_KEY_SIZE); + return curve25519(pub, secret, basepoint); +} +EXPORT_SYMBOL(curve25519_generate_public); + +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + get_random_bytes_wait(secret, CURVE25519_KEY_SIZE); + curve25519_clamp_secret(secret); +} +EXPORT_SYMBOL(curve25519_generate_secret); + +#include "../selftest/curve25519.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init curve25519_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + curve25519_fpu_init(); + if (!selftest_run("curve25519", curve25519_selftest, curve25519_nobs, + ARRAY_SIZE(curve25519_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c new file mode 100644 index 0000000000000..291fe4ba98b0c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); +asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool poly1305_use_neon __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { &poly1305_use_neon }; + +static void __init poly1305_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + poly1305_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + poly1305_use_neon = elf_hwcap & HWCAP_NEON; +#endif +} + +#if defined(CONFIG_ZINC_ARCH_ARM64) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1, h2; + }; + }; + u64 is_base2_26; + u64 r[2]; +}; +#elif defined(CONFIG_ZINC_ARCH_ARM) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1; + u32 h2; + } __packed; + }; + u32 r[4]; + u32 is_base2_26; +}; +#endif + +/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit + * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON + * and then having to go back to scalar -- because the user is silly and has + * called the update function from two separate contexts -- then we need to + * convert back to the original base before proceeding. The below function is + * written for 64-bit integers, and so we have to swap words at the end on + * big-endian 32-bit. It is possible to reason that the initial reduction below + * is sufficient given the implementation invariants. However, for an avoidance + * of doubt and because this is not performance critical, we do the full + * reduction anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->h2 = state->h[4] >> 24; + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) { + state->h0 = rol64(state->h0, 32); + state->h1 = rol64(state->h1, 32); + } +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->h2 >> 2) + (state->h2 & ~3ULL); + state->h2 &= 3; + state->h0 += cy; + state->h1 += (cy = ULT(state->h0, cy)); + state->h2 += ULT(state->h1, cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_arm(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_arm(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + poly1305_blocks_neon(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_arm(ctx, mac, nonce); + } else + poly1305_emit_neon(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl new file mode 100644 index 0000000000000..468f41b76fbd2 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl @@ -0,0 +1,1276 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# IALU(*)/gcc-4.4 NEON +# +# ARM11xx(ARMv6) 7.78/+100% - +# Cortex-A5 6.35/+130% 3.00 +# Cortex-A8 6.25/+115% 2.36 +# Cortex-A9 5.10/+95% 2.55 +# Cortex-A15 3.85/+85% 1.25(**) +# Snapdragon S4 5.70/+100% 1.48(**) +# +# (*) this is for -march=armv6, i.e. with bunch of ldrb loading data; +# (**) these are trade-off results, they can be improved by ~8% but at +# the cost of 15/12% regression on Cortex-A5/A7, it's even possible +# to improve Cortex-A9 result, but then A5/A7 loose more than 20%; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +($ctx,$inp,$len,$padbit)=map("r$_",(0..3)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text +#if defined(__thumb2__) +.syntax unified +.thumb +#else +.code 32 +#endif + +.globl poly1305_emit +.globl poly1305_blocks +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: +.Lpoly1305_init: + stmdb sp!,{r4-r11} + + eor r3,r3,r3 + cmp $inp,#0 + str r3,[$ctx,#0] @ zero hash value + str r3,[$ctx,#4] + str r3,[$ctx,#8] + str r3,[$ctx,#12] + str r3,[$ctx,#16] + str r3,[$ctx,#36] @ is_base2_26 + add $ctx,$ctx,#20 + +#ifdef __thumb2__ + it eq +#endif + moveq r0,#0 + beq .Lno_key + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + adr r11,.Lpoly1305_init + ldr r12,.LOPENSSL_armcap +#endif + ldrb r4,[$inp,#0] + mov r10,#0x0fffffff + ldrb r5,[$inp,#1] + and r3,r10,#-4 @ 0x0ffffffc + ldrb r6,[$inp,#2] + ldrb r7,[$inp,#3] + orr r4,r4,r5,lsl#8 + ldrb r5,[$inp,#4] + orr r4,r4,r6,lsl#16 + ldrb r6,[$inp,#5] + orr r4,r4,r7,lsl#24 + ldrb r7,[$inp,#6] + and r4,r4,r10 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,[r11,r12] @ OPENSSL_armcap_P +# ifdef __APPLE__ + ldr r12,[r12] +# endif +#endif + ldrb r8,[$inp,#7] + orr r5,r5,r6,lsl#8 + ldrb r6,[$inp,#8] + orr r5,r5,r7,lsl#16 + ldrb r7,[$inp,#9] + orr r5,r5,r8,lsl#24 + ldrb r8,[$inp,#10] + and r5,r5,r3 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + tst r12,#ARMV7_NEON @ check for NEON +# ifdef __APPLE__ + adr r9,poly1305_blocks_neon + adr r11,poly1305_blocks +# ifdef __thumb2__ + it ne +# endif + movne r11,r9 + adr r12,poly1305_emit + adr r10,poly1305_emit_neon +# ifdef __thumb2__ + it ne +# endif + movne r12,r10 +# else +# ifdef __thumb2__ + itete eq +# endif + addeq r12,r11,#(poly1305_emit-.Lpoly1305_init) + addne r12,r11,#(poly1305_emit_neon-.Lpoly1305_init) + addeq r11,r11,#(poly1305_blocks-.Lpoly1305_init) + addne r11,r11,#(poly1305_blocks_neon-.Lpoly1305_init) +# endif +# ifdef __thumb2__ + orr r12,r12,#1 @ thumb-ify address + orr r11,r11,#1 +# endif +#endif + ldrb r9,[$inp,#11] + orr r6,r6,r7,lsl#8 + ldrb r7,[$inp,#12] + orr r6,r6,r8,lsl#16 + ldrb r8,[$inp,#13] + orr r6,r6,r9,lsl#24 + ldrb r9,[$inp,#14] + and r6,r6,r3 + + ldrb r10,[$inp,#15] + orr r7,r7,r8,lsl#8 + str r4,[$ctx,#0] + orr r7,r7,r9,lsl#16 + str r5,[$ctx,#4] + orr r7,r7,r10,lsl#24 + str r6,[$ctx,#8] + and r7,r7,r3 + str r7,[$ctx,#12] +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + stmia r2,{r11,r12} @ fill functions table + mov r0,#1 +#else + mov r0,#0 +#endif +.Lno_key: + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_init,.-poly1305_init +___ +{ +my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12)); +my ($s1,$s2,$s3)=($r1,$r2,$r3); + +$code.=<<___; +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: +.Lpoly1305_blocks: + stmdb sp!,{r3-r11,lr} + + ands $len,$len,#-16 + beq .Lno_data + + cmp $padbit,#0 + add $len,$len,$inp @ end pointer + sub sp,sp,#32 + + ldmia $ctx,{$h0-$r3} @ load context + + str $ctx,[sp,#12] @ offload stuff + mov lr,$inp + str $len,[sp,#16] + str $r1,[sp,#20] + str $r2,[sp,#24] + str $r3,[sp,#28] + b .Loop + +.Loop: +#if __ARM_ARCH__<7 + ldrb r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ 1<<128 + ldrb r1,[lr,#-15] + ldrb r2,[lr,#-14] + ldrb r3,[lr,#-13] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-12] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-11] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-10] + adds $h0,$h0,r3 @ accumulate input + + ldrb r3,[lr,#-9] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-8] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-7] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-6] + adcs $h1,$h1,r3 + + ldrb r3,[lr,#-5] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-4] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-3] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-2] + adcs $h2,$h2,r3 + + ldrb r3,[lr,#-1] + orr r1,r0,r1,lsl#8 + str lr,[sp,#8] @ offload input pointer + orr r2,r1,r2,lsl#16 + add $s1,$r1,$r1,lsr#2 + orr r3,r2,r3,lsl#24 +#else + ldr r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ padbit + ldr r1,[lr,#-12] + ldr r2,[lr,#-8] + ldr r3,[lr,#-4] +# ifdef __ARMEB__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +# endif + adds $h0,$h0,r0 @ accumulate input + str lr,[sp,#8] @ offload input pointer + adcs $h1,$h1,r1 + add $s1,$r1,$r1,lsr#2 + adcs $h2,$h2,r2 +#endif + add $s2,$r2,$r2,lsr#2 + adcs $h3,$h3,r3 + add $s3,$r3,$r3,lsr#2 + + umull r2,r3,$h1,$r0 + adc $h4,$h4,#0 + umull r0,r1,$h0,$r0 + umlal r2,r3,$h4,$s1 + umlal r0,r1,$h3,$s1 + ldr $r1,[sp,#20] @ reload $r1 + umlal r2,r3,$h2,$s3 + umlal r0,r1,$h1,$s3 + umlal r2,r3,$h3,$s2 + umlal r0,r1,$h2,$s2 + umlal r2,r3,$h0,$r1 + str r0,[sp,#0] @ future $h0 + mul r0,$s2,$h4 + ldr $r2,[sp,#24] @ reload $r2 + adds r2,r2,r1 @ d1+=d0>>32 + eor r1,r1,r1 + adc lr,r3,#0 @ future $h2 + str r2,[sp,#4] @ future $h1 + + mul r2,$s3,$h4 + eor r3,r3,r3 + umlal r0,r1,$h3,$s3 + ldr $r3,[sp,#28] @ reload $r3 + umlal r2,r3,$h3,$r0 + umlal r0,r1,$h2,$r0 + umlal r2,r3,$h2,$r1 + umlal r0,r1,$h1,$r1 + umlal r2,r3,$h1,$r2 + umlal r0,r1,$h0,$r2 + umlal r2,r3,$h0,$r3 + ldr $h0,[sp,#0] + mul $h4,$r0,$h4 + ldr $h1,[sp,#4] + + adds $h2,lr,r0 @ d2+=d1>>32 + ldr lr,[sp,#8] @ reload input pointer + adc r1,r1,#0 + adds $h3,r2,r1 @ d3+=d2>>32 + ldr r0,[sp,#16] @ reload end pointer + adc r3,r3,#0 + add $h4,$h4,r3 @ h4+=d3>>32 + + and r1,$h4,#-4 + and $h4,$h4,#3 + add r1,r1,r1,lsr#2 @ *=5 + adds $h0,$h0,r1 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + cmp r0,lr @ done yet? + bhi .Loop + + ldr $ctx,[sp,#12] + add sp,sp,#32 + stmia $ctx,{$h0-$h4} @ store the result + +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r3-r11,pc} +#else + ldmia sp!,{r3-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_blocks,.-poly1305_blocks +___ +} +{ +my ($ctx,$mac,$nonce)=map("r$_",(0..2)); +my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11)); +my $g4=$h4; + +$code.=<<___; +.type poly1305_emit,%function +.align 5 +poly1305_emit: + stmdb sp!,{r4-r11} +.Lpoly1305_emit_enter: + + ldmia $ctx,{$h0-$h4} + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + +#ifdef __thumb2__ + it ne +#endif + movne $h0,$g0 + ldr $g0,[$nonce,#0] +#ifdef __thumb2__ + it ne +#endif + movne $h1,$g1 + ldr $g1,[$nonce,#4] +#ifdef __thumb2__ + it ne +#endif + movne $h2,$g2 + ldr $g2,[$nonce,#8] +#ifdef __thumb2__ + it ne +#endif + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +#if __ARM_ARCH__>=7 +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] +#else + strb $h0,[$mac,#0] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#4] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#8] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#12] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#1] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#5] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#9] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#13] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#2] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#6] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#10] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#14] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#3] + strb $h1,[$mac,#7] + strb $h2,[$mac,#11] + strb $h3,[$mac,#15] +#endif + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_emit,.-poly1305_emit +___ +{ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9)); +my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14)); +my ($T0,$T1,$MASK) = map("q$_",(15,4,0)); + +my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7)); + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.fpu neon + +.type poly1305_init_neon,%function +.align 5 +poly1305_init_neon: +.Lpoly1305_init_neon: + ldr r4,[$ctx,#20] @ load key base 2^32 + ldr r5,[$ctx,#24] + ldr r6,[$ctx,#28] + ldr r7,[$ctx,#32] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + and r3,r3,#0x03ffffff + and r4,r4,#0x03ffffff + and r5,r5,#0x03ffffff + + vdup.32 $R0,r2 @ r^1 in both lanes + add r2,r3,r3,lsl#2 @ *5 + vdup.32 $R1,r3 + add r3,r4,r4,lsl#2 + vdup.32 $S1,r2 + vdup.32 $R2,r4 + add r4,r5,r5,lsl#2 + vdup.32 $S2,r3 + vdup.32 $R3,r5 + add r5,r6,r6,lsl#2 + vdup.32 $S3,r4 + vdup.32 $R4,r6 + vdup.32 $S4,r5 + + mov $zeros,#2 @ counter + +.Lsquare_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + + vmull.u32 $D0,$R0,${R0}[1] + vmull.u32 $D1,$R1,${R0}[1] + vmull.u32 $D2,$R2,${R0}[1] + vmull.u32 $D3,$R3,${R0}[1] + vmull.u32 $D4,$R4,${R0}[1] + + vmlal.u32 $D0,$R4,${S1}[1] + vmlal.u32 $D1,$R0,${R1}[1] + vmlal.u32 $D2,$R1,${R1}[1] + vmlal.u32 $D3,$R2,${R1}[1] + vmlal.u32 $D4,$R3,${R1}[1] + + vmlal.u32 $D0,$R3,${S2}[1] + vmlal.u32 $D1,$R4,${S2}[1] + vmlal.u32 $D3,$R1,${R2}[1] + vmlal.u32 $D2,$R0,${R2}[1] + vmlal.u32 $D4,$R2,${R2}[1] + + vmlal.u32 $D0,$R2,${S3}[1] + vmlal.u32 $D3,$R0,${R3}[1] + vmlal.u32 $D1,$R3,${S3}[1] + vmlal.u32 $D2,$R4,${S3}[1] + vmlal.u32 $D4,$R1,${R3}[1] + + vmlal.u32 $D3,$R4,${S4}[1] + vmlal.u32 $D0,$R1,${S4}[1] + vmlal.u32 $D1,$R2,${S4}[1] + vmlal.u32 $D2,$R3,${S4}[1] + vmlal.u32 $D4,$R0,${R4}[1] + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + @ and P. Schwabe + @ + @ H0>>+H1>>+H2>>+H3>>+H4 + @ H3>>+H4>>*5+H0>>+H1 + @ + @ Trivia. + @ + @ Result of multiplication of n-bit number by m-bit number is + @ n+m bits wide. However! Even though 2^n is a n+1-bit number, + @ m-bit number multiplied by 2^n is still n+m bits wide. + @ + @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2, + @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit + @ one is n+1 bits wide. + @ + @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that + @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4 + @ can be 27. However! In cases when their width exceeds 26 bits + @ they are limited by 2^26+2^6. This in turn means that *sum* + @ of the products with these values can still be viewed as sum + @ of 52-bit numbers as long as the amount of addends is not a + @ power of 2. For example, + @ + @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4, + @ + @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or + @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than + @ 8 * (2^52) or 2^55. However, the value is then multiplied by + @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12), + @ which is less than 32 * (2^52) or 2^57. And when processing + @ data we are looking at triple as many addends... + @ + @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and + @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the + @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while + @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32 + @ instruction accepts 2x32-bit input and writes 2x64-bit result. + @ This means that result of reduction have to be compressed upon + @ loop wrap-around. This can be done in the process of reduction + @ to minimize amount of instructions [as well as amount of + @ 128-bit instructions, which benefits low-end processors], but + @ one has to watch for H2 (which is narrower than H0) and 5*H4 + @ not being wider than 58 bits, so that result of right shift + @ by 26 bits fits in 32 bits. This is also useful on x86, + @ because it allows to use paddd in place for paddq, which + @ benefits Atom, where paddq is ridiculously slow. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vbic.i32 $D4#lo,#0xfc000000 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vbic.i32 $D2#lo,#0xfc000000 + + vshr.u32 $T0#lo,$D0#lo,#26 + vbic.i32 $D0#lo,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + + subs $zeros,$zeros,#1 + beq .Lsquare_break_neon + + add $tbl0,$ctx,#(48+0*9*4) + add $tbl1,$ctx,#(48+1*9*4) + + vtrn.32 $R0,$D0#lo @ r^2:r^1 + vtrn.32 $R2,$D2#lo + vtrn.32 $R3,$D3#lo + vtrn.32 $R1,$D1#lo + vtrn.32 $R4,$D4#lo + + vshl.u32 $S2,$R2,#2 @ *5 + vshl.u32 $S3,$R3,#2 + vshl.u32 $S1,$R1,#2 + vshl.u32 $S4,$R4,#2 + vadd.i32 $S2,$S2,$R2 + vadd.i32 $S1,$S1,$R1 + vadd.i32 $S3,$S3,$R3 + vadd.i32 $S4,$S4,$R4 + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0,:32] + vst1.32 {${S4}[1]},[$tbl1,:32] + + b .Lsquare_neon + +.align 4 +.Lsquare_break_neon: + add $tbl0,$ctx,#(48+2*4*9) + add $tbl1,$ctx,#(48+3*4*9) + + vmov $R0,$D0#lo @ r^4:r^3 + vshl.u32 $S1,$D1#lo,#2 @ *5 + vmov $R1,$D1#lo + vshl.u32 $S2,$D2#lo,#2 + vmov $R2,$D2#lo + vshl.u32 $S3,$D3#lo,#2 + vmov $R3,$D3#lo + vshl.u32 $S4,$D4#lo,#2 + vmov $R4,$D4#lo + vadd.i32 $S1,$S1,$D1#lo + vadd.i32 $S2,$S2,$D2#lo + vadd.i32 $S3,$S3,$D3#lo + vadd.i32 $S4,$S4,$D4#lo + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0] + vst1.32 {${S4}[1]},[$tbl1] + + ret @ bx lr +.size poly1305_init_neon,.-poly1305_init_neon + +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +#endif +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + ands $len,$len,#-16 + beq .Lno_data_neon + + cmp $len,#64 + bhs .Lenter_neon + tst ip,ip @ is_base2_26? + beq .Lpoly1305_blocks + +.Lenter_neon: + stmdb sp!,{r4-r7} + vstmdb sp!,{d8-d15} @ ABI specification says so + + tst ip,ip @ is_base2_26? + bne .Lbase2_26_neon + + stmdb sp!,{r1-r3,lr} + bl .Lpoly1305_init_neon + + ldr r4,[$ctx,#0] @ load hash value base 2^32 + ldr r5,[$ctx,#4] + ldr r6,[$ctx,#8] + ldr r7,[$ctx,#12] + ldr ip,[$ctx,#16] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + veor $D0#lo,$D0#lo,$D0#lo + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + veor $D1#lo,$D1#lo,$D1#lo + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + veor $D2#lo,$D2#lo,$D2#lo + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + veor $D3#lo,$D3#lo,$D3#lo + and r3,r3,#0x03ffffff + orr r6,r6,ip,lsl#24 + veor $D4#lo,$D4#lo,$D4#lo + and r4,r4,#0x03ffffff + mov r1,#1 + and r5,r5,#0x03ffffff + str r1,[$ctx,#36] @ is_base2_26 + + vmov.32 $D0#lo[0],r2 + vmov.32 $D1#lo[0],r3 + vmov.32 $D2#lo[0],r4 + vmov.32 $D3#lo[0],r5 + vmov.32 $D4#lo[0],r6 + adr $zeros,.Lzeros + + ldmia sp!,{r1-r3,lr} + b .Lbase2_32_neon + +.align 4 +.Lbase2_26_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ load hash value + + veor $D0#lo,$D0#lo,$D0#lo + veor $D1#lo,$D1#lo,$D1#lo + veor $D2#lo,$D2#lo,$D2#lo + veor $D3#lo,$D3#lo,$D3#lo + veor $D4#lo,$D4#lo,$D4#lo + vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + adr $zeros,.Lzeros + vld1.32 {$D4#lo[0]},[$ctx] + sub $ctx,$ctx,#16 @ rewind + +.Lbase2_32_neon: + add $in2,$inp,#32 + mov $padbit,$padbit,lsl#24 + tst $len,#31 + beq .Leven + + vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]! + vmov.32 $H4#lo[0],$padbit + sub $len,$len,#16 + add $in2,$inp,#32 + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3#lo,$H3#lo,#18 + + vsri.u32 $H3#lo,$H2#lo,#14 + vshl.u32 $H2#lo,$H2#lo,#12 + vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi + + vbic.i32 $H3#lo,#0xfc000000 + vsri.u32 $H2#lo,$H1#lo,#20 + vshl.u32 $H1#lo,$H1#lo,#6 + + vbic.i32 $H2#lo,#0xfc000000 + vsri.u32 $H1#lo,$H0#lo,#26 + vadd.i32 $H3#hi,$H3#lo,$D3#lo + + vbic.i32 $H0#lo,#0xfc000000 + vbic.i32 $H1#lo,#0xfc000000 + vadd.i32 $H2#hi,$H2#lo,$D2#lo + + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + + mov $tbl1,$zeros + add $tbl0,$ctx,#48 + + cmp $len,$len + b .Long_tail + +.align 4 +.Leven: + subs $len,$len,#64 + it lo + movlo $in2,$zeros + + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + itt hi + addhi $tbl1,$ctx,#(48+1*9*4) + addhi $tbl0,$ctx,#(48+3*9*4) + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3,$H3,#18 + + vsri.u32 $H3,$H2,#14 + vshl.u32 $H2,$H2,#12 + + vbic.i32 $H3,#0xfc000000 + vsri.u32 $H2,$H1,#20 + vshl.u32 $H1,$H1,#6 + + vbic.i32 $H2,#0xfc000000 + vsri.u32 $H1,$H0,#26 + + vbic.i32 $H0,#0xfc000000 + vbic.i32 $H1,#0xfc000000 + + bls .Lskip_loop + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + b .Loop_neon + +.align 5 +.Loop_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + @ \___________________/ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + @ \___________________/ \____________________/ + @ + @ Note that we start with inp[2:3]*r^2. This is because it + @ doesn't depend on reduction in previous iteration. + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ inp[2:3]*r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1] + vmull.u32 $D2,$H2#hi,${R0}[1] + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,${R0}[1] + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,${R0}[1] + vmlal.u32 $D2,$H1#hi,${R1}[1] + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,${R0}[1] + + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,${R0}[1] + subs $len,$len,#64 + vmlal.u32 $D0,$H4#hi,${S1}[1] + it lo + movlo $in2,$zeros + vmlal.u32 $D3,$H2#hi,${R1}[1] + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D1,$H0#hi,${R1}[1] + vmlal.u32 $D4,$H3#hi,${R1}[1] + + vmlal.u32 $D0,$H3#hi,${S2}[1] + vmlal.u32 $D3,$H1#hi,${R2}[1] + vmlal.u32 $D4,$H2#hi,${R2}[1] + vmlal.u32 $D1,$H4#hi,${S2}[1] + vmlal.u32 $D2,$H0#hi,${R2}[1] + + vmlal.u32 $D3,$H0#hi,${R3}[1] + vmlal.u32 $D0,$H2#hi,${S3}[1] + vmlal.u32 $D4,$H1#hi,${R3}[1] + vmlal.u32 $D1,$H3#hi,${S3}[1] + vmlal.u32 $D2,$H4#hi,${S3}[1] + + vmlal.u32 $D3,$H4#hi,${S4}[1] + vmlal.u32 $D0,$H1#hi,${S4}[1] + vmlal.u32 $D4,$H0#hi,${R4}[1] + vmlal.u32 $D1,$H2#hi,${S4}[1] + vmlal.u32 $D2,$H3#hi,${S4}[1] + + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4 and accumulate + + vmlal.u32 $D3,$H3#lo,${R0}[0] + vmlal.u32 $D0,$H0#lo,${R0}[0] + vmlal.u32 $D4,$H4#lo,${R0}[0] + vmlal.u32 $D1,$H1#lo,${R0}[0] + vmlal.u32 $D2,$H2#lo,${R0}[0] + vld1.32 ${S4}[0],[$tbl0,:32] + + vmlal.u32 $D3,$H2#lo,${R1}[0] + vmlal.u32 $D0,$H4#lo,${S1}[0] + vmlal.u32 $D4,$H3#lo,${R1}[0] + vmlal.u32 $D1,$H0#lo,${R1}[0] + vmlal.u32 $D2,$H1#lo,${R1}[0] + + vmlal.u32 $D3,$H1#lo,${R2}[0] + vmlal.u32 $D0,$H3#lo,${S2}[0] + vmlal.u32 $D4,$H2#lo,${R2}[0] + vmlal.u32 $D1,$H4#lo,${S2}[0] + vmlal.u32 $D2,$H0#lo,${R2}[0] + + vmlal.u32 $D3,$H0#lo,${R3}[0] + vmlal.u32 $D0,$H2#lo,${S3}[0] + vmlal.u32 $D4,$H1#lo,${R3}[0] + vmlal.u32 $D1,$H3#lo,${S3}[0] + vmlal.u32 $D3,$H4#lo,${S4}[0] + + vmlal.u32 $D2,$H4#lo,${S3}[0] + vmlal.u32 $D0,$H1#lo,${S4}[0] + vmlal.u32 $D4,$H0#lo,${R4}[0] + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vmlal.u32 $D1,$H2#lo,${S4}[0] + vmlal.u32 $D2,$H3#lo,${S4}[0] + + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 + vrev32.8 $H3,$H3 +# endif + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction interleaved with base 2^32 -> base 2^26 of + @ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vshl.u32 $H3,$H3,#18 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vsri.u32 $H3,$H2,#14 + vbic.i32 $D4#lo,#0xfc000000 + vshl.u32 $H2,$H2,#12 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vbic.i32 $H3,#0xfc000000 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec] + vsri.u32 $H2,$H1,#20 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vshl.u32 $H1,$H1,#6 + vbic.i32 $D2#lo,#0xfc000000 + vbic.i32 $H2,#0xfc000000 + + vshrn.u64 $T0#lo,$D0,#26 @ re-narrow + vmovn.i64 $D0#lo,$D0 + vsri.u32 $H1,$H0,#26 + vbic.i32 $H0,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vbic.i32 $D0#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + vbic.i32 $H1,#0xfc000000 + + bhi .Loop_neon + +.Lskip_loop: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + add $tbl1,$ctx,#(48+0*9*4) + add $tbl0,$ctx,#(48+1*9*4) + adds $len,$len,#32 + it ne + movne $len,#0 + bne .Long_tail + + vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H3#hi,$H3#lo,$D3#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + vadd.i32 $H4#hi,$H4#lo,$D4#lo + +.Long_tail: + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant + vmull.u32 $D2,$H2#hi,$R0 + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,$R0 + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,$R0 + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,$R0 + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,$R0 + + vmlal.u32 $D0,$H4#hi,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#hi,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#hi,$R1 + vmlal.u32 $D4,$H3#hi,$R1 + vmlal.u32 $D2,$H1#hi,$R1 + + vmlal.u32 $D3,$H1#hi,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#hi,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#hi,$R2 + vmlal.u32 $D1,$H4#hi,$S2 + vmlal.u32 $D2,$H0#hi,$R2 + + vmlal.u32 $D3,$H0#hi,$R3 + it ne + addne $tbl1,$ctx,#(48+2*9*4) + vmlal.u32 $D0,$H2#hi,$S3 + it ne + addne $tbl0,$ctx,#(48+3*9*4) + vmlal.u32 $D4,$H1#hi,$R3 + vmlal.u32 $D1,$H3#hi,$S3 + vmlal.u32 $D2,$H4#hi,$S3 + + vmlal.u32 $D3,$H4#hi,$S4 + vorn $MASK,$MASK,$MASK @ all-ones, can be redundant + vmlal.u32 $D0,$H1#hi,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#hi,$R4 + vmlal.u32 $D1,$H2#hi,$S4 + vmlal.u32 $D2,$H3#hi,$S4 + + beq .Lshort_tail + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4:r^3 and accumulate + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + + vmlal.u32 $D2,$H2#lo,$R0 + vmlal.u32 $D0,$H0#lo,$R0 + vmlal.u32 $D3,$H3#lo,$R0 + vmlal.u32 $D1,$H1#lo,$R0 + vmlal.u32 $D4,$H4#lo,$R0 + + vmlal.u32 $D0,$H4#lo,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#lo,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#lo,$R1 + vmlal.u32 $D4,$H3#lo,$R1 + vmlal.u32 $D2,$H1#lo,$R1 + + vmlal.u32 $D3,$H1#lo,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#lo,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#lo,$R2 + vmlal.u32 $D1,$H4#lo,$S2 + vmlal.u32 $D2,$H0#lo,$R2 + + vmlal.u32 $D3,$H0#lo,$R3 + vmlal.u32 $D0,$H2#lo,$S3 + vmlal.u32 $D4,$H1#lo,$R3 + vmlal.u32 $D1,$H3#lo,$S3 + vmlal.u32 $D2,$H4#lo,$S3 + + vmlal.u32 $D3,$H4#lo,$S4 + vorn $MASK,$MASK,$MASK @ all-ones + vmlal.u32 $D0,$H1#lo,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#lo,$R4 + vmlal.u32 $D1,$H2#lo,$S4 + vmlal.u32 $D2,$H3#lo,$S4 + +.Lshort_tail: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ horizontal addition + + vadd.i64 $D3#lo,$D3#lo,$D3#hi + vadd.i64 $D0#lo,$D0#lo,$D0#hi + vadd.i64 $D4#lo,$D4#lo,$D4#hi + vadd.i64 $D1#lo,$D1#lo,$D1#hi + vadd.i64 $D2#lo,$D2#lo,$D2#hi + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction, but without narrowing + + vshr.u64 $T0,$D3,#26 + vand.i64 $D3,$D3,$MASK + vshr.u64 $T1,$D0,#26 + vand.i64 $D0,$D0,$MASK + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + + vshr.u64 $T0,$D4,#26 + vand.i64 $D4,$D4,$MASK + vshr.u64 $T1,$D1,#26 + vand.i64 $D1,$D1,$MASK + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + + vadd.i64 $D0,$D0,$T0 + vshl.u64 $T0,$T0,#2 + vshr.u64 $T1,$D2,#26 + vand.i64 $D2,$D2,$MASK + vadd.i64 $D0,$D0,$T0 @ h4 -> h0 + vadd.i64 $D3,$D3,$T1 @ h2 -> h3 + + vshr.u64 $T0,$D0,#26 + vand.i64 $D0,$D0,$MASK + vshr.u64 $T1,$D3,#26 + vand.i64 $D3,$D3,$MASK + vadd.i64 $D1,$D1,$T0 @ h0 -> h1 + vadd.i64 $D4,$D4,$T1 @ h3 -> h4 + + cmp $len,#0 + bne .Leven + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ store hash value + + vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + vst1.32 {$D4#lo[0]},[$ctx] + + vldmia sp!,{d8-d15} @ epilogue + ldmia sp!,{r4-r7} +.Lno_data_neon: + ret @ bx lr +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +#ifdef __KERNEL__ +.globl poly1305_emit_neon +#endif +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + + stmdb sp!,{r4-r11} + + tst ip,ip + beq .Lpoly1305_emit_enter + + ldmia $ctx,{$h0-$h4} + eor $g0,$g0,$g0 + + adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32 + mov $h1,$h1,lsr#6 + adcs $h1,$h1,$h2,lsl#20 + mov $h2,$h2,lsr#12 + adcs $h2,$h2,$h3,lsl#14 + mov $h3,$h3,lsr#18 + adcs $h3,$h3,$h4,lsl#8 + adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ... + + and $g0,$h4,#-4 @ ... so reduce + and $h4,$h3,#3 + add $g0,$g0,$g0,lsr#2 @ *= 5 + adds $h0,$h0,$g0 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + + it ne + movne $h0,$g0 + ldr $g0,[$nonce,#0] + it ne + movne $h1,$g1 + ldr $g1,[$nonce,#4] + it ne + movne $h2,$g2 + ldr $g2,[$nonce,#8] + it ne + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 @ accumulate nonce + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] @ store the result + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] + + ldmia sp!,{r4-r11} + ret @ bx lr +.size poly1305_emit_neon,.-poly1305_emit_neon + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +# ifndef __KERNEL__ +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.Lpoly1305_init +# endif +#endif +___ +} } +$code.=<<___; +.align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} +close STDOUT; # enforce flush diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl new file mode 100644 index 0000000000000..d513b45a149bf --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl @@ -0,0 +1,974 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for ARMv8. +# +# June 2015 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc-4.9 NEON +# +# Apple A7 1.86/+5% 0.72 +# Cortex-A53 2.69/+58% 1.47 +# Cortex-A57 2.70/+7% 1.14 +# Denver 1.64/+50% 1.18(*) +# X-Gene 2.13/+68% 2.27 +# Mongoose 1.77/+75% 1.12 +# Kryo 2.70/+55% 1.13 +# +# (*) estimate based on resources availability is less than 1.0, +# i.e. measured result is worse than expected, presumably binary +# translator is not almighty; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3)); +my ($mac,$nonce)=($inp,$len); + +my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text + +// forward "declarations" are required for Apple +.globl poly1305_blocks +.globl poly1305_emit +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: + cmp $inp,xzr + stp xzr,xzr,[$ctx] // zero hash value + stp xzr,xzr,[$ctx,#16] // [along with is_base2_26] + + csel x0,xzr,x0,eq + b.eq .Lno_key + +#ifndef __KERNEL__ +# ifdef __ILP32__ + ldrsw $t1,.LOPENSSL_armcap_P +# else + ldr $t1,.LOPENSSL_armcap_P +# endif + adr $t0,.LOPENSSL_armcap_P + ldr w17,[$t0,$t1] +#endif + + ldp $r0,$r1,[$inp] // load key + mov $s1,#0xfffffffc0fffffff + movk $s1,#0x0fff,lsl#48 +#ifdef __AARCH64EB__ + rev $r0,$r0 // flip bytes + rev $r1,$r1 +#endif + and $r0,$r0,$s1 // &=0ffffffc0fffffff + and $s1,$s1,#-4 + and $r1,$r1,$s1 // &=0ffffffc0ffffffc + stp $r0,$r1,[$ctx,#32] // save key value + +#ifndef __KERNEL__ + tst w17,#ARMV7_NEON + + adr $d0,poly1305_blocks + adr $r0,poly1305_blocks_neon + adr $d1,poly1305_emit + adr $r1,poly1305_emit_neon + + csel $d0,$d0,$r0,eq + csel $d1,$d1,$r1,eq + +# ifdef __ILP32__ + stp w12,w13,[$len] +# else + stp $d0,$d1,[$len] +# endif + + mov x0,#1 +#else + mov x0,#0 +#endif +.Lno_key: + ret +.size poly1305_init,.-poly1305_init + +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: + ands $len,$len,#-16 + b.eq .Lno_data + + ldp $h0,$h1,[$ctx] // load hash value + ldp $r0,$r1,[$ctx,#32] // load key value + ldr $h2,[$ctx,#16] + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + b .Loop + +.align 5 +.Loop: + ldp $t0,$t1,[$inp],#16 // load input + sub $len,$len,#16 +#ifdef __AARCH64EB__ + rev $t0,$t0 + rev $t1,$t1 +#endif + adds $h0,$h0,$t0 // accumulate input + adcs $h1,$h1,$t1 + + mul $d0,$h0,$r0 // h0*r0 + adc $h2,$h2,$padbit + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + cbnz $len,.Loop + + stp $h0,$h1,[$ctx] // store hash value + str $h2,[$ctx,#16] + +.Lno_data: + ret +.size poly1305_blocks,.-poly1305_blocks + +.type poly1305_emit,%function +.align 5 +poly1305_emit: + ldp $h0,$h1,[$ctx] // load hash base 2^64 + ldr $h2,[$ctx,#16] + ldp $t0,$t1,[$nonce] // load nonce + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit,.-poly1305_emit +___ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8)); +my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13)); +my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18)); +my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23)); +my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28)); +my ($T0,$T1,$MASK) = map("v$_",(29..31)); + +my ($in2,$zeros)=("x16","x17"); +my $is_base2_26 = $zeros; # borrow + +$code.=<<___; +.type __poly1305_mult,%function +.align 5 +__poly1305_mult: + mul $d0,$h0,$r0 // h0*r0 + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + ret +.size __poly1305_mult,.-__poly1305_mult + +.type __poly1305_splat,%function +.align 5 +__poly1305_splat: + and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x13,$h0,#26,#26 + extr x14,$h1,$h0,#52 + and x14,x14,#0x03ffffff + ubfx x15,$h1,#14,#26 + extr x16,$h2,$h1,#40 + + str w12,[$ctx,#16*0] // r0 + add w12,w13,w13,lsl#2 // r1*5 + str w13,[$ctx,#16*1] // r1 + add w13,w14,w14,lsl#2 // r2*5 + str w12,[$ctx,#16*2] // s1 + str w14,[$ctx,#16*3] // r2 + add w14,w15,w15,lsl#2 // r3*5 + str w13,[$ctx,#16*4] // s2 + str w15,[$ctx,#16*5] // r3 + add w15,w16,w16,lsl#2 // r4*5 + str w14,[$ctx,#16*6] // s3 + str w16,[$ctx,#16*7] // r4 + str w15,[$ctx,#16*8] // s4 + + ret +.size __poly1305_splat,.-__poly1305_splat + +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +.globl poly1305_emit_neon +#endif + +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr $is_base2_26,[$ctx,#24] + cmp $len,#128 + b.hs .Lblocks_neon + cbz $is_base2_26,poly1305_blocks + +.Lblocks_neon: + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + + ands $len,$len,#-16 + b.eq .Lno_data_neon + + cbz $is_base2_26,.Lbase2_64_neon + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + tst $len,#31 + b.eq .Leven_neon + + ldp $r0,$r1,[$ctx,#32] // load key value + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $d2,$h2,xzr // can be partially reduced... + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + + and $t0,$d2,#-4 // ... so reduce + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$h0,$t0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + ldr x30,[sp,#8] + + cbz $padbit,.Lstore_base2_64_neon + + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + cbnz $len,.Leven_neon + + stp w10,w11,[$ctx] // store hash value base 2^26 + stp w12,w13,[$ctx,#8] + str w14,[$ctx,#16] + b .Lno_data_neon + +.align 4 +.Lstore_base2_64_neon: + stp $h0,$h1,[$ctx] // store hash value base 2^64 + stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed + b .Lno_data_neon + +.align 4 +.Lbase2_64_neon: + ldp $r0,$r1,[$ctx,#32] // load key value + + ldp $h0,$h1,[$ctx] // load hash value base 2^64 + ldr $h2,[$ctx,#16] + + tst $len,#31 + b.eq .Linit_neon + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + +.Linit_neon: + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + + ////////////////////////////////// initialize r^n table + mov $h0,$r0 // r^1 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + mov $h1,$r1 + mov $h2,xzr + add $ctx,$ctx,#48+12 + bl __poly1305_splat + + bl __poly1305_mult // r^2 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^3 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^4 + sub $ctx,$ctx,#4 + bl __poly1305_splat + ldr x30,[sp,#8] + + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + mov x4,#1 + str x4,[$ctx,#-24] // set is_base2_26 + sub $ctx,$ctx,#48 // restore original $ctx + b .Ldo_neon + +.align 4 +.Leven_neon: + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + +.Ldo_neon: + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + ldp x9,x13,[$in2],#48 + + lsl $padbit,$padbit,#24 + add x15,$ctx,#48 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN23_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN23_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + fmov $IN23_2,x8 + fmov $IN23_3,x10 + fmov $IN23_4,x12 + + ldp x8,x12,[$inp],#16 // inp[0:1] + ldp x9,x13,[$inp],#48 + + ld1 {$R0,$R1,$S1,$R2},[x15],#64 + ld1 {$S2,$R3,$S3,$R4},[x15],#64 + ld1 {$S4},[x15] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN01_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN01_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + movi $MASK.2d,#-1 + fmov $IN01_2,x8 + fmov $IN01_3,x10 + fmov $IN01_4,x12 + ushr $MASK.2d,$MASK.2d,#38 + + b.ls .Lskip_loop + +.align 4 +.Loop_neon: + //////////////////////////////////////////////////////////////// + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + // \___________________/ + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + // \___________________/ \____________________/ + // + // Note that we start with inp[2:3]*r^2. This is because it + // doesn't depend on reduction in previous iteration. + //////////////////////////////////////////////////////////////// + // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 + // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 + // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 + // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 + // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 + + subs $len,$len,#64 + umull $ACC4,$IN23_0,${R4}[2] + csel $in2,$zeros,$in2,lo + umull $ACC3,$IN23_0,${R3}[2] + umull $ACC2,$IN23_0,${R2}[2] + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + umull $ACC1,$IN23_0,${R1}[2] + ldp x9,x13,[$in2],#48 + umull $ACC0,$IN23_0,${R0}[2] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + umlal $ACC4,$IN23_1,${R3}[2] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC3,$IN23_1,${R2}[2] + and x5,x9,#0x03ffffff + umlal $ACC2,$IN23_1,${R1}[2] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN23_1,${R0}[2] + ubfx x7,x9,#26,#26 + umlal $ACC0,$IN23_1,${S4}[2] + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + + umlal $ACC4,$IN23_2,${R2}[2] + extr x8,x12,x8,#52 + umlal $ACC3,$IN23_2,${R1}[2] + extr x9,x13,x9,#52 + umlal $ACC2,$IN23_2,${R0}[2] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC1,$IN23_2,${S4}[2] + fmov $IN23_0,x4 + umlal $ACC0,$IN23_2,${S3}[2] + and x8,x8,#0x03ffffff + + umlal $ACC4,$IN23_3,${R1}[2] + and x9,x9,#0x03ffffff + umlal $ACC3,$IN23_3,${R0}[2] + ubfx x10,x12,#14,#26 + umlal $ACC2,$IN23_3,${S4}[2] + ubfx x11,x13,#14,#26 + umlal $ACC1,$IN23_3,${S3}[2] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC0,$IN23_3,${S2}[2] + fmov $IN23_1,x6 + + add $IN01_2,$IN01_2,$H2 + add x12,$padbit,x12,lsr#40 + umlal $ACC4,$IN23_4,${R0}[2] + add x13,$padbit,x13,lsr#40 + umlal $ACC3,$IN23_4,${S4}[2] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC2,$IN23_4,${S3}[2] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN23_4,${S2}[2] + fmov $IN23_2,x8 + umlal $ACC0,$IN23_4,${S1}[2] + fmov $IN23_3,x10 + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4 and accumulate + + add $IN01_0,$IN01_0,$H0 + fmov $IN23_4,x12 + umlal $ACC3,$IN01_2,${R1}[0] + ldp x8,x12,[$inp],#16 // inp[0:1] + umlal $ACC0,$IN01_2,${S3}[0] + ldp x9,x13,[$inp],#48 + umlal $ACC4,$IN01_2,${R2}[0] + umlal $ACC1,$IN01_2,${S4}[0] + umlal $ACC2,$IN01_2,${R0}[0] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3}[0] + umlal $ACC4,$IN01_0,${R4}[0] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC2,$IN01_0,${R2}[0] + and x5,x9,#0x03ffffff + umlal $ACC0,$IN01_0,${R0}[0] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN01_0,${R1}[0] + ubfx x7,x9,#26,#26 + + add $IN01_3,$IN01_3,$H3 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + umlal $ACC3,$IN01_1,${R2}[0] + extr x8,x12,x8,#52 + umlal $ACC4,$IN01_1,${R3}[0] + extr x9,x13,x9,#52 + umlal $ACC0,$IN01_1,${S4}[0] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC2,$IN01_1,${R1}[0] + fmov $IN01_0,x4 + umlal $ACC1,$IN01_1,${R0}[0] + and x8,x8,#0x03ffffff + + add $IN01_4,$IN01_4,$H4 + and x9,x9,#0x03ffffff + umlal $ACC3,$IN01_3,${R0}[0] + ubfx x10,x12,#14,#26 + umlal $ACC0,$IN01_3,${S2}[0] + ubfx x11,x13,#14,#26 + umlal $ACC4,$IN01_3,${R1}[0] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC1,$IN01_3,${S3}[0] + fmov $IN01_1,x6 + umlal $ACC2,$IN01_3,${S4}[0] + add x12,$padbit,x12,lsr#40 + + umlal $ACC3,$IN01_4,${S4}[0] + add x13,$padbit,x13,lsr#40 + umlal $ACC0,$IN01_4,${S1}[0] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC4,$IN01_4,${R0}[0] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN01_4,${S2}[0] + fmov $IN01_2,x8 + umlal $ACC2,$IN01_4,${S3}[0] + fmov $IN01_3,x10 + fmov $IN01_4,x12 + + ///////////////////////////////////////////////////////////////// + // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + // and P. Schwabe + // + // [see discussion in poly1305-armv4 module] + + ushr $T0.2d,$ACC3,#26 + xtn $H3,$ACC3 + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + bic $H3,#0xfc,lsl#24 // &=0x03ffffff + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + xtn $H4,$ACC4 + ushr $T1.2d,$ACC1,#26 + xtn $H1,$ACC1 + bic $H4,#0xfc,lsl#24 + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + shrn $T1.2s,$ACC2,#26 + xtn $H2,$ACC2 + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + bic $H1,#0xfc,lsl#24 + add $H3,$H3,$T1.2s // h2 -> h3 + bic $H2,#0xfc,lsl#24 + + shrn $T0.2s,$ACC0,#26 + xtn $H0,$ACC0 + ushr $T1.2s,$H3,#26 + bic $H3,#0xfc,lsl#24 + bic $H0,#0xfc,lsl#24 + add $H1,$H1,$T0.2s // h0 -> h1 + add $H4,$H4,$T1.2s // h3 -> h4 + + b.hi .Loop_neon + +.Lskip_loop: + dup $IN23_2,${IN23_2}[0] + add $IN01_2,$IN01_2,$H2 + + //////////////////////////////////////////////////////////////// + // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + adds $len,$len,#32 + b.ne .Long_tail + + dup $IN23_2,${IN01_2}[0] + add $IN23_0,$IN01_0,$H0 + add $IN23_3,$IN01_3,$H3 + add $IN23_1,$IN01_1,$H1 + add $IN23_4,$IN01_4,$H4 + +.Long_tail: + dup $IN23_0,${IN23_0}[0] + umull2 $ACC0,$IN23_2,${S3} + umull2 $ACC3,$IN23_2,${R1} + umull2 $ACC4,$IN23_2,${R2} + umull2 $ACC2,$IN23_2,${R0} + umull2 $ACC1,$IN23_2,${S4} + + dup $IN23_1,${IN23_1}[0] + umlal2 $ACC0,$IN23_0,${R0} + umlal2 $ACC2,$IN23_0,${R2} + umlal2 $ACC3,$IN23_0,${R3} + umlal2 $ACC4,$IN23_0,${R4} + umlal2 $ACC1,$IN23_0,${R1} + + dup $IN23_3,${IN23_3}[0] + umlal2 $ACC0,$IN23_1,${S4} + umlal2 $ACC3,$IN23_1,${R2} + umlal2 $ACC2,$IN23_1,${R1} + umlal2 $ACC4,$IN23_1,${R3} + umlal2 $ACC1,$IN23_1,${R0} + + dup $IN23_4,${IN23_4}[0] + umlal2 $ACC3,$IN23_3,${R0} + umlal2 $ACC4,$IN23_3,${R1} + umlal2 $ACC0,$IN23_3,${S2} + umlal2 $ACC1,$IN23_3,${S3} + umlal2 $ACC2,$IN23_3,${S4} + + umlal2 $ACC3,$IN23_4,${S4} + umlal2 $ACC0,$IN23_4,${S1} + umlal2 $ACC4,$IN23_4,${R0} + umlal2 $ACC1,$IN23_4,${S2} + umlal2 $ACC2,$IN23_4,${S3} + + b.eq .Lshort_tail + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4:r^3 and accumulate + + add $IN01_0,$IN01_0,$H0 + umlal $ACC3,$IN01_2,${R1} + umlal $ACC0,$IN01_2,${S3} + umlal $ACC4,$IN01_2,${R2} + umlal $ACC1,$IN01_2,${S4} + umlal $ACC2,$IN01_2,${R0} + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3} + umlal $ACC0,$IN01_0,${R0} + umlal $ACC4,$IN01_0,${R4} + umlal $ACC1,$IN01_0,${R1} + umlal $ACC2,$IN01_0,${R2} + + add $IN01_3,$IN01_3,$H3 + umlal $ACC3,$IN01_1,${R2} + umlal $ACC0,$IN01_1,${S4} + umlal $ACC4,$IN01_1,${R3} + umlal $ACC1,$IN01_1,${R0} + umlal $ACC2,$IN01_1,${R1} + + add $IN01_4,$IN01_4,$H4 + umlal $ACC3,$IN01_3,${R0} + umlal $ACC0,$IN01_3,${S2} + umlal $ACC4,$IN01_3,${R1} + umlal $ACC1,$IN01_3,${S3} + umlal $ACC2,$IN01_3,${S4} + + umlal $ACC3,$IN01_4,${S4} + umlal $ACC0,$IN01_4,${S1} + umlal $ACC4,$IN01_4,${R0} + umlal $ACC1,$IN01_4,${S2} + umlal $ACC2,$IN01_4,${S3} + +.Lshort_tail: + //////////////////////////////////////////////////////////////// + // horizontal add + + addp $ACC3,$ACC3,$ACC3 + ldp d8,d9,[sp,#16] // meet ABI requirements + addp $ACC0,$ACC0,$ACC0 + ldp d10,d11,[sp,#32] + addp $ACC4,$ACC4,$ACC4 + ldp d12,d13,[sp,#48] + addp $ACC1,$ACC1,$ACC1 + ldp d14,d15,[sp,#64] + addp $ACC2,$ACC2,$ACC2 + + //////////////////////////////////////////////////////////////// + // lazy reduction, but without narrowing + + ushr $T0.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + and $ACC4,$ACC4,$MASK.2d + ushr $T1.2d,$ACC1,#26 + and $ACC1,$ACC1,$MASK.2d + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + ushr $T1.2d,$ACC2,#26 + and $ACC2,$ACC2,$MASK.2d + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + add $ACC3,$ACC3,$T1.2d // h2 -> h3 + + ushr $T0.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + ushr $T1.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + add $ACC1,$ACC1,$T0.2d // h0 -> h1 + add $ACC4,$ACC4,$T1.2d // h3 -> h4 + + //////////////////////////////////////////////////////////////// + // write the result, can be partially reduced + + st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16 + st1 {$ACC4}[0],[$ctx] + +.Lno_data_neon: + ldr x29,[sp],#80 + ret +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr $is_base2_26,[$ctx,#24] + cbz $is_base2_26,poly1305_emit + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $h2,$h2,xzr // can be partially reduced... + + ldp $t0,$t1,[$nonce] // load nonce + + and $d0,$h2,#-4 // ... so reduce + add $d0,$d0,$h2,lsr#2 + and $h2,$h2,#3 + adds $h0,$h0,$d0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit_neon,.-poly1305_emit_neon +#endif + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +#ifdef __ILP32__ +.long OPENSSL_armcap_P-. +#else +.quad OPENSSL_armcap_P-. +#endif +#endif +.align 2 +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or + s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or + (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or + (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or + (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or + (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or + (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1)); + + s/\.[124]([sd])\[/.$1\[/; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c new file mode 100644 index 0000000000000..527ccc3b59cc8 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +struct poly1305_internal { + u32 h[5]; + u32 r[5]; + u32 s[4]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + st->r[0] = (get_unaligned_le32(&key[0])) & 0x3ffffff; + st->r[1] = (get_unaligned_le32(&key[3]) >> 2) & 0x3ffff03; + st->r[2] = (get_unaligned_le32(&key[6]) >> 4) & 0x3ffc0ff; + st->r[3] = (get_unaligned_le32(&key[9]) >> 6) & 0x3f03fff; + st->r[4] = (get_unaligned_le32(&key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + st->s[0] = st->r[1] * 5; + st->s[1] = st->r[2] * 5; + st->s[2] = st->r[3] * 5; + st->s[3] = st->r[4] * 5; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; + st->h[3] = 0; + st->h[4] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u32 hibit = padbit << 24; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + r3 = st->r[3]; + r4 = st->r[4]; + + s1 = st->s[0]; + s2 = st->s[1]; + s3 = st->s[2]; + s4 = st->s[3]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + while (len >= POLY1305_BLOCK_SIZE) { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; + st->h[3] = h3; + st->h[4] = h4; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c new file mode 100644 index 0000000000000..131f1dda1b1db --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +typedef __uint128_t u128; + +struct poly1305_internal { + u64 r[3]; + u64 h[3]; + u64 s[2]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&key[0]); + t1 = get_unaligned_le64(&key[8]); + + st->r[0] = t0 & 0xffc0fffffffULL; + st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + st->r[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + st->s[0] = st->r[1] * 20; + st->s[1] = st->r[2] * 20; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u64 hibit = ((u64)padbit) << 40; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + s1 = st->s[0]; + s2 = st->s[1]; + + while (len >= POLY1305_BLOCK_SIZE) { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c new file mode 100644 index 0000000000000..a540e9c4eee8b --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_mips(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + poly1305_blocks_mips(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + poly1305_emit_mips(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S new file mode 100644 index 0000000000000..4291c156815b3 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#else +#define MSB 3 +#define LSB 0 +#endif + +#define POLY1305_BLOCK_SIZE 16 +.text +#define H0 $t0 +#define H1 $t1 +#define H2 $t2 +#define H3 $t3 +#define H4 $t4 + +#define R0 $t5 +#define R1 $t6 +#define R2 $t7 +#define R3 $t8 + +#define O0 $s0 +#define O1 $s4 +#define O2 $v1 +#define O3 $t9 +#define O4 $s5 + +#define S1 $s1 +#define S2 $s2 +#define S3 $s3 + +#define SC $at +#define CA $v0 + +/* Input arguments */ +#define poly $a0 +#define src $a1 +#define srclen $a2 +#define hibit $a3 + +/* Location in the opaque buffer + * R[0..3], CA, H[0..4] + */ +#define PTR_POLY1305_R(n) ( 0 + (n*4)) ## ($a0) +#define PTR_POLY1305_CA (16 ) ## ($a0) +#define PTR_POLY1305_H(n) (20 + (n*4)) ## ($a0) + +#define POLY1305_BLOCK_SIZE 16 +#define POLY1305_STACK_SIZE 32 + +.set noat +.align 4 +.globl poly1305_blocks_mips +.ent poly1305_blocks_mips +poly1305_blocks_mips: + .frame $sp, POLY1305_STACK_SIZE, $ra + /* srclen &= 0xFFFFFFF0 */ + ins srclen, $zero, 0, 4 + + addiu $sp, -(POLY1305_STACK_SIZE) + + /* check srclen >= 16 bytes */ + beqz srclen, .Lpoly1305_blocks_mips_end + + /* Calculate last round based on src address pointer. + * last round src ptr (srclen) = src + (srclen & 0xFFFFFFF0) + */ + addu srclen, src + + lw R0, PTR_POLY1305_R(0) + lw R1, PTR_POLY1305_R(1) + lw R2, PTR_POLY1305_R(2) + lw R3, PTR_POLY1305_R(3) + + /* store the used save registers. */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Sx = Rx + (Rx >> 2) */ + srl S1, R1, 2 + srl S2, R2, 2 + srl S3, R3, 2 + addu S1, R1 + addu S2, R2 + addu S3, R3 + + addiu SC, $zero, 1 + +.Lpoly1305_loop: + lwl O0, 0+MSB(src) + lwl O1, 4+MSB(src) + lwl O2, 8+MSB(src) + lwl O3,12+MSB(src) + lwr O0, 0+LSB(src) + lwr O1, 4+LSB(src) + lwr O2, 8+LSB(src) + lwr O3,12+LSB(src) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh O0 + wsbh O1 + wsbh O2 + wsbh O3 + rotr O0, 16 + rotr O1, 16 + rotr O2, 16 + rotr O3, 16 +#endif + + /* h0 = (u32)(d0 = (u64)h0 + inp[0] + c 'Carry_previous cycle'); */ + addu H0, CA + sltu CA, H0, CA + addu O0, H0 + sltu H0, O0, H0 + addu CA, H0 + + /* h1 = (u32)(d1 = (u64)h1 + (d0 >> 32) + inp[4]); */ + addu H1, CA + sltu CA, H1, CA + addu O1, H1 + sltu H1, O1, H1 + addu CA, H1 + + /* h2 = (u32)(d2 = (u64)h2 + (d1 >> 32) + inp[8]); */ + addu H2, CA + sltu CA, H2, CA + addu O2, H2 + sltu H2, O2, H2 + addu CA, H2 + + /* h3 = (u32)(d3 = (u64)h3 + (d2 >> 32) + inp[12]); */ + addu H3, CA + sltu CA, H3, CA + addu O3, H3 + sltu H3, O3, H3 + addu CA, H3 + + /* h4 += (u32)(d3 >> 32) + padbit; */ + addu H4, hibit + addu O4, H4, CA + + /* D0 */ + multu O0, R0 + maddu O1, S3 + maddu O2, S2 + maddu O3, S1 + mfhi CA + mflo H0 + + /* D1 */ + multu O0, R1 + maddu O1, R0 + maddu O2, S3 + maddu O3, S2 + maddu O4, S1 + maddu CA, SC + mfhi CA + mflo H1 + + /* D2 */ + multu O0, R2 + maddu O1, R1 + maddu O2, R0 + maddu O3, S3 + maddu O4, S2 + maddu CA, SC + mfhi CA + mflo H2 + + /* D4 */ + mul H4, O4, R0 + + /* D3 */ + multu O0, R3 + maddu O1, R2 + maddu O2, R1 + maddu O3, R0 + maddu O4, S3 + maddu CA, SC + mfhi CA + mflo H3 + + addiu src, POLY1305_BLOCK_SIZE + + /* h4 += (u32)(d3 >> 32); */ + addu O4, H4, CA + /* h4 &= 3 */ + andi H4, O4, 3 + /* c = (h4 >> 2) + (h4 & ~3U); */ + srl CA, O4, 2 + ins O4, $zero, 0, 2 + + addu CA, O4 + + /* able to do a 16 byte block. */ + bne src, srclen, .Lpoly1305_loop + + /* restore the used save registers. */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + + /* store Hx and Carry */ + sw CA, PTR_POLY1305_CA + sw H0, PTR_POLY1305_H(0) + sw H1, PTR_POLY1305_H(1) + sw H2, PTR_POLY1305_H(2) + sw H3, PTR_POLY1305_H(3) + sw H4, PTR_POLY1305_H(4) + +.Lpoly1305_blocks_mips_end: + addiu $sp, POLY1305_STACK_SIZE + + /* Jump Back */ + jr $ra +.end poly1305_blocks_mips +.set at + +/* Input arguments CTX=$a0, MAC=$a1, NONCE=$a2 */ +#define MAC $a1 +#define NONCE $a2 + +#define G0 $t5 +#define G1 $t6 +#define G2 $t7 +#define G3 $t8 +#define G4 $t9 + +.set noat +.align 4 +.globl poly1305_emit_mips +.ent poly1305_emit_mips +poly1305_emit_mips: + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Add left over carry */ + addu H0, CA + sltu CA, H0, CA + addu H1, CA + sltu CA, H1, CA + addu H2, CA + sltu CA, H2, CA + addu H3, CA + sltu CA, H3, CA + addu H4, CA + + /* compare to modulus by computing h + -p */ + addiu G0, H0, 5 + sltu CA, G0, H0 + addu G1, H1, CA + sltu CA, G1, H1 + addu G2, H2, CA + sltu CA, G2, H2 + addu G3, H3, CA + sltu CA, G3, H3 + addu G4, H4, CA + + srl SC, G4, 2 + + /* if there was carry into 131st bit, h3:h0 = g3:g0 */ + movn H0, G0, SC + movn H1, G1, SC + movn H2, G2, SC + movn H3, G3, SC + + lwl G0, 0+MSB(NONCE) + lwl G1, 4+MSB(NONCE) + lwl G2, 8+MSB(NONCE) + lwl G3,12+MSB(NONCE) + lwr G0, 0+LSB(NONCE) + lwr G1, 4+LSB(NONCE) + lwr G2, 8+LSB(NONCE) + lwr G3,12+LSB(NONCE) + + /* mac = (h + nonce) % (2^128) */ + addu H0, G0 + sltu CA, H0, G0 + + /* H1 */ + addu H1, CA + sltu CA, H1, CA + addu H1, G1 + sltu G1, H1, G1 + addu CA, G1 + + /* H2 */ + addu H2, CA + sltu CA, H2, CA + addu H2, G2 + sltu G2, H2, G2 + addu CA, G2 + + /* H3 */ + addu H3, CA + addu H3, G3 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh H0 + wsbh H1 + wsbh H2 + wsbh H3 + rotr H0, 16 + rotr H1, 16 + rotr H2, 16 + rotr H3, 16 +#endif + + /* store MAC */ + swl H0, 0+MSB(MAC) + swl H1, 4+MSB(MAC) + swl H2, 8+MSB(MAC) + swl H3,12+MSB(MAC) + swr H0, 0+LSB(MAC) + swr H1, 4+LSB(MAC) + swr H2, 8+LSB(MAC) + swr H3,12+LSB(MAC) + + jr $ra +.end poly1305_emit_mips + +#define PR0 $t0 +#define PR1 $t1 +#define PR2 $t2 +#define PR3 $t3 +#define PT0 $t4 + +/* Input arguments CTX=$a0, KEY=$a1 */ + +.align 4 +.globl poly1305_init_mips +.ent poly1305_init_mips +poly1305_init_mips: + lwl PR0, 0+MSB($a1) + lwl PR1, 4+MSB($a1) + lwl PR2, 8+MSB($a1) + lwl PR3,12+MSB($a1) + lwr PR0, 0+LSB($a1) + lwr PR1, 4+LSB($a1) + lwr PR2, 8+LSB($a1) + lwr PR3,12+LSB($a1) + + /* store Hx and Carry */ + sw $zero, PTR_POLY1305_CA + sw $zero, PTR_POLY1305_H(0) + sw $zero, PTR_POLY1305_H(1) + sw $zero, PTR_POLY1305_H(2) + sw $zero, PTR_POLY1305_H(3) + sw $zero, PTR_POLY1305_H(4) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh PR0 + wsbh PR1 + wsbh PR2 + wsbh PR3 + rotr PR0, 16 + rotr PR1, 16 + rotr PR2, 16 + rotr PR3, 16 +#endif + + lui PT0, 0x0FFF + ori PT0, 0xFFFC + + /* AND 0x0fffffff; */ + ext PR0, PR0, 0, (32-4) + + /* AND 0x0ffffffc; */ + and PR1, PT0 + and PR2, PT0 + and PR3, PT0 + + /* store Rx */ + sw PR0, PTR_POLY1305_R(0) + sw PR1, PTR_POLY1305_R(1) + sw PR2, PTR_POLY1305_R(2) + sw PR3, PTR_POLY1305_R(3) + + /* Jump Back */ + jr $ra +.end poly1305_init_mips diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl new file mode 100644 index 0000000000000..4adb9d4007ce6 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl @@ -0,0 +1,466 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# Poly1305 hash for MIPS64. +# +# May 2016 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc +# R1x000 5.64/+120% (big-endian) +# Octeon II 3.80/+280% (little-endian) + +###################################################################### +# There is a number of MIPS ABI in use, O32 and N32/64 are most +# widely used. Then there is a new contender: NUBI. It appears that if +# one picks the latter, it's possible to arrange code in ABI neutral +# manner. Therefore let's stick to NUBI register layout: +# +($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23)); +($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31)); +# +# The return value is placed in $a0. Following coding rules facilitate +# interoperability: +# +# - never ever touch $tp, "thread pointer", former $gp [o32 can be +# excluded from the rule, because it's specified volatile]; +# - copy return value to $t0, former $v0 [or to $a0 if you're adapting +# old code]; +# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary; +# +# For reference here is register layout for N32/64 MIPS ABIs: +# +# ($zero,$at,$v0,$v1)=map("\$$_",(0..3)); +# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25)); +# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); +# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); +# +# +# +###################################################################### + +$flavour = shift || "64"; # supported flavours are o32,n32,64,nubi32,nubi64 + +die "MIPS64 only" unless ($flavour =~ /64|n32/i); + +$v0 = ($flavour =~ /nubi/i) ? $a0 : $t0; +$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0x0003f000" : "0x00030000"; + +($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3); +($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$at,$t0,$t1); + +$code.=<<___; +#if (defined(_MIPS_ARCH_MIPS64R3) || defined(_MIPS_ARCH_MIPS64R5) || \\ + defined(_MIPS_ARCH_MIPS64R6)) \\ + && !defined(_MIPS_ARCH_MIPS64R2) +# define _MIPS_ARCH_MIPS64R2 +#endif + +#if defined(_MIPS_ARCH_MIPS64R6) +# define dmultu(rs,rt) +# define mflo(rd,rs,rt) dmulu rd,rs,rt +# define mfhi(rd,rs,rt) dmuhu rd,rs,rt +#else +# define dmultu(rs,rt) dmultu rs,rt +# define mflo(rd,rs,rt) mflo rd +# define mfhi(rd,rs,rt) mfhi rd +#endif + +#ifdef __KERNEL__ +# define poly1305_init poly1305_init_mips +# define poly1305_blocks poly1305_blocks_mips +# define poly1305_emit poly1305_emit_mips +#endif + +#if defined(__MIPSEB__) && !defined(MIPSEB) +# define MIPSEB +#endif + +#ifdef MIPSEB +# define MSB 0 +# define LSB 7 +#else +# define MSB 7 +# define LSB 0 +#endif + +.text +.set noat +.set noreorder + +.align 5 +.globl poly1305_init +.ent poly1305_init +poly1305_init: + .frame $sp,0,$ra + .set reorder + + sd $zero,0($ctx) + sd $zero,8($ctx) + sd $zero,16($ctx) + + beqz $inp,.Lno_key + +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + li $tmp0,1 + dsll $tmp0,32 + daddiu $tmp0,-63 + dsll $tmp0,28 + daddiu $tmp0,-1 # 0ffffffc0fffffff + + and $in0,$tmp0 + daddiu $tmp0,-3 # 0ffffffc0ffffffc + and $in1,$tmp0 + + sd $in0,24($ctx) + dsrl $tmp0,$in1,2 + sd $in1,32($ctx) + daddu $tmp0,$in1 # s1 = r1 + (r1 >> 2) + sd $tmp0,40($ctx) + +.Lno_key: + li $v0,0 # return 0 + jr $ra +.end poly1305_init +___ +{ +my ($h0,$h1,$h2,$r0,$r1,$s1,$d0,$d1,$d2) = + ($s0,$s1,$s2,$s3,$s4,$s5,$in0,$in1,$t2); + +$code.=<<___; +.align 5 +.globl poly1305_blocks +.ent poly1305_blocks +poly1305_blocks: + .set noreorder + dsrl $len,4 # number of complete blocks + bnez $len,poly1305_blocks_internal + nop + jr $ra + nop +.end poly1305_blocks + +.align 5 +.ent poly1305_blocks_internal +poly1305_blocks_internal: + .frame $sp,6*8,$ra + .mask $SAVED_REGS_MASK,-8 + .set noreorder + dsubu $sp,6*8 + sd $s5,40($sp) + sd $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi prologue + sd $s3,24($sp) + sd $s2,16($sp) + sd $s1,8($sp) + sd $s0,0($sp) +___ +$code.=<<___; + .set reorder + + ld $h0,0($ctx) # load hash value + ld $h1,8($ctx) + ld $h2,16($ctx) + + ld $r0,24($ctx) # load key + ld $r1,32($ctx) + ld $s1,40($ctx) + +.Loop: +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) # load input + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) # load input + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif + daddiu $len,-1 + daddiu $inp,16 +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + daddu $h0,$in0 # accumulate input + daddu $h1,$in1 + sltu $tmp0,$h0,$in0 + sltu $tmp1,$h1,$in1 + daddu $h1,$tmp0 + + dmultu ($r0,$h0) # h0*r0 + daddu $h2,$padbit + sltu $tmp0,$h1,$tmp0 + mflo ($d0,$r0,$h0) + mfhi ($d1,$r0,$h0) + + dmultu ($s1,$h1) # h1*5*r1 + daddu $tmp0,$tmp1 + daddu $h2,$tmp0 + mflo ($tmp0,$s1,$h1) + mfhi ($tmp1,$s1,$h1) + + dmultu ($r1,$h0) # h0*r1 + daddu $d0,$tmp0 + daddu $d1,$tmp1 + mflo ($tmp2,$r1,$h0) + mfhi ($d2,$r1,$h0) + sltu $tmp0,$d0,$tmp0 + daddu $d1,$tmp0 + + dmultu ($r0,$h1) # h1*r0 + daddu $d1,$tmp2 + sltu $tmp2,$d1,$tmp2 + mflo ($tmp0,$r0,$h1) + mfhi ($tmp1,$r0,$h1) + daddu $d2,$tmp2 + + dmultu ($s1,$h2) # h2*5*r1 + daddu $d1,$tmp0 + daddu $d2,$tmp1 + mflo ($tmp2,$s1,$h2) + + dmultu ($r0,$h2) # h2*r0 + sltu $tmp0,$d1,$tmp0 + daddu $d2,$tmp0 + mflo ($tmp3,$r0,$h2) + + daddu $d1,$tmp2 + daddu $d2,$tmp3 + sltu $tmp2,$d1,$tmp2 + daddu $d2,$tmp2 + + li $tmp0,-4 # final reduction + and $tmp0,$d2 + dsrl $tmp1,$d2,2 + andi $h2,$d2,3 + daddu $tmp0,$tmp1 + daddu $h0,$d0,$tmp0 + sltu $tmp0,$h0,$tmp0 + daddu $h1,$d1,$tmp0 + sltu $tmp0,$h1,$tmp0 + daddu $h2,$h2,$tmp0 + + bnez $len,.Loop + + sd $h0,0($ctx) # store hash value + sd $h1,8($ctx) + sd $h2,16($ctx) + + .set noreorder + ld $s5,40($sp) # epilogue + ld $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi epilogue + ld $s3,24($sp) + ld $s2,16($sp) + ld $s1,8($sp) + ld $s0,0($sp) +___ +$code.=<<___; + jr $ra + daddu $sp,6*8 +.end poly1305_blocks_internal +___ +} +{ +my ($ctx,$mac,$nonce) = ($a0,$a1,$a2); + +$code.=<<___; +.align 5 +.globl poly1305_emit +.ent poly1305_emit +poly1305_emit: + .frame $sp,0,$ra + .set reorder + + ld $tmp0,0($ctx) + ld $tmp1,8($ctx) + ld $tmp2,16($ctx) + + daddiu $in0,$tmp0,5 # compare to modulus + sltiu $tmp3,$in0,5 + daddu $in1,$tmp1,$tmp3 + sltu $tmp3,$in1,$tmp3 + daddu $tmp2,$tmp2,$tmp3 + + dsrl $tmp2,2 # see if it carried/borrowed + dsubu $tmp2,$zero,$tmp2 + nor $tmp3,$zero,$tmp2 + + and $in0,$tmp2 + and $tmp0,$tmp3 + and $in1,$tmp2 + and $tmp1,$tmp3 + or $in0,$tmp0 + or $in1,$tmp1 + + lwu $tmp0,0($nonce) # load nonce + lwu $tmp1,4($nonce) + lwu $tmp2,8($nonce) + lwu $tmp3,12($nonce) + dsll $tmp1,32 + dsll $tmp3,32 + or $tmp0,$tmp1 + or $tmp2,$tmp3 + + daddu $in0,$tmp0 # accumulate nonce + daddu $in1,$tmp2 + sltu $tmp0,$in0,$tmp0 + daddu $in1,$tmp0 + + dsrl $tmp0,$in0,8 # write mac value + dsrl $tmp1,$in0,16 + dsrl $tmp2,$in0,24 + sb $in0,0($mac) + dsrl $tmp3,$in0,32 + sb $tmp0,1($mac) + dsrl $tmp0,$in0,40 + sb $tmp1,2($mac) + dsrl $tmp1,$in0,48 + sb $tmp2,3($mac) + dsrl $tmp2,$in0,56 + sb $tmp3,4($mac) + dsrl $tmp3,$in1,8 + sb $tmp0,5($mac) + dsrl $tmp0,$in1,16 + sb $tmp1,6($mac) + dsrl $tmp1,$in1,24 + sb $tmp2,7($mac) + + sb $in1,8($mac) + dsrl $tmp2,$in1,32 + sb $tmp3,9($mac) + dsrl $tmp3,$in1,40 + sb $tmp0,10($mac) + dsrl $tmp0,$in1,48 + sb $tmp1,11($mac) + dsrl $tmp1,$in1,56 + sb $tmp2,12($mac) + sb $tmp3,13($mac) + sb $tmp0,14($mac) + sb $tmp1,15($mac) + + jr $ra +.end poly1305_emit +.rdata +.align 2 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +$output=pop and open STDOUT,">$output"; +print $code; +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c new file mode 100644 index 0000000000000..ce48a42f7654c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); + +static bool poly1305_use_avx __ro_after_init; +static bool poly1305_use_avx2 __ro_after_init; +static bool poly1305_use_avx512 __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { + &poly1305_use_avx, &poly1305_use_avx2, &poly1305_use_avx512 }; + +static void __init poly1305_fpu_init(void) +{ + poly1305_use_avx = + boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); + poly1305_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + poly1305_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; +#endif +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_x86_64(ctx, key); + return true; +} + +struct poly1305_arch_internal { + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; + u64 r[2]; + u64 pad; + struct { u32 r2, r1, r4, r3; } rn[9]; +}; + +/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + !state->is_base2_26 || !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_x86_64(ctx, mac, nonce); + } else + poly1305_emit_avx(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl new file mode 100644 index 0000000000000..c9a8e0037bdbd --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl @@ -0,0 +1,4266 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for x86_64. +# +# March 2015 +# +# Initial release. +# +# December 2016 +# +# Add AVX512F+VL+BW code path. +# +# November 2017 +# +# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be +# executed even on Knights Landing. Trigger for modification was +# observation that AVX512 code paths can negatively affect overall +# Skylake-X system performance. Since we are likely to suppress +# AVX512F capability flag [at least on Skylake-X], conversion serves +# as kind of "investment protection". Note that next *lake processor, +# Cannonlake, has AVX512IFMA code path to execute... +# +# Numbers are cycles per processed byte with poly1305_blocks alone, +# measured with rdtsc at fixed clock frequency. +# +# IALU/gcc-4.8(*) AVX(**) AVX2 AVX-512 +# P4 4.46/+120% - +# Core 2 2.41/+90% - +# Westmere 1.88/+120% - +# Sandy Bridge 1.39/+140% 1.10 +# Haswell 1.14/+175% 1.11 0.65 +# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35] +# Silvermont 2.83/+95% - +# Knights L 3.60/? 1.65 1.10 0.41(***) +# Goldmont 1.70/+180% - +# VIA Nano 1.82/+150% - +# Sledgehammer 1.38/+160% - +# Bulldozer 2.30/+130% 0.97 +# Ryzen 1.15/+200% 1.08 1.18 +# +# (*) improvement coefficients relative to clang are more modest and +# are ~50% on most processors, in both cases we are comparing to +# __int128 code; +# (**) SSE2 implementation was attempted, but among non-AVX processors +# it was faster than integer-only code only on older Intel P4 and +# Core processors, 50-30%, less newer processor is, but slower on +# contemporary ones, for example almost 2x slower on Atom, and as +# former are naturally disappearing, SSE2 is deemed unnecessary; +# (***) strangely enough performance seems to vary from core to core, +# listed result is best case; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +$code.=<<___ if $kernel; +#include +___ + +if ($avx) { +$code.=<<___ if $kernel; +.section .rodata +___ +$code.=<<___; +.align 64 +.Lconst: +.Lmask24: +.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 +.L129: +.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 +.Lmask26: +.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 +.Lpermd_avx2: +.long 2,2,2,3,2,0,2,1 +.Lpermd_avx512: +.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 + +.L2_44_inp_permd: +.long 0,1,1,2,2,3,7,7 +.L2_44_inp_shift: +.quad 0,12,24,64 +.L2_44_mask: +.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff +.L2_44_shift_rgt: +.quad 44,44,42,64 +.L2_44_shift_lft: +.quad 8,8,10,64 + +.align 64 +.Lx_mask44: +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.Lx_mask42: +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +___ +} +$code.=<<___ if (!$kernel); +.asciz "Poly1305 for x86_64, CRYPTOGAMS by " +.align 16 +___ + +my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); +my ($mac,$nonce)=($inp,$len); # *_emit arguments +my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); +my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); + +sub poly1305_iteration { +# input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 +# output: $h0-$h2 *= $r0-$r1 +$code.=<<___; + mulq $h0 # h0*r1 + mov %rax,$d2 + mov $r0,%rax + mov %rdx,$d3 + + mulq $h0 # h0*r0 + mov %rax,$h0 # future $h0 + mov $r0,%rax + mov %rdx,$d1 + + mulq $h1 # h1*r0 + add %rax,$d2 + mov $s1,%rax + adc %rdx,$d3 + + mulq $h1 # h1*s1 + mov $h2,$h1 # borrow $h1 + add %rax,$h0 + adc %rdx,$d1 + + imulq $s1,$h1 # h2*s1 + add $h1,$d2 + mov $d1,$h1 + adc \$0,$d3 + + imulq $r0,$h2 # h2*r0 + add $d2,$h1 + mov \$-4,%rax # mask value + adc $h2,$d3 + + and $d3,%rax # last reduction step + mov $d3,$h2 + shr \$2,$d3 + and \$3,$h2 + add $d3,%rax + add %rax,$h0 + adc \$0,$h1 + adc \$0,$h2 +___ +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^64 +# unsigned __int64 r[2]; # key value base 2^64 + +$code.=<<___; +.text +___ +$code.=<<___ if (!$kernel); +.extern OPENSSL_ia32cap_P + +.globl poly1305_init_x86_64 +.hidden poly1305_init_x86_64 +.globl poly1305_blocks_x86_64 +.hidden poly1305_blocks_x86_64 +.globl poly1305_emit_x86_64 +.hidden poly1305_emit_x86_64 +___ +&declare_function("poly1305_init_x86_64", 32, 3); +$code.=<<___; + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + + cmp \$0,$inp + je .Lno_key +___ +$code.=<<___ if (!$kernel); + lea poly1305_blocks_x86_64(%rip),%r10 + lea poly1305_emit_x86_64(%rip),%r11 +___ +$code.=<<___ if (!$kernel && $avx); + mov OPENSSL_ia32cap_P+4(%rip),%r9 + lea poly1305_blocks_avx(%rip),%rax + lea poly1305_emit_avx(%rip),%rcx + bt \$`60-32`,%r9 # AVX? + cmovc %rax,%r10 + cmovc %rcx,%r11 +___ +$code.=<<___ if (!$kernel && $avx>1); + lea poly1305_blocks_avx2(%rip),%rax + bt \$`5+32`,%r9 # AVX2? + cmovc %rax,%r10 +___ +$code.=<<___ if (!$kernel && $avx>3); + mov \$`(1<<31|1<<21|1<<16)`,%rax + shr \$32,%r9 + and %rax,%r9 + cmp %rax,%r9 + je .Linit_base2_44 +___ +$code.=<<___; + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + and 8($inp),%rcx + mov %rax,24($ctx) + mov %rcx,32($ctx) +___ +$code.=<<___ if (!$kernel && $flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if (!$kernel && $flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax +.Lno_key: + ret +___ +&end_function("poly1305_init_x86_64"); + +&declare_function("poly1305_blocks_x86_64", 32, 4); +$code.=<<___; +.cfi_startproc +.Lblocks: + shr \$4,$len + jz .Lno_data # too short + + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + push $ctx +.cfi_push $ctx +.Lblocks_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2 + + mov $s1,$r1 + shr \$2,$s1 + mov $r1,%rax + add $r1,$s1 # s1 = r1 + (r1 >> 2) + jmp .Loop + +.align 32 +.Loop: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 +___ + + &poly1305_iteration(); + +$code.=<<___; + mov $r1,%rax + dec %r15 # len-=16 + jnz .Loop + + mov 0(%rsp),$ctx +.cfi_restore $ctx + + mov $h0,0($ctx) # store hash value + mov $h1,8($ctx) + mov $h2,16($ctx) + + mov 8(%rsp),%r15 +.cfi_restore %r15 + mov 16(%rsp),%r14 +.cfi_restore %r14 + mov 24(%rsp),%r13 +.cfi_restore %r13 + mov 32(%rsp),%r12 +.cfi_restore %r12 + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lno_data: +.Lblocks_epilogue: + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_x86_64"); + +&declare_function("poly1305_emit_x86_64", 32, 3); +$code.=<<___; +.Lemit: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_x86_64"); +if ($avx) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX\n"; +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int32 h[5]; # current hash value base 2^26 +# unsigned __int32 is_base2_26; +# unsigned __int64 r[2]; # key value base 2^64 +# unsigned __int64 pad; +# struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9]; +# +# where r^n are base 2^26 digits of degrees of multiplier key. There are +# 5 digits, but last four are interleaved with multiples of 5, totalling +# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. + +my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) = + map("%xmm$_",(0..15)); + +$code.=<<___; +.type __poly1305_block,\@abi-omnipotent +.align 32 +__poly1305_block: + push $ctx +___ + &poly1305_iteration(); +$code.=<<___; + pop $ctx + ret +.size __poly1305_block,.-__poly1305_block + +.type __poly1305_init_avx,\@abi-omnipotent +.align 32 +__poly1305_init_avx: + push %rbp + mov %rsp,%rbp + mov $r0,$h0 + mov $r1,$h1 + xor $h2,$h2 + + lea 48+64($ctx),$ctx # size optimization + + mov $r1,%rax + call __poly1305_block # r^2 + + mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26 + mov \$0x3ffffff,%edx + mov $h0,$d1 + and $h0#d,%eax + mov $r0,$d2 + and $r0#d,%edx + mov %eax,`16*0+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*0+4-64`($ctx) + shr \$26,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*1+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*1+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*2+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*2+4-64`($ctx) + shr \$26,$d2 + + mov $h1,%rax + mov $r1,%rdx + shl \$12,%rax + shl \$12,%rdx + or $d1,%rax + or $d2,%rdx + and \$0x3ffffff,%eax + and \$0x3ffffff,%edx + mov %eax,`16*3+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*3+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*4+0-64`($ctx) + mov $h1,$d1 + mov %edx,`16*4+4-64`($ctx) + mov $r1,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + shr \$14,$d1 + shr \$14,$d2 + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*5+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*5+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*6+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*6+4-64`($ctx) + shr \$26,$d2 + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+0-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d2#d,`16*7+4-64`($ctx) + lea ($d2,$d2,4),$d2 # *5 + mov $d1#d,`16*8+0-64`($ctx) + mov $d2#d,`16*8+4-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^3 + + mov \$0x3ffffff,%eax # save r^3 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+12-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+12-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+12-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+12-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+12-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+12-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+12-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^4 + + mov \$0x3ffffff,%eax # save r^4 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+8-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+8-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+8-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+8-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+8-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+8-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+8-64`($ctx) + + lea -48-64($ctx),$ctx # size [de-]optimization + pop %rbp + ret +.size __poly1305_init_avx,.-__poly1305_init_avx +___ + +&declare_function("poly1305_blocks_avx", 32, 4); +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx: + and \$-16,$len + jz .Lno_data_avx + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx + + test \$31,$len + jz .Leven_avx + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx_body: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + + call __poly1305_block + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + sub \$16,%r15 + jz .Lstore_base2_26_avx + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + jmp .Lproceed_avx + +.align 32 +.Lstore_base2_64_avx: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx + +.align 16 +.Lstore_base2_26_avx: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx: +.Lblocks_avx_epilogue: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$31,$len + jz .Linit_avx + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + +.Linit_avx: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx: + mov %r15,$len + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx_epilogue: + jmp .Ldo_avx +.cfi_endproc + +.align 32 +.Leven_avx: +.cfi_startproc + vmovd 4*0($ctx),$H0 # load hash value + vmovd 4*1($ctx),$H1 + vmovd 4*2($ctx),$H2 + vmovd 4*3($ctx),$H3 + vmovd 4*4($ctx),$H4 + +.Ldo_avx: +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + and \$-32,%rsp + sub \$-8,%rsp + lea -0x58(%rsp),%r11 + sub \$0x178,%rsp + +___ +$code.=<<___ if ($win64); + lea -0xf8(%rsp),%r11 + sub \$0x218,%rsp + vmovdqa %xmm6,0x50(%r11) + vmovdqa %xmm7,0x60(%r11) + vmovdqa %xmm8,0x70(%r11) + vmovdqa %xmm9,0x80(%r11) + vmovdqa %xmm10,0x90(%r11) + vmovdqa %xmm11,0xa0(%r11) + vmovdqa %xmm12,0xb0(%r11) + vmovdqa %xmm13,0xc0(%r11) + vmovdqa %xmm14,0xd0(%r11) + vmovdqa %xmm15,0xe0(%r11) +.Ldo_avx_body: +___ +$code.=<<___; + sub \$64,$len + lea -32($inp),%rax + cmovc %rax,$inp + + vmovdqu `16*3`($ctx),$D4 # preload r0^2 + lea `16*3+64`($ctx),$ctx # size optimization + lea .Lconst(%rip),%rcx + + ################################################################ + # load input + vmovdqu 16*2($inp),$T0 + vmovdqu 16*3($inp),$T1 + vmovdqa 64(%rcx),$MASK # .Lmask26 + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + vpsrlq \$40,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + jbe .Lskip_loop_avx + + # expand and copy pre-calculated table to stack + vmovdqu `16*1-64`($ctx),$D1 + vmovdqu `16*2-64`($ctx),$D2 + vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434 + vpshufd \$0x44,$D4,$D0 # xx12 -> 1212 + vmovdqa $D3,-0x90(%r11) + vmovdqa $D0,0x00(%rsp) + vpshufd \$0xEE,$D1,$D4 + vmovdqu `16*3-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x80(%r11) + vmovdqa $D1,0x10(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqu `16*4-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x70(%r11) + vmovdqa $D2,0x20(%rsp) + vpshufd \$0xEE,$D0,$D4 + vmovdqu `16*5-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D4,-0x60(%r11) + vmovdqa $D0,0x30(%rsp) + vpshufd \$0xEE,$D1,$D3 + vmovdqu `16*6-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D3,-0x50(%r11) + vmovdqa $D1,0x40(%rsp) + vpshufd \$0xEE,$D2,$D4 + vmovdqu `16*7-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D4,-0x40(%r11) + vmovdqa $D2,0x50(%rsp) + vpshufd \$0xEE,$D0,$D3 + vmovdqu `16*8-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D3,-0x30(%r11) + vmovdqa $D0,0x60(%rsp) + vpshufd \$0xEE,$D1,$D4 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x20(%r11) + vmovdqa $D1,0x70(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x10(%r11) + vmovdqa $D2,0x80(%rsp) + + jmp .Loop_avx + +.align 32 +.Loop_avx: + ################################################################ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + # \___________________/ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + # \___________________/ \____________________/ + # + # Note that we start with inp[2:3]*r^2. This is because it + # doesn't depend on reduction in previous iteration. + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # though note that $Tx and $Hx are "reversed" in this section, + # and $D4 is preloaded with r0^2... + + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vmovdqa $H2,0x20(%r11) # offload hash + vpmuludq $T2,$D4,$D2 # d3 = h2*r0 + vmovdqa 0x10(%rsp),$H2 # r1^2 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vmovdqa $H0,0x00(%r11) # + vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1 + vmovdqa $H1,0x10(%r11) # + vpmuludq $T3,$H2,$H1 # h3*r1 + vpaddq $H0,$D0,$D0 # d0 += h4*s1 + vpaddq $H1,$D4,$D4 # d4 += h3*r1 + vmovdqa $H3,0x30(%r11) # + vpmuludq $T2,$H2,$H0 # h2*r1 + vpmuludq $T1,$H2,$H1 # h1*r1 + vpaddq $H0,$D3,$D3 # d3 += h2*r1 + vmovdqa 0x30(%rsp),$H3 # r2^2 + vpaddq $H1,$D2,$D2 # d2 += h1*r1 + vmovdqa $H4,0x40(%r11) # + vpmuludq $T0,$H2,$H2 # h0*r1 + vpmuludq $T2,$H3,$H0 # h2*r2 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + + vmovdqa 0x40(%rsp),$H4 # s2^2 + vpaddq $H0,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H3,$H1 # h1*r2 + vpmuludq $T0,$H3,$H3 # h0*r2 + vpaddq $H1,$D3,$D3 # d3 += h1*r2 + vmovdqa 0x50(%rsp),$H2 # r3^2 + vpaddq $H3,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H4,$H0 # h4*s2 + vpmuludq $T3,$H4,$H4 # h3*s2 + vpaddq $H0,$D1,$D1 # d1 += h4*s2 + vmovdqa 0x60(%rsp),$H3 # s3^2 + vpaddq $H4,$D0,$D0 # d0 += h3*s2 + + vmovdqa 0x80(%rsp),$H4 # s4^2 + vpmuludq $T1,$H2,$H1 # h1*r3 + vpmuludq $T0,$H2,$H2 # h0*r3 + vpaddq $H1,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $T4,$H3,$H0 # h4*s3 + vpmuludq $T3,$H3,$H1 # h3*s3 + vpaddq $H0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*0($inp),$H0 # load input + vpaddq $H1,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H3,$H3 # h2*s3 + vpmuludq $T2,$H4,$T2 # h2*s4 + vpaddq $H3,$D0,$D0 # d0 += h2*s3 + + vmovdqu 16*1($inp),$H1 # + vpaddq $T2,$D1,$D1 # d1 += h2*s4 + vpmuludq $T3,$H4,$T3 # h3*s4 + vpmuludq $T4,$H4,$T4 # h4*s4 + vpsrldq \$6,$H0,$H2 # splat input + vpaddq $T3,$D2,$D2 # d2 += h3*s4 + vpaddq $T4,$D3,$D3 # d3 += h4*s4 + vpsrldq \$6,$H1,$H3 # + vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4 + vpmuludq $T1,$H4,$T0 # h1*s4 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpaddq $T4,$D4,$D4 # d4 += h0*r4 + vmovdqa -0x90(%r11),$T4 # r0^4 + vpaddq $T0,$D0,$D0 # d0 += h1*s4 + + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + #vpsrlq \$40,$H4,$H4 # 4 + vpsrldq \$`40/8`,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpand 0(%rcx),$H4,$H4 # .Lmask24 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpaddq 0x00(%r11),$H0,$H0 # add hash value + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + lea 16*2($inp),%rax + lea 16*4($inp),$inp + sub \$64,$len + cmovc %rax,$inp + + ################################################################ + # Now we accumulate (inp[0:1]+hash)*r^4 + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vmovdqa -0x80(%r11),$T2 # r1^4 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T0,$D2,$D2 + vpaddq $T1,$D3,$D3 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpmuludq -0x70(%r11),$H4,$T0 # h4*s1 + vpaddq $T4,$D4,$D4 + + vpaddq $T0,$D0,$D0 # d0 += h4*s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vmovdqa -0x60(%r11),$T3 # r2^4 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpmuludq $H1,$T2,$T1 # h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T1,$D2,$D2 # d2 += h1*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + + vmovdqa -0x50(%r11),$T4 # s2^4 + vpmuludq $H2,$T3,$T0 # h2*r2 + vpmuludq $H1,$T3,$T1 # h1*r2 + vpaddq $T0,$D4,$D4 # d4 += h2*r2 + vpaddq $T1,$D3,$D3 # d3 += h1*r2 + vmovdqa -0x40(%r11),$T2 # r3^4 + vpmuludq $H0,$T3,$T3 # h0*r2 + vpmuludq $H4,$T4,$T0 # h4*s2 + vpaddq $T3,$D2,$D2 # d2 += h0*r2 + vpaddq $T0,$D1,$D1 # d1 += h4*s2 + vmovdqa -0x30(%r11),$T3 # s3^4 + vpmuludq $H3,$T4,$T4 # h3*s2 + vpmuludq $H1,$T2,$T1 # h1*r3 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + + vmovdqa -0x10(%r11),$T4 # s4^4 + vpaddq $T1,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T2,$T2 # h0*r3 + vpmuludq $H4,$T3,$T0 # h4*s3 + vpaddq $T2,$D3,$D3 # d3 += h0*r3 + vpaddq $T0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*2($inp),$T0 # load input + vpmuludq $H3,$T3,$T2 # h3*s3 + vpmuludq $H2,$T3,$T3 # h2*s3 + vpaddq $T2,$D1,$D1 # d1 += h3*s3 + vmovdqu 16*3($inp),$T1 # + vpaddq $T3,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H2,$T4,$H2 # h2*s4 + vpmuludq $H3,$T4,$H3 # h3*s4 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $H2,$D1,$D1 # d1 += h2*s4 + vpmuludq $H4,$T4,$H4 # h4*s4 + vpsrldq \$6,$T1,$T3 # + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4 + vpmuludq -0x20(%r11),$H0,$H4 # h0*r4 + vpmuludq $H1,$T4,$H0 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + #vpsrlq \$40,$T4,$T4 # 4 + vpsrldq \$`40/8`,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpand 0(%rcx),$T4,$T4 # .Lmask24 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + ################################################################ + # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + # and P. Schwabe + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D0 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D0,$H0,$H0 + vpsllq \$2,$D0,$D0 + vpaddq $D0,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + ja .Loop_avx + +.Lskip_loop_avx: + ################################################################ + # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2 + add \$32,$len + jnz .Long_tail_avx + + vpaddq $H2,$T2,$T2 + vpaddq $H0,$T0,$T0 + vpaddq $H1,$T1,$T1 + vpaddq $H3,$T3,$T3 + vpaddq $H4,$T4,$T4 + +.Long_tail_avx: + vmovdqa $H2,0x20(%r11) + vmovdqa $H0,0x00(%r11) + vmovdqa $H1,0x10(%r11) + vmovdqa $H3,0x30(%r11) + vmovdqa $H4,0x40(%r11) + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $T2,$D4,$D2 # d2 = h2*r0 + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vpmuludq $T3,$H2,$H0 # h3*r1 + vpaddq $H0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n + vpmuludq $T2,$H2,$H1 # h2*r1 + vpaddq $H1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n + vpmuludq $T1,$H2,$H0 # h1*r1 + vpaddq $H0,$D2,$D2 # d2 += h1*r1 + vpmuludq $T0,$H2,$H2 # h0*r1 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + vpmuludq $T4,$H3,$H3 # h4*s1 + vpaddq $H3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n + vpmuludq $T2,$H4,$H1 # h2*r2 + vpaddq $H1,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H4,$H0 # h1*r2 + vpaddq $H0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n + vpmuludq $T0,$H4,$H4 # h0*r2 + vpaddq $H4,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H2,$H1 # h4*s2 + vpaddq $H1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n + vpmuludq $T3,$H2,$H2 # h3*s2 + vpaddq $H2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $T1,$H3,$H0 # h1*r3 + vpaddq $H0,$D4,$D4 # d4 += h1*r3 + vpmuludq $T0,$H3,$H3 # h0*r3 + vpaddq $H3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n + vpmuludq $T4,$H4,$H1 # h4*s3 + vpaddq $H1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n + vpmuludq $T3,$H4,$H0 # h3*s3 + vpaddq $H0,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H4,$H4 # h2*s3 + vpaddq $H4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $T0,$H2,$H2 # h0*r4 + vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4 + vpmuludq $T4,$H3,$H1 # h4*s4 + vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4 + vpmuludq $T3,$H3,$H0 # h3*s4 + vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4 + vpmuludq $T2,$H3,$H1 # h2*s4 + vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4 + vpmuludq $T1,$H3,$H3 # h1*s4 + vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4 + + jz .Lshort_tail_avx + + vmovdqu 16*0($inp),$H0 # load input + vmovdqu 16*1($inp),$H1 + + vpsrldq \$6,$H0,$H2 # splat input + vpsrldq \$6,$H1,$H3 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + vpsrlq \$40,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4 + vpaddq 0x00(%r11),$H0,$H0 + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + ################################################################ + # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpaddq $T0,$D0,$D0 # d0 += h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T1,$D1,$D1 # d1 += h1*r0 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpaddq $T0,$D2,$D2 # d2 += h2*r0 + vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T1,$D3,$D3 # d3 += h3*r0 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpaddq $T4,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2 + vpmuludq $H1,$T2,$T0 # h1*r1 + vpaddq $T0,$D2,$D2 # d2 += h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + vpmuludq $H4,$T3,$T3 # h4*s1 + vpaddq $T3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2 + vpmuludq $H2,$T4,$T1 # h2*r2 + vpaddq $T1,$D4,$D4 # d4 += h2*r2 + vpmuludq $H1,$T4,$T0 # h1*r2 + vpaddq $T0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3 + vpmuludq $H0,$T4,$T4 # h0*r2 + vpaddq $T4,$D2,$D2 # d2 += h0*r2 + vpmuludq $H4,$T2,$T1 # h4*s2 + vpaddq $T1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3 + vpmuludq $H3,$T2,$T2 # h3*s2 + vpaddq $T2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $H1,$T3,$T0 # h1*r3 + vpaddq $T0,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T3,$T3 # h0*r3 + vpaddq $T3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4 + vpmuludq $H4,$T4,$T1 # h4*s3 + vpaddq $T1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4 + vpmuludq $H3,$T4,$T0 # h3*s3 + vpaddq $T0,$D1,$D1 # d1 += h3*s3 + vpmuludq $H2,$T4,$T4 # h2*s3 + vpaddq $T4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H0,$T2,$T2 # h0*r4 + vpaddq $T2,$D4,$D4 # d4 += h0*r4 + vpmuludq $H4,$T3,$T1 # h4*s4 + vpaddq $T1,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$T3,$T0 # h3*s4 + vpaddq $T0,$D2,$D2 # d2 += h3*s4 + vpmuludq $H2,$T3,$T1 # h2*s4 + vpaddq $T1,$D1,$D1 # d1 += h2*s4 + vpmuludq $H1,$T3,$T3 # h1*s4 + vpaddq $T3,$D0,$D0 # d0 += h1*s4 + +.Lshort_tail_avx: + ################################################################ + # horizontal addition + + vpsrldq \$8,$D4,$T4 + vpsrldq \$8,$D3,$T3 + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$D0,$T0 + vpsrldq \$8,$D2,$T2 + vpaddq $T3,$D3,$D3 + vpaddq $T4,$D4,$D4 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$D2,$D2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D4,$H4 + vpand $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$H1 + vpand $MASK,$D1,$D1 + vpaddq $H1,$D2,$D2 # h1 -> h2 + + vpaddq $H4,$D0,$D0 + vpsllq \$2,$H4,$H4 + vpaddq $H4,$D0,$D0 # h4 -> h0 + + vpsrlq \$26,$D2,$H2 + vpand $MASK,$D2,$D2 + vpaddq $H2,$D3,$D3 # h2 -> h3 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vmovd $D0,`4*0-48-64`($ctx) # save partially reduced + vmovd $D1,`4*1-48-64`($ctx) + vmovd $D2,`4*2-48-64`($ctx) + vmovd $D3,`4*3-48-64`($ctx) + vmovd $D4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa 0x50(%r11),%xmm6 + vmovdqa 0x60(%r11),%xmm7 + vmovdqa 0x70(%r11),%xmm8 + vmovdqa 0x80(%r11),%xmm9 + vmovdqa 0x90(%r11),%xmm10 + vmovdqa 0xa0(%r11),%xmm11 + vmovdqa 0xb0(%r11),%xmm12 + vmovdqa 0xc0(%r11),%xmm13 + vmovdqa 0xd0(%r11),%xmm14 + vmovdqa 0xe0(%r11),%xmm15 + lea 0xf8(%r11),%rsp +.Ldo_avx_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_avx"); + +&declare_function("poly1305_emit_avx", 32, 3); +$code.=<<___; + cmpl \$0,20($ctx) # is_base2_26? + je .Lemit + + mov 0($ctx),%eax # load hash value base 2^26 + mov 4($ctx),%ecx + mov 8($ctx),%r8d + mov 12($ctx),%r11d + mov 16($ctx),%r10d + + shl \$26,%rcx # base 2^26 -> base 2^64 + mov %r8,%r9 + shl \$52,%r8 + add %rcx,%rax + shr \$12,%r9 + add %rax,%r8 # h0 + adc \$0,%r9 + + shl \$14,%r11 + mov %r10,%rax + shr \$24,%r10 + add %r11,%r9 + shl \$40,%rax + add %rax,%r9 # h1 + adc \$0,%r10 # h2 + + mov %r10,%rax # could be partially reduced, so reduce + mov %r10,%rcx + and \$3,%r10 + shr \$2,%rax + and \$-4,%rcx + add %rcx,%rax + add %rax,%r8 + adc \$0,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_avx"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if ($avx>1) { + +if ($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = + map("%ymm$_",(0..15)); +my $S4=$MASK; + +sub poly1305_blocks_avxN { + my ($avx512) = @_; + my $suffix = $avx512 ? "_avx512" : ""; +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx2$suffix + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx2$suffix: + and \$-16,$len + jz .Lno_data_avx2$suffix + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx2$suffix + + test \$63,$len + jz .Leven_avx2$suffix + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + +.Lbase2_26_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_26_pre_avx2$suffix + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + test %r15,%r15 + jz .Lstore_base2_26_avx2$suffix + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + jmp .Lproceed_avx2$suffix + +.align 32 +.Lstore_base2_64_avx2$suffix: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx2$suffix + +.align 16 +.Lstore_base2_26_avx2$suffix: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx2$suffix: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx2$suffix: +.Lblocks_avx2_epilogue$suffix: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx2$suffix: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$63,$len + jz .Linit_avx2$suffix + +.Lbase2_64_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_64_pre_avx2$suffix + +.Linit_avx2$suffix: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx2$suffix: + mov %r15,$len # restore $len +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d + mov \$`(1<<31|1<<30|1<<16)`,%r11d +___ +$code.=<<___; + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx2_epilogue$suffix: + jmp .Ldo_avx2$suffix +.cfi_endproc + +.align 32 +.Leven_avx2$suffix: +.cfi_startproc +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d +___ +$code.=<<___; + vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 + vmovd 4*1($ctx),%x#$H1 + vmovd 4*2($ctx),%x#$H2 + vmovd 4*3($ctx),%x#$H3 + vmovd 4*4($ctx),%x#$H4 + +.Ldo_avx2$suffix: +___ +$code.=<<___ if (!$kernel && $avx>2); + cmp \$512,$len + jb .Lskip_avx512 + and %r11d,%r9d + test \$`1<<16`,%r9d # check for AVX512F + jnz .Lblocks_avx512 +.Lskip_avx512$suffix: +___ +$code.=<<___ if ($avx > 2 && $avx512 && $kernel); + cmp \$512,$len + jae .Lblocks_avx512 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx2_body$suffix: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),$T0 # .Lpermd_avx2 + + # expand and copy pre-calculated table to stack + vmovdqu `16*0-64`($ctx),%x#$T2 + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$T3 + vmovdqu `16*2-64`($ctx),%x#$T4 + vmovdqu `16*3-64`($ctx),%x#$D0 + vmovdqu `16*4-64`($ctx),%x#$D1 + vmovdqu `16*5-64`($ctx),%x#$D2 + lea 0x90(%rsp),%rax # size optimization + vmovdqu `16*6-64`($ctx),%x#$D3 + vpermd $T2,$T0,$T2 # 00003412 -> 14243444 + vmovdqu `16*7-64`($ctx),%x#$D4 + vpermd $T3,$T0,$T3 + vmovdqu `16*8-64`($ctx),%x#$MASK + vpermd $T4,$T0,$T4 + vmovdqa $T2,0x00(%rsp) + vpermd $D0,$T0,$D0 + vmovdqa $T3,0x20-0x90(%rax) + vpermd $D1,$T0,$D1 + vmovdqa $T4,0x40-0x90(%rax) + vpermd $D2,$T0,$D2 + vmovdqa $D0,0x60-0x90(%rax) + vpermd $D3,$T0,$D3 + vmovdqa $D1,0x80-0x90(%rax) + vpermd $D4,$T0,$D4 + vmovdqa $D2,0xa0-0x90(%rax) + vpermd $MASK,$T0,$MASK + vmovdqa $D3,0xc0-0x90(%rax) + vmovdqa $D4,0xe0-0x90(%rax) + vmovdqa $MASK,0x100-0x90(%rax) + vmovdqa 64(%rcx),$MASK # .Lmask26 + + ################################################################ + # load input + vmovdqu 16*0($inp),%x#$T0 + vmovdqu 16*1($inp),%x#$T1 + vinserti128 \$1,16*2($inp),$T0,$T0 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$64,$len + jz .Ltail_avx2$suffix + jmp .Loop_avx2$suffix + +.align 32 +.Loop_avx2$suffix: + ################################################################ + # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 + # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 + # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2 + # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1 + # \________/\__________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqa `32*0`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqa `32*1`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqa `32*3`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqa `32*6-0x90`(%rax),$T3 # s3^4 + vmovdqa `32*8-0x90`(%rax),$S4 # s4^4 + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4 + # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + vmovdqa `32*4-0x90`(%rax),$T1 # s2 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vmovdqu 16*0($inp),%x#$T0 # load input + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + vinserti128 \$1,16*2($inp),$T0,$T0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vmovdqu 16*1($inp),%x#$T1 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqa `32*5-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpsrldq \$6,$T1,$T3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + vpunpckhqdq $T1,$T0,$T4 # 4 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # lazy reduction (interleaved with tail of input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$4,$T3,$T2 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpand $MASK,$T2,$T2 # 2 + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$30,$T3,$T3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + sub \$64,$len + jnz .Loop_avx2$suffix + + .byte 0x66,0x90 +.Ltail_avx2$suffix: + ################################################################ + # while above multiplications were by r^4 in all lanes, in last + # iteration we multiply least significant lane by r^4 and most + # significant one by r, so copy of above except that references + # to the precomputed table are displaced by 4... + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqu `32*0+4`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqu `32*1+4`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqu `32*3+4`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4 + vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1 + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vmovdqu `32*4+4-0x90`(%rax),$T1 # s2 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqu `32*5+4-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # horizontal addition + + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$H2,$T2 + vpsrldq \$8,$H3,$T3 + vpsrldq \$8,$H4,$T4 + vpsrldq \$8,$H0,$T0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + + vpermq \$0x2,$H3,$T3 + vpermq \$0x2,$H4,$T4 + vpermq \$0x2,$H0,$T0 + vpermq \$0x2,$D1,$T1 + vpermq \$0x2,$H2,$T2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa -0xb0(%r10),%xmm6 + vmovdqa -0xa0(%r10),%xmm7 + vmovdqa -0x90(%r10),%xmm8 + vmovdqa -0x80(%r10),%xmm9 + vmovdqa -0x70(%r10),%xmm10 + vmovdqa -0x60(%r10),%xmm11 + vmovdqa -0x50(%r10),%xmm12 + vmovdqa -0x40(%r10),%xmm13 + vmovdqa -0x30(%r10),%xmm14 + vmovdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx2_epilogue$suffix: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +if($avx > 2 && $avx512) { +my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); +my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); +my $PADBIT="%zmm30"; + +map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain +map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4)); +map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); +map(s/%y/%z/,($MASK)); + +$code.=<<___; +.cfi_startproc +.Lblocks_avx512: + mov \$15,%eax + kmovw %eax,%k2 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx512_body: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2 + + # expand pre-calculated table + vmovdqu `16*0-64`($ctx),%x#$D0 # will become expanded ${R0} + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$D1 # will become ... ${R1} + mov \$0x20,%rax + vmovdqu `16*2-64`($ctx),%x#$T0 # ... ${S1} + vmovdqu `16*3-64`($ctx),%x#$D2 # ... ${R2} + vmovdqu `16*4-64`($ctx),%x#$T1 # ... ${S2} + vmovdqu `16*5-64`($ctx),%x#$D3 # ... ${R3} + vmovdqu `16*6-64`($ctx),%x#$T3 # ... ${S3} + vmovdqu `16*7-64`($ctx),%x#$D4 # ... ${R4} + vmovdqu `16*8-64`($ctx),%x#$T4 # ... ${S4} + vpermd $D0,$T2,$R0 # 00003412 -> 14243444 + vpbroadcastq 64(%rcx),$MASK # .Lmask26 + vpermd $D1,$T2,$R1 + vpermd $T0,$T2,$S1 + vpermd $D2,$T2,$R2 + vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0 + vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304 + vpermd $T1,$T2,$S2 + vmovdqu64 $R1,0x00(%rsp,%rax){%k2} + vpsrlq \$32,$R1,$T1 + vpermd $D3,$T2,$R3 + vmovdqa64 $S1,0x40(%rsp){%k2} + vpermd $T3,$T2,$S3 + vpermd $D4,$T2,$R4 + vmovdqu64 $R2,0x40(%rsp,%rax){%k2} + vpermd $T4,$T2,$S4 + vmovdqa64 $S2,0x80(%rsp){%k2} + vmovdqu64 $R3,0x80(%rsp,%rax){%k2} + vmovdqa64 $S3,0xc0(%rsp){%k2} + vmovdqu64 $R4,0xc0(%rsp,%rax){%k2} + vmovdqa64 $S4,0x100(%rsp){%k2} + + ################################################################ + # calculate 5th through 8th powers of the key + # + # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1 + # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2 + # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3 + # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4 + # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0 + + vpmuludq $T0,$R0,$D0 # d0 = r0'*r0 + vpmuludq $T0,$R1,$D1 # d1 = r0'*r1 + vpmuludq $T0,$R2,$D2 # d2 = r0'*r2 + vpmuludq $T0,$R3,$D3 # d3 = r0'*r3 + vpmuludq $T0,$R4,$D4 # d4 = r0'*r4 + vpsrlq \$32,$R2,$T2 + + vpmuludq $T1,$S4,$M0 + vpmuludq $T1,$R0,$M1 + vpmuludq $T1,$R1,$M2 + vpmuludq $T1,$R2,$M3 + vpmuludq $T1,$R3,$M4 + vpsrlq \$32,$R3,$T3 + vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4 + vpaddq $M1,$D1,$D1 # d1 += r1'*r0 + vpaddq $M2,$D2,$D2 # d2 += r1'*r1 + vpaddq $M3,$D3,$D3 # d3 += r1'*r2 + vpaddq $M4,$D4,$D4 # d4 += r1'*r3 + + vpmuludq $T2,$S3,$M0 + vpmuludq $T2,$S4,$M1 + vpmuludq $T2,$R1,$M3 + vpmuludq $T2,$R2,$M4 + vpmuludq $T2,$R0,$M2 + vpsrlq \$32,$R4,$T4 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4 + vpaddq $M3,$D3,$D3 # d3 += r2'*r1 + vpaddq $M4,$D4,$D4 # d4 += r2'*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*r0 + + vpmuludq $T3,$S2,$M0 + vpmuludq $T3,$R0,$M3 + vpmuludq $T3,$R1,$M4 + vpmuludq $T3,$S3,$M1 + vpmuludq $T3,$S4,$M2 + vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2 + vpaddq $M3,$D3,$D3 # d3 += r3'*r0 + vpaddq $M4,$D4,$D4 # d4 += r3'*r1 + vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3 + vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4 + + vpmuludq $T4,$S4,$M3 + vpmuludq $T4,$R0,$M4 + vpmuludq $T4,$S1,$M0 + vpmuludq $T4,$S2,$M1 + vpmuludq $T4,$S3,$M2 + vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4 + vpaddq $M4,$D4,$D4 # d4 += r2'*r0 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3 + + ################################################################ + # load input + vmovdqu64 16*0($inp),%z#$T3 + vmovdqu64 16*4($inp),%z#$T4 + lea 16*8($inp),$inp + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D4,$M4 + vpandq $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$M1 + vpandq $MASK,$D1,$D1 + vpaddq $M1,$D2,$D2 # d1 -> d2 + + vpaddq $M4,$D0,$D0 + vpsllq \$2,$M4,$M4 + vpaddq $M4,$D0,$D0 # d4 -> d0 + + vpsrlq \$26,$D2,$M2 + vpandq $MASK,$D2,$D2 + vpaddq $M2,$D3,$D3 # d2 -> d3 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + ################################################################ + # at this point we have 14243444 in $R0-$S4 and 05060708 in + # $D0-$D4, ... + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + # ... since input 64-bit lanes are ordered as 73625140, we could + # "vperm" it to 76543210 (here and in each loop iteration), *or* + # we could just flow along, hence the goal for $R0-$S4 is + # 1858286838784888 ... + + vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512: + mov \$0x7777,%eax + kmovw %eax,%k1 + + vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4--- + vpermd $R1,$M0,$R1 + vpermd $R2,$M0,$R2 + vpermd $R3,$M0,$R3 + vpermd $R4,$M0,$R4 + + vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888 + vpermd $D1,$M0,${R1}{%k1} + vpermd $D2,$M0,${R2}{%k1} + vpermd $D3,$M0,${R3}{%k1} + vpermd $D4,$M0,${R4}{%k1} + + vpslld \$2,$R1,$S1 # *5 + vpslld \$2,$R2,$S2 + vpslld \$2,$R3,$S3 + vpslld \$2,$R4,$S4 + vpaddd $R1,$S1,$S1 + vpaddd $R2,$S2,$S2 + vpaddd $R3,$S3,$S3 + vpaddd $R4,$S4,$S4 + + vpbroadcastq 32(%rcx),$PADBIT # .L129 + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + vporq $T3,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$14,$T4,$T3 + vpsrlq \$40,$T4,$T4 # 4 + vpandq $MASK,$T2,$T2 # 2 + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$192,$len + jbe .Ltail_avx512 + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: + ################################################################ + # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8 + # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7 + # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6 + # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5 + # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4 + # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3 + # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2 + # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1 + # \________/\___________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4 + # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0 + # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1 + # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2 + # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpaddq $H0,$T0,$H0 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu64 16*0($inp),$T3 # load input + vmovdqu64 16*4($inp),$T4 + lea 16*8($inp),$inp + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpaddq $M3,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$S4,$M2 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + + vpsrlq \$26,$D3,$H3 + vpandq $MASK,$D3,$D3 + vpaddq $H3,$D4,$H4 # h3 -> h4 + + vporq $T3,$T2,$T2 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpandq $MASK,$T2,$T2 # 2 + + vpsrlq \$26,$H4,$D4 + vpandq $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpandq $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpandq $MASK,$H2,$H2 + vpaddq $D2,$D3,$H3 # h2 -> h3 + + vpsrlq \$14,$T4,$T3 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpandq $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + sub \$128,$len + ja .Loop_avx512 + +.Ltail_avx512: + ################################################################ + # while above multiplications were by r^8 in all lanes, in last + # iteration we multiply least significant lane by r^8 and most + # significant one by r, that's why table gets shifted... + + vpsrlq \$32,$R0,$R0 # 0105020603070408 + vpsrlq \$32,$R1,$R1 + vpsrlq \$32,$R2,$R2 + vpsrlq \$32,$S3,$S3 + vpsrlq \$32,$S4,$S4 + vpsrlq \$32,$R3,$R3 + vpsrlq \$32,$R4,$R4 + vpsrlq \$32,$S1,$S1 + vpsrlq \$32,$S2,$S2 + + ################################################################ + # load either next or last 64 byte of input + lea ($inp,$len),$inp + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu 16*0($inp),%x#$T0 + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vmovdqu 16*1($inp),%x#$T1 + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0 + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1 + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpmuludq $H3,$S4,$M2 + vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # horizontal addition + + mov \$1,%eax + vpermq \$0xb1,$H3,$D3 + vpermq \$0xb1,$D4,$H4 + vpermq \$0xb1,$H0,$D0 + vpermq \$0xb1,$H1,$D1 + vpermq \$0xb1,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + kmovw %eax,%k3 + vpermq \$0x2,$H3,$D3 + vpermq \$0x2,$H4,$D4 + vpermq \$0x2,$H0,$D0 + vpermq \$0x2,$H1,$D1 + vpermq \$0x2,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + vextracti64x4 \$0x1,$H3,%y#$D3 + vextracti64x4 \$0x1,$H4,%y#$D4 + vextracti64x4 \$0x1,$H0,%y#$D0 + vextracti64x4 \$0x1,$H1,%y#$D1 + vextracti64x4 \$0x1,$H2,%y#$D2 + vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case + vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2 + vpaddq $D0,$H0,${H0}{%k3}{z} + vpaddq $D1,$H1,${H1}{%k3}{z} + vpaddq $D2,$H2,${H2}{%k3}{z} +___ +map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT)); +map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK)); +$code.=<<___; + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2 + vpand $MASK,$T1,$T1 # 1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + vpaddq $D3,$H4,$H4 # h3 -> h4 + + lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 + add \$64,$len + jnz .Ltail_avx2$suffix + + vpsubq $T2,$H2,$H2 # undo input accumulation + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) + vzeroall +___ +$code.=<<___ if ($win64); + movdqa -0xb0(%r10),%xmm6 + movdqa -0xa0(%r10),%xmm7 + movdqa -0x90(%r10),%xmm8 + movdqa -0x80(%r10),%xmm9 + movdqa -0x70(%r10),%xmm10 + movdqa -0x60(%r10),%xmm11 + movdqa -0x50(%r10),%xmm12 + movdqa -0x40(%r10),%xmm13 + movdqa -0x30(%r10),%xmm14 + movdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx512_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + ret +.cfi_endproc +___ + +} + +} + +&declare_function("poly1305_blocks_avx2", 32, 4); +poly1305_blocks_avxN(0); +&end_function("poly1305_blocks_avx2"); + +if($kernel) { + $code .= "#endif\n"; +} + +####################################################################### +if ($avx>2) { +# On entry we have input length divisible by 64. But since inner loop +# processes 128 bytes per iteration, cases when length is not divisible +# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this +# reason stack layout is kept identical to poly1305_blocks_avx2. If not +# for this tail, we wouldn't have to even allocate stack frame... + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +&declare_function("poly1305_blocks_avx512", 32, 4); +poly1305_blocks_avxN(1); +&end_function("poly1305_blocks_avx512"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if (!$kernel && $avx>3) { +######################################################################## +# VPMADD52 version using 2^44 radix. +# +# One can argue that base 2^52 would be more natural. Well, even though +# some operations would be more natural, one has to recognize couple of +# things. Base 2^52 doesn't provide advantage over base 2^44 if you look +# at amount of multiply-n-accumulate operations. Secondly, it makes it +# impossible to pre-compute multiples of 5 [referred to as s[]/sN in +# reference implementations], which means that more such operations +# would have to be performed in inner loop, which in turn makes critical +# path longer. In other words, even though base 2^44 reduction might +# look less elegant, overall critical path is actually shorter... + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^44 +# unsigned __int64 s[2]; # key value*20 base 2^44 +# unsigned __int64 r[3]; # key value base 2^44 +# struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4]; +# # r^n positions reflect +# # placement in register, not +# # memory, R[3] is R[1]*20 + +$code.=<<___; +.type poly1305_init_base2_44,\@function,3 +.align 32 +poly1305_init_base2_44: + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + +.Linit_base2_44: + lea poly1305_blocks_vpmadd52(%rip),%r10 + lea poly1305_emit_base2_44(%rip),%r11 + + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + mov \$0x00000fffffffffff,%r8 + and 8($inp),%rcx + mov \$0x00000fffffffffff,%r9 + and %rax,%r8 + shrd \$44,%rcx,%rax + mov %r8,40($ctx) # r0 + and %r9,%rax + shr \$24,%rcx + mov %rax,48($ctx) # r1 + lea (%rax,%rax,4),%rax # *5 + mov %rcx,56($ctx) # r2 + shl \$2,%rax # magic <<2 + lea (%rcx,%rcx,4),%rcx # *5 + shl \$2,%rcx # magic <<2 + mov %rax,24($ctx) # s1 + mov %rcx,32($ctx) # s2 + movq \$-1,64($ctx) # write impossible value +___ +$code.=<<___ if ($flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if ($flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax + ret +.size poly1305_init_base2_44,.-poly1305_init_base2_44 +___ +{ +my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17)); +my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21)); +my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52,\@function,4 +.align 32 +poly1305_blocks_vpmadd52: + shr \$4,$len + jz .Lno_data_vpmadd52 # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + # if powers of the key are not calculated yet, process up to 3 + # blocks with this single-block subroutine, otherwise ensure that + # length is divisible by 2 blocks and pass the rest down to next + # subroutine... + + mov \$3,%rax + mov \$1,%r10 + cmp \$4,$len # is input long + cmovae %r10,%rax + test %r8,%r8 # is power value impossible? + cmovns %r10,%rax + + and $len,%rax # is input of favourable length? + jz .Lblocks_vpmadd52_4x + + sub %rax,$len + mov \$7,%r10d + mov \$1,%r11d + kmovw %r10d,%k7 + lea .L2_44_inp_permd(%rip),%r10 + kmovw %r11d,%k1 + + vmovq $padbit,%x#$PAD + vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd + vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift + vpermq \$0xcf,$PAD,$PAD + vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask + + vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value + vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys + vmovdqu64 32($ctx),${r1r0s2}{%k7}{z} + vmovdqu64 24($ctx),${r0s2s1}{%k7}{z} + + vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt + vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft + + jmp .Loop_vpmadd52 + +.align 32 +.Loop_vpmadd52: + vmovdqu32 0($inp),%x#$T0 # load input as ----3210 + lea 16($inp),$inp + + vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110 + vpsrlvq $inp_shift,$T0,$T0 + vpandq $reduc_mask,$T0,$T0 + vporq $PAD,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo # accumulate input + + vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value + vpermq \$0b01010101,$Dlo,${H1}{%k7}{z} + vpermq \$0b10101010,$Dlo,${H2}{%k7}{z} + + vpxord $Dlo,$Dlo,$Dlo + vpxord $Dhi,$Dhi,$Dhi + + vpmadd52luq $r2r1r0,$H0,$Dlo + vpmadd52huq $r2r1r0,$H0,$Dhi + + vpmadd52luq $r1r0s2,$H1,$Dlo + vpmadd52huq $r1r0s2,$H1,$Dhi + + vpmadd52luq $r0s2s1,$H2,$Dlo + vpmadd52huq $r0s2s1,$H2,$Dhi + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword + vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword + vpandq $reduc_mask,$Dlo,$Dlo + + vpaddq $T0,$Dhi,$Dhi + + vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword + + vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-) + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word + vpandq $reduc_mask,$Dlo,$Dlo + + vpermq \$0b10010011,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + vpermq \$0b10010011,$Dlo,${T0}{%k1}{z} + + vpaddq $T0,$Dlo,$Dlo + vpsllq \$2,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + dec %rax # len-=16 + jnz .Loop_vpmadd52 + + vmovdqu64 $Dlo,0($ctx){%k7} # store hash value + + test $len,$len + jnz .Lblocks_vpmadd52_4x + +.Lno_data_vpmadd52: + ret +.size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52 +___ +} +{ +######################################################################## +# As implied by its name 4x subroutine processes 4 blocks in parallel +# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power +# and is handled in 256-bit %ymm registers. + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_4x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_4x: + shr \$4,$len + jz .Lno_data_vpmadd52_4x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + +.Lblocks_vpmadd52_4x: + vpbroadcastq $padbit,$PAD + + vmovdqa64 .Lx_mask44(%rip),$mask44 + mov \$5,%eax + vmovdqa64 .Lx_mask42(%rip),$mask42 + kmovw %eax,%k1 # used in 2x path + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Lblocks_vpmadd52_2x_do + +.Lblocks_vpmadd52_4x_do: + vpbroadcastq 64($ctx),$R0 # load 4th power of the key + vpbroadcastq 96($ctx),$R1 + vpbroadcastq 128($ctx),$R2 + vpbroadcastq 160($ctx),$S1 + +.Lblocks_vpmadd52_4x_key_loaded: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + test \$7,$len # is len 8*n? + jz .Lblocks_vpmadd52_8x + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 3-1-2-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$4,$len + jz .Ltail_vpmadd52_4x + jmp .Loop_vpmadd52_4x + ud2 + +.align 32 +.Linit_vpmadd52: + vmovq 24($ctx),%x#$S1 # load key + vmovq 56($ctx),%x#$H2 + vmovq 32($ctx),%x#$S2 + vmovq 40($ctx),%x#$R0 + vmovq 48($ctx),%x#$R1 + + vmovdqa $R0,$H0 + vmovdqa $R1,$H1 + vmovdqa $H2,$R2 + + mov \$2,%eax + +.Lmul_init_vpmadd52: + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + dec %eax + jz .Ldone_init_vpmadd52 + + vpunpcklqdq $R1,$H1,$R1 # 1,2 + vpbroadcastq %x#$H1,%x#$H1 # 2,2 + vpunpcklqdq $R2,$H2,$R2 + vpbroadcastq %x#$H2,%x#$H2 + vpunpcklqdq $R0,$H0,$R0 + vpbroadcastq %x#$H0,%x#$H0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R1,$S1,$S1 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S1,$S1 + vpsllq \$2,$S2,$S2 + + jmp .Lmul_init_vpmadd52 + ud2 + +.align 32 +.Ldone_init_vpmadd52: + vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4 + vinserti128 \$1,%x#$R2,$H2,$R2 + vinserti128 \$1,%x#$R0,$H0,$R0 + + vpermq \$0b11011000,$R1,$R1 # 1,3,2,4 + vpermq \$0b11011000,$R2,$R2 + vpermq \$0b11011000,$R0,$R0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpaddq $R1,$S1,$S1 + vpsllq \$2,$S1,$S1 + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Ldone_init_vpmadd52_2x + + vmovdqu64 $R0,64($ctx) # save key powers + vpbroadcastq %x#$R0,$R0 # broadcast 4th power + vmovdqu64 $R1,96($ctx) + vpbroadcastq %x#$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpbroadcastq %x#$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpbroadcastq %x#$S1,$S1 + + jmp .Lblocks_vpmadd52_4x_key_loaded + ud2 + +.align 32 +.Ldone_init_vpmadd52_2x: + vmovdqu64 $R0,64($ctx) # save key powers + vpsrldq \$8,$R0,$R0 # 0-1-0-2 + vmovdqu64 $R1,96($ctx) + vpsrldq \$8,$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpsrldq \$8,$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpsrldq \$8,$S1,$S1 + jmp .Lblocks_vpmadd52_2x_key_loaded + ud2 + +.align 32 +.Lblocks_vpmadd52_2x_do: + vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers + vmovdqu64 160+8($ctx),${S1}{%k1}{z} + vmovdqu64 64+8($ctx),${R0}{%k1}{z} + vmovdqu64 96+8($ctx),${R1}{%k1}{z} + +.Lblocks_vpmadd52_2x_key_loaded: + vmovdqu64 16*0($inp),$T2 # load data + vpxorq $T3,$T3,$T3 + lea 16*2($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as x-1-x-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + jmp .Ltail_vpmadd52_2x + ud2 + +.align 32 +.Loop_vpmadd52_4x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$4,$len # len-=64 + jnz .Loop_vpmadd52_4x + +.Ltail_vpmadd52_4x: + vmovdqu64 128($ctx),$R2 # load all key powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + +.Ltail_vpmadd52_2x: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + # at this point $len is + # either 4*n+2 or 0... + sub \$2,$len # len-=32 + ja .Lblocks_vpmadd52_4x_do + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_4x: + ret +.size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x +___ +} +{ +######################################################################## +# As implied by its name 8x subroutine processes 8 blocks in parallel... +# This is intermediate version, as it's used only in cases when input +# length is either 8*n, 8*n+1 or 8*n+2... + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); +my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_8x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_8x: + shr \$4,$len + jz .Lno_data_vpmadd52_8x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + vmovdqa64 .Lx_mask44(%rip),$mask44 + vmovdqa64 .Lx_mask42(%rip),$mask42 + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + +.Lblocks_vpmadd52_8x: + ################################################################ + # fist we calculate more key powers + + vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + vpbroadcastq %x#$R2,$RR2 # broadcast 4th power + vpbroadcastq %x#$R0,$RR0 + vpbroadcastq %x#$R1,$RR1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $RR2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $RR2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $RR2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $RR2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $RR2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $RR2,$R0,$D2hi + + vpmadd52luq $RR0,$R0,$D0lo + vpmadd52huq $RR0,$R0,$D0hi + vpmadd52luq $RR0,$R1,$D1lo + vpmadd52huq $RR0,$R1,$D1hi + vpmadd52luq $RR0,$R2,$D2lo + vpmadd52huq $RR0,$R2,$D2hi + + vpmadd52luq $RR1,$S2,$D0lo + vpmadd52huq $RR1,$S2,$D0hi + vpmadd52luq $RR1,$R0,$D1lo + vpmadd52huq $RR1,$R0,$D1hi + vpmadd52luq $RR1,$R1,$D2lo + vpmadd52huq $RR1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$RR0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$RR1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$RR2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + + vpsrlq \$44,$RR0,$tmp # additional step + vpandq $mask44,$RR0,$RR0 + + vpaddq $tmp,$RR1,$RR1 + + ################################################################ + # At this point Rx holds 1324 powers, RRx - 5768, and the goal + # is 15263748, which reflects how data is loaded... + + vpunpcklqdq $R2,$RR2,$T2 # 3748 + vpunpckhqdq $R2,$RR2,$R2 # 1526 + vpunpcklqdq $R0,$RR0,$T0 + vpunpckhqdq $R0,$RR0,$R0 + vpunpcklqdq $R1,$RR1,$T1 + vpunpckhqdq $R1,$RR1,$R1 +___ +######## switch to %zmm +map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); +map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2); + +$code.=<<___; + vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748 + vshufi64x2 \$0x44,$R0,$T0,$RR0 + vshufi64x2 \$0x44,$R1,$T1,$RR1 + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + + vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4 + vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4 + vpaddq $RR2,$SS2,$SS2 + vpaddq $RR1,$SS1,$SS1 + vpsllq \$2,$SS2,$SS2 + vpsllq \$2,$SS1,$SS1 + + vpbroadcastq $padbit,$PAD + vpbroadcastq %x#$mask44,$mask44 + vpbroadcastq %x#$mask42,$mask42 + + vpbroadcastq %x#$SS1,$S1 # broadcast 8th power + vpbroadcastq %x#$SS2,$S2 + vpbroadcastq %x#$RR0,$R0 + vpbroadcastq %x#$RR1,$R1 + vpbroadcastq %x#$RR2,$R2 + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 73625140 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$8,$len + jz .Ltail_vpmadd52_8x + jmp .Loop_vpmadd52_8x + +.align 32 +.Loop_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$8,$len # len-=128 + jnz .Loop_vpmadd52_8x + +.Ltail_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$SS1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$SS1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$SS2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$SS2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$RR0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$RR0,$D2hi + + vpmadd52luq $H0,$RR0,$D0lo + vpmadd52huq $H0,$RR0,$D0hi + vpmadd52luq $H0,$RR1,$D1lo + vpmadd52huq $H0,$RR1,$D1hi + vpmadd52luq $H0,$RR2,$D2lo + vpmadd52huq $H0,$RR2,$D2hi + + vpmadd52luq $H1,$SS2,$D0lo + vpmadd52huq $H1,$SS2,$D0hi + vpmadd52luq $H1,$RR0,$D1lo + vpmadd52huq $H1,$RR0,$D1hi + vpmadd52luq $H1,$RR1,$D2lo + vpmadd52huq $H1,$RR1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vextracti64x4 \$1,$D0lo,%y#$T0 + vextracti64x4 \$1,$D0hi,%y#$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vextracti64x4 \$1,$D1lo,%y#$T1 + vextracti64x4 \$1,$D1hi,%y#$H1 + vextracti64x4 \$1,$D2lo,%y#$T2 + vextracti64x4 \$1,$D2hi,%y#$H2 +___ +######## switch back to %ymm +map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); + +$code.=<<___; + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + ################################################################ + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_8x: + ret +.size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x +___ +} +$code.=<<___; +.type poly1305_emit_base2_44,\@function,3 +.align 32 +poly1305_emit_base2_44: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r9,%rax + shr \$20,%r9 + shl \$44,%rax + mov %r10,%rcx + shr \$40,%r10 + shl \$24,%rcx + + add %rax,%r8 + adc %rcx,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +.size poly1305_emit_base2_44,.-poly1305_emit_base2_44 +___ +} } } +} + +if (!$kernel) +{ # chacha20-poly1305 helpers +my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order +$code.=<<___; +.globl xor128_encrypt_n_pad +.type xor128_encrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_encrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_enc + nop +.Loop_enc_xmm: + movdqu ($inp,$otp),%xmm0 + pxor ($otp),%xmm0 + movdqu %xmm0,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_enc_xmm + + and \$15,%r10 # len % 16 + jz .Ldone_enc + +.Ltail_enc: + mov \$16,$len + sub %r10,$len + xor %eax,%eax +.Loop_enc_byte: + mov ($inp,$otp),%al + xor ($otp),%al + mov %al,($out,$otp) + mov %al,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_enc_byte + + xor %eax,%eax +.Loop_enc_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_enc_pad + +.Ldone_enc: + mov $otp,%rax + ret +.size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad + +.globl xor128_decrypt_n_pad +.type xor128_decrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_decrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_dec + nop +.Loop_dec_xmm: + movdqu ($inp,$otp),%xmm0 + movdqa ($otp),%xmm1 + pxor %xmm0,%xmm1 + movdqu %xmm1,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_dec_xmm + + pxor %xmm1,%xmm1 + and \$15,%r10 # len % 16 + jz .Ldone_dec + +.Ltail_dec: + mov \$16,$len + sub %r10,$len + xor %eax,%eax + xor %r11,%r11 +.Loop_dec_byte: + mov ($inp,$otp),%r11b + mov ($otp),%al + xor %r11b,%al + mov %al,($out,$otp) + mov %r11b,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_dec_byte + + xor %eax,%eax +.Loop_dec_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_dec_pad + +.Ldone_dec: + mov $otp,%rax + ret +.size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad +___ +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + + jmp .Lcommon_seh_tail +.size se_handler,.-se_handler + +.type avx_handler,\@abi-omnipotent +.align 16 +avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 208($context),%rax # pull context->R11 + + lea 0x50(%rax),%rsi + lea 0xf8(%rax),%rax + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size avx_handler,.-avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_end_poly1305_init_x86_64 + .rva .LSEH_info_poly1305_init_x86_64 + + .rva .LSEH_begin_poly1305_blocks_x86_64 + .rva .LSEH_end_poly1305_blocks_x86_64 + .rva .LSEH_info_poly1305_blocks_x86_64 + + .rva .LSEH_begin_poly1305_emit_x86_64 + .rva .LSEH_end_poly1305_emit_x86_64 + .rva .LSEH_info_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_poly1305_blocks_avx + .rva .Lbase2_64_avx + .rva .LSEH_info_poly1305_blocks_avx_1 + + .rva .Lbase2_64_avx + .rva .Leven_avx + .rva .LSEH_info_poly1305_blocks_avx_2 + + .rva .Leven_avx + .rva .LSEH_end_poly1305_blocks_avx + .rva .LSEH_info_poly1305_blocks_avx_3 + + .rva .LSEH_begin_poly1305_emit_avx + .rva .LSEH_end_poly1305_emit_avx + .rva .LSEH_info_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_poly1305_blocks_avx2 + .rva .Lbase2_64_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_1 + + .rva .Lbase2_64_avx2 + .rva .Leven_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_2 + + .rva .Leven_avx2 + .rva .LSEH_end_poly1305_blocks_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_3 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_poly1305_blocks_avx512 + .rva .LSEH_end_poly1305_blocks_avx512 + .rva .LSEH_info_poly1305_blocks_avx512 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_poly1305_init_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 + +.LSEH_info_poly1305_blocks_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_body,.Lblocks_epilogue + +.LSEH_info_poly1305_emit_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); +.LSEH_info_poly1305_blocks_avx_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_emit_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); +.LSEH_info_poly1305_blocks_avx2_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[] +___ +$code.=<<___ if ($avx>2); +.LSEH_info_poly1305_blocks_avx512: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[] +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split('\n',$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + s/%r([a-z]+)#d/%e$1/g; + s/%r([0-9]+)#d/%r$1d/g; + s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305.c b/net/wireguard/crypto/zinc/poly1305/poly1305.c new file mode 100644 index 0000000000000..7d373b90e011c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the Poly1305 message authenticator. + * + * Information: https://cr.yp.to/mac.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "poly1305-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "poly1305-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64) +#include "poly1305-mips-glue.c" +#else +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + return false; +} +static inline bool poly1305_blocks_arch(void *ctx, const u8 *input, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + return false; +} +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + return false; +} +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "poly1305-donna64.c" +#else +#include "poly1305-donna32.c" +#endif + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]) +{ + ctx->nonce[0] = get_unaligned_le32(&key[16]); + ctx->nonce[1] = get_unaligned_le32(&key[20]); + ctx->nonce[2] = get_unaligned_le32(&key[24]); + ctx->nonce[3] = get_unaligned_le32(&key[28]); + + if (!poly1305_init_arch(ctx->opaque, key)) + poly1305_init_generic(ctx->opaque, key); + + ctx->num = 0; +} +EXPORT_SYMBOL(poly1305_init); + +static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len, + const u32 padbit, + simd_context_t *simd_context) +{ + if (!poly1305_blocks_arch(ctx, input, len, padbit, simd_context)) + poly1305_blocks_generic(ctx, input, len, padbit); +} + +static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!poly1305_emit_arch(ctx, mac, nonce, simd_context)) + poly1305_emit_generic(ctx, mac, nonce); +} + +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context) +{ + const size_t num = ctx->num; + size_t rem; + + if (num) { + rem = POLY1305_BLOCK_SIZE - num; + if (len < rem) { + memcpy(ctx->data + num, input, len); + ctx->num = num + len; + return; + } + memcpy(ctx->data + num, input, rem); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, + simd_context); + input += rem; + len -= rem; + } + + rem = len % POLY1305_BLOCK_SIZE; + len -= rem; + + if (len >= POLY1305_BLOCK_SIZE) { + poly1305_blocks(ctx->opaque, input, len, 1, simd_context); + input += len; + } + + if (rem) + memcpy(ctx->data, input, rem); + + ctx->num = rem; +} +EXPORT_SYMBOL(poly1305_update); + +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context) +{ + size_t num = ctx->num; + + if (num) { + ctx->data[num++] = 1; + while (num < POLY1305_BLOCK_SIZE) + ctx->data[num++] = 0; + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, + simd_context); + } + + poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context); + + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL(poly1305_final); + +#include "../selftest/poly1305.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + poly1305_fpu_init(); + if (!selftest_run("poly1305", poly1305_selftest, poly1305_nobs, + ARRAY_SIZE(poly1305_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Poly1305 one-time authenticator"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/selftest/blake2s.c b/net/wireguard/crypto/zinc/selftest/blake2s.c new file mode 100644 index 0000000000000..1b5c210dc7a84 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/blake2s.c @@ -0,0 +1,2090 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x69, 0x21, 0x7a, 0x30, 0x79, 0x90, 0x80, 0x94, + 0xe1, 0x11, 0x21, 0xd0, 0x42, 0x35, 0x4a, 0x7c, + 0x1f, 0x55, 0xb6, 0x48, 0x2c, 0xa1, 0xa5, 0x1e, + 0x1b, 0x25, 0x0d, 0xfd, 0x1e, 0xd0, 0xee, 0xf9 }, + { 0xe3, 0x4d, 0x74, 0xdb, 0xaf, 0x4f, 0xf4, 0xc6, + 0xab, 0xd8, 0x71, 0xcc, 0x22, 0x04, 0x51, 0xd2, + 0xea, 0x26, 0x48, 0x84, 0x6c, 0x77, 0x57, 0xfb, + 0xaa, 0xc8, 0x2f, 0xe5, 0x1a, 0xd6, 0x4b, 0xea }, + { 0xdd, 0xad, 0x9a, 0xb1, 0x5d, 0xac, 0x45, 0x49, + 0xba, 0x42, 0xf4, 0x9d, 0x26, 0x24, 0x96, 0xbe, + 0xf6, 0xc0, 0xba, 0xe1, 0xdd, 0x34, 0x2a, 0x88, + 0x08, 0xf8, 0xea, 0x26, 0x7c, 0x6e, 0x21, 0x0c }, + { 0xe8, 0xf9, 0x1c, 0x6e, 0xf2, 0x32, 0xa0, 0x41, + 0x45, 0x2a, 0xb0, 0xe1, 0x49, 0x07, 0x0c, 0xdd, + 0x7d, 0xd1, 0x76, 0x9e, 0x75, 0xb3, 0xa5, 0x92, + 0x1b, 0xe3, 0x78, 0x76, 0xc4, 0x5c, 0x99, 0x00 }, + { 0x0c, 0xc7, 0x0e, 0x00, 0x34, 0x8b, 0x86, 0xba, + 0x29, 0x44, 0xd0, 0xc3, 0x20, 0x38, 0xb2, 0x5c, + 0x55, 0x58, 0x4f, 0x90, 0xdf, 0x23, 0x04, 0xf5, + 0x5f, 0xa3, 0x32, 0xaf, 0x5f, 0xb0, 0x1e, 0x20 }, + { 0xec, 0x19, 0x64, 0x19, 0x10, 0x87, 0xa4, 0xfe, + 0x9d, 0xf1, 0xc7, 0x95, 0x34, 0x2a, 0x02, 0xff, + 0xc1, 0x91, 0xa5, 0xb2, 0x51, 0x76, 0x48, 0x56, + 0xae, 0x5b, 0x8b, 0x57, 0x69, 0xf0, 0xc6, 0xcd }, + { 0xe1, 0xfa, 0x51, 0x61, 0x8d, 0x7d, 0xf4, 0xeb, + 0x70, 0xcf, 0x0d, 0x5a, 0x9e, 0x90, 0x6f, 0x80, + 0x6e, 0x9d, 0x19, 0xf7, 0xf4, 0xf0, 0x1e, 0x3b, + 0x62, 0x12, 0x88, 0xe4, 0x12, 0x04, 0x05, 0xd6 }, + { 0x59, 0x80, 0x01, 0xfa, 0xfb, 0xe8, 0xf9, 0x4e, + 0xc6, 0x6d, 0xc8, 0x27, 0xd0, 0x12, 0xcf, 0xcb, + 0xba, 0x22, 0x28, 0x56, 0x9f, 0x44, 0x8e, 0x89, + 0xea, 0x22, 0x08, 0xc8, 0xbf, 0x76, 0x92, 0x93 }, + { 0xc7, 0xe8, 0x87, 0xb5, 0x46, 0x62, 0x36, 0x35, + 0xe9, 0x3e, 0x04, 0x95, 0x59, 0x8f, 0x17, 0x26, + 0x82, 0x19, 0x96, 0xc2, 0x37, 0x77, 0x05, 0xb9, + 0x3a, 0x1f, 0x63, 0x6f, 0x87, 0x2b, 0xfa, 0x2d }, + { 0xc3, 0x15, 0xa4, 0x37, 0xdd, 0x28, 0x06, 0x2a, + 0x77, 0x0d, 0x48, 0x19, 0x67, 0x13, 0x6b, 0x1b, + 0x5e, 0xb8, 0x8b, 0x21, 0xee, 0x53, 0xd0, 0x32, + 0x9c, 0x58, 0x97, 0x12, 0x6e, 0x9d, 0xb0, 0x2c }, + { 0xbb, 0x47, 0x3d, 0xed, 0xdc, 0x05, 0x5f, 0xea, + 0x62, 0x28, 0xf2, 0x07, 0xda, 0x57, 0x53, 0x47, + 0xbb, 0x00, 0x40, 0x4c, 0xd3, 0x49, 0xd3, 0x8c, + 0x18, 0x02, 0x63, 0x07, 0xa2, 0x24, 0xcb, 0xff }, + { 0x68, 0x7e, 0x18, 0x73, 0xa8, 0x27, 0x75, 0x91, + 0xbb, 0x33, 0xd9, 0xad, 0xf9, 0xa1, 0x39, 0x12, + 0xef, 0xef, 0xe5, 0x57, 0xca, 0xfc, 0x39, 0xa7, + 0x95, 0x26, 0x23, 0xe4, 0x72, 0x55, 0xf1, 0x6d }, + { 0x1a, 0xc7, 0xba, 0x75, 0x4d, 0x6e, 0x2f, 0x94, + 0xe0, 0xe8, 0x6c, 0x46, 0xbf, 0xb2, 0x62, 0xab, + 0xbb, 0x74, 0xf4, 0x50, 0xef, 0x45, 0x6d, 0x6b, + 0x4d, 0x97, 0xaa, 0x80, 0xce, 0x6d, 0xa7, 0x67 }, + { 0x01, 0x2c, 0x97, 0x80, 0x96, 0x14, 0x81, 0x6b, + 0x5d, 0x94, 0x94, 0x47, 0x7d, 0x4b, 0x68, 0x7d, + 0x15, 0xb9, 0x6e, 0xb6, 0x9c, 0x0e, 0x80, 0x74, + 0xa8, 0x51, 0x6f, 0x31, 0x22, 0x4b, 0x5c, 0x98 }, + { 0x91, 0xff, 0xd2, 0x6c, 0xfa, 0x4d, 0xa5, 0x13, + 0x4c, 0x7e, 0xa2, 0x62, 0xf7, 0x88, 0x9c, 0x32, + 0x9f, 0x61, 0xf6, 0xa6, 0x57, 0x22, 0x5c, 0xc2, + 0x12, 0xf4, 0x00, 0x56, 0xd9, 0x86, 0xb3, 0xf4 }, + { 0xd9, 0x7c, 0x82, 0x8d, 0x81, 0x82, 0xa7, 0x21, + 0x80, 0xa0, 0x6a, 0x78, 0x26, 0x83, 0x30, 0x67, + 0x3f, 0x7c, 0x4e, 0x06, 0x35, 0x94, 0x7c, 0x04, + 0xc0, 0x23, 0x23, 0xfd, 0x45, 0xc0, 0xa5, 0x2d }, + { 0xef, 0xc0, 0x4c, 0xdc, 0x39, 0x1c, 0x7e, 0x91, + 0x19, 0xbd, 0x38, 0x66, 0x8a, 0x53, 0x4e, 0x65, + 0xfe, 0x31, 0x03, 0x6d, 0x6a, 0x62, 0x11, 0x2e, + 0x44, 0xeb, 0xeb, 0x11, 0xf9, 0xc5, 0x70, 0x80 }, + { 0x99, 0x2c, 0xf5, 0xc0, 0x53, 0x44, 0x2a, 0x5f, + 0xbc, 0x4f, 0xaf, 0x58, 0x3e, 0x04, 0xe5, 0x0b, + 0xb7, 0x0d, 0x2f, 0x39, 0xfb, 0xb6, 0xa5, 0x03, + 0xf8, 0x9e, 0x56, 0xa6, 0x3e, 0x18, 0x57, 0x8a }, + { 0x38, 0x64, 0x0e, 0x9f, 0x21, 0x98, 0x3e, 0x67, + 0xb5, 0x39, 0xca, 0xcc, 0xae, 0x5e, 0xcf, 0x61, + 0x5a, 0xe2, 0x76, 0x4f, 0x75, 0xa0, 0x9c, 0x9c, + 0x59, 0xb7, 0x64, 0x83, 0xc1, 0xfb, 0xc7, 0x35 }, + { 0x21, 0x3d, 0xd3, 0x4c, 0x7e, 0xfe, 0x4f, 0xb2, + 0x7a, 0x6b, 0x35, 0xf6, 0xb4, 0x00, 0x0d, 0x1f, + 0xe0, 0x32, 0x81, 0xaf, 0x3c, 0x72, 0x3e, 0x5c, + 0x9f, 0x94, 0x74, 0x7a, 0x5f, 0x31, 0xcd, 0x3b }, + { 0xec, 0x24, 0x6e, 0xee, 0xb9, 0xce, 0xd3, 0xf7, + 0xad, 0x33, 0xed, 0x28, 0x66, 0x0d, 0xd9, 0xbb, + 0x07, 0x32, 0x51, 0x3d, 0xb4, 0xe2, 0xfa, 0x27, + 0x8b, 0x60, 0xcd, 0xe3, 0x68, 0x2a, 0x4c, 0xcd }, + { 0xac, 0x9b, 0x61, 0xd4, 0x46, 0x64, 0x8c, 0x30, + 0x05, 0xd7, 0x89, 0x2b, 0xf3, 0xa8, 0x71, 0x9f, + 0x4c, 0x81, 0x81, 0xcf, 0xdc, 0xbc, 0x2b, 0x79, + 0xfe, 0xf1, 0x0a, 0x27, 0x9b, 0x91, 0x10, 0x95 }, + { 0x7b, 0xf8, 0xb2, 0x29, 0x59, 0xe3, 0x4e, 0x3a, + 0x43, 0xf7, 0x07, 0x92, 0x23, 0xe8, 0x3a, 0x97, + 0x54, 0x61, 0x7d, 0x39, 0x1e, 0x21, 0x3d, 0xfd, + 0x80, 0x8e, 0x41, 0xb9, 0xbe, 0xad, 0x4c, 0xe7 }, + { 0x68, 0xd4, 0xb5, 0xd4, 0xfa, 0x0e, 0x30, 0x2b, + 0x64, 0xcc, 0xc5, 0xaf, 0x79, 0x29, 0x13, 0xac, + 0x4c, 0x88, 0xec, 0x95, 0xc0, 0x7d, 0xdf, 0x40, + 0x69, 0x42, 0x56, 0xeb, 0x88, 0xce, 0x9f, 0x3d }, + { 0xb2, 0xc2, 0x42, 0x0f, 0x05, 0xf9, 0xab, 0xe3, + 0x63, 0x15, 0x91, 0x93, 0x36, 0xb3, 0x7e, 0x4e, + 0x0f, 0xa3, 0x3f, 0xf7, 0xe7, 0x6a, 0x49, 0x27, + 0x67, 0x00, 0x6f, 0xdb, 0x5d, 0x93, 0x54, 0x62 }, + { 0x13, 0x4f, 0x61, 0xbb, 0xd0, 0xbb, 0xb6, 0x9a, + 0xed, 0x53, 0x43, 0x90, 0x45, 0x51, 0xa3, 0xe6, + 0xc1, 0xaa, 0x7d, 0xcd, 0xd7, 0x7e, 0x90, 0x3e, + 0x70, 0x23, 0xeb, 0x7c, 0x60, 0x32, 0x0a, 0xa7 }, + { 0x46, 0x93, 0xf9, 0xbf, 0xf7, 0xd4, 0xf3, 0x98, + 0x6a, 0x7d, 0x17, 0x6e, 0x6e, 0x06, 0xf7, 0x2a, + 0xd1, 0x49, 0x0d, 0x80, 0x5c, 0x99, 0xe2, 0x53, + 0x47, 0xb8, 0xde, 0x77, 0xb4, 0xdb, 0x6d, 0x9b }, + { 0x85, 0x3e, 0x26, 0xf7, 0x41, 0x95, 0x3b, 0x0f, + 0xd5, 0xbd, 0xb4, 0x24, 0xe8, 0xab, 0x9e, 0x8b, + 0x37, 0x50, 0xea, 0xa8, 0xef, 0x61, 0xe4, 0x79, + 0x02, 0xc9, 0x1e, 0x55, 0x4e, 0x9c, 0x73, 0xb9 }, + { 0xf7, 0xde, 0x53, 0x63, 0x61, 0xab, 0xaa, 0x0e, + 0x15, 0x81, 0x56, 0xcf, 0x0e, 0xa4, 0xf6, 0x3a, + 0x99, 0xb5, 0xe4, 0x05, 0x4f, 0x8f, 0xa4, 0xc9, + 0xd4, 0x5f, 0x62, 0x85, 0xca, 0xd5, 0x56, 0x94 }, + { 0x4c, 0x23, 0x06, 0x08, 0x86, 0x0a, 0x99, 0xae, + 0x8d, 0x7b, 0xd5, 0xc2, 0xcc, 0x17, 0xfa, 0x52, + 0x09, 0x6b, 0x9a, 0x61, 0xbe, 0xdb, 0x17, 0xcb, + 0x76, 0x17, 0x86, 0x4a, 0xd2, 0x9c, 0xa7, 0xa6 }, + { 0xae, 0xb9, 0x20, 0xea, 0x87, 0x95, 0x2d, 0xad, + 0xb1, 0xfb, 0x75, 0x92, 0x91, 0xe3, 0x38, 0x81, + 0x39, 0xa8, 0x72, 0x86, 0x50, 0x01, 0x88, 0x6e, + 0xd8, 0x47, 0x52, 0xe9, 0x3c, 0x25, 0x0c, 0x2a }, + { 0xab, 0xa4, 0xad, 0x9b, 0x48, 0x0b, 0x9d, 0xf3, + 0xd0, 0x8c, 0xa5, 0xe8, 0x7b, 0x0c, 0x24, 0x40, + 0xd4, 0xe4, 0xea, 0x21, 0x22, 0x4c, 0x2e, 0xb4, + 0x2c, 0xba, 0xe4, 0x69, 0xd0, 0x89, 0xb9, 0x31 }, + { 0x05, 0x82, 0x56, 0x07, 0xd7, 0xfd, 0xf2, 0xd8, + 0x2e, 0xf4, 0xc3, 0xc8, 0xc2, 0xae, 0xa9, 0x61, + 0xad, 0x98, 0xd6, 0x0e, 0xdf, 0xf7, 0xd0, 0x18, + 0x98, 0x3e, 0x21, 0x20, 0x4c, 0x0d, 0x93, 0xd1 }, + { 0xa7, 0x42, 0xf8, 0xb6, 0xaf, 0x82, 0xd8, 0xa6, + 0xca, 0x23, 0x57, 0xc5, 0xf1, 0xcf, 0x91, 0xde, + 0xfb, 0xd0, 0x66, 0x26, 0x7d, 0x75, 0xc0, 0x48, + 0xb3, 0x52, 0x36, 0x65, 0x85, 0x02, 0x59, 0x62 }, + { 0x2b, 0xca, 0xc8, 0x95, 0x99, 0x00, 0x0b, 0x42, + 0xc9, 0x5a, 0xe2, 0x38, 0x35, 0xa7, 0x13, 0x70, + 0x4e, 0xd7, 0x97, 0x89, 0xc8, 0x4f, 0xef, 0x14, + 0x9a, 0x87, 0x4f, 0xf7, 0x33, 0xf0, 0x17, 0xa2 }, + { 0xac, 0x1e, 0xd0, 0x7d, 0x04, 0x8f, 0x10, 0x5a, + 0x9e, 0x5b, 0x7a, 0xb8, 0x5b, 0x09, 0xa4, 0x92, + 0xd5, 0xba, 0xff, 0x14, 0xb8, 0xbf, 0xb0, 0xe9, + 0xfd, 0x78, 0x94, 0x86, 0xee, 0xa2, 0xb9, 0x74 }, + { 0xe4, 0x8d, 0x0e, 0xcf, 0xaf, 0x49, 0x7d, 0x5b, + 0x27, 0xc2, 0x5d, 0x99, 0xe1, 0x56, 0xcb, 0x05, + 0x79, 0xd4, 0x40, 0xd6, 0xe3, 0x1f, 0xb6, 0x24, + 0x73, 0x69, 0x6d, 0xbf, 0x95, 0xe0, 0x10, 0xe4 }, + { 0x12, 0xa9, 0x1f, 0xad, 0xf8, 0xb2, 0x16, 0x44, + 0xfd, 0x0f, 0x93, 0x4f, 0x3c, 0x4a, 0x8f, 0x62, + 0xba, 0x86, 0x2f, 0xfd, 0x20, 0xe8, 0xe9, 0x61, + 0x15, 0x4c, 0x15, 0xc1, 0x38, 0x84, 0xed, 0x3d }, + { 0x7c, 0xbe, 0xe9, 0x6e, 0x13, 0x98, 0x97, 0xdc, + 0x98, 0xfb, 0xef, 0x3b, 0xe8, 0x1a, 0xd4, 0xd9, + 0x64, 0xd2, 0x35, 0xcb, 0x12, 0x14, 0x1f, 0xb6, + 0x67, 0x27, 0xe6, 0xe5, 0xdf, 0x73, 0xa8, 0x78 }, + { 0xeb, 0xf6, 0x6a, 0xbb, 0x59, 0x7a, 0xe5, 0x72, + 0xa7, 0x29, 0x7c, 0xb0, 0x87, 0x1e, 0x35, 0x5a, + 0xcc, 0xaf, 0xad, 0x83, 0x77, 0xb8, 0xe7, 0x8b, + 0xf1, 0x64, 0xce, 0x2a, 0x18, 0xde, 0x4b, 0xaf }, + { 0x71, 0xb9, 0x33, 0xb0, 0x7e, 0x4f, 0xf7, 0x81, + 0x8c, 0xe0, 0x59, 0xd0, 0x08, 0x82, 0x9e, 0x45, + 0x3c, 0x6f, 0xf0, 0x2e, 0xc0, 0xa7, 0xdb, 0x39, + 0x3f, 0xc2, 0xd8, 0x70, 0xf3, 0x7a, 0x72, 0x86 }, + { 0x7c, 0xf7, 0xc5, 0x13, 0x31, 0x22, 0x0b, 0x8d, + 0x3e, 0xba, 0xed, 0x9c, 0x29, 0x39, 0x8a, 0x16, + 0xd9, 0x81, 0x56, 0xe2, 0x61, 0x3c, 0xb0, 0x88, + 0xf2, 0xb0, 0xe0, 0x8a, 0x1b, 0xe4, 0xcf, 0x4f }, + { 0x3e, 0x41, 0xa1, 0x08, 0xe0, 0xf6, 0x4a, 0xd2, + 0x76, 0xb9, 0x79, 0xe1, 0xce, 0x06, 0x82, 0x79, + 0xe1, 0x6f, 0x7b, 0xc7, 0xe4, 0xaa, 0x1d, 0x21, + 0x1e, 0x17, 0xb8, 0x11, 0x61, 0xdf, 0x16, 0x02 }, + { 0x88, 0x65, 0x02, 0xa8, 0x2a, 0xb4, 0x7b, 0xa8, + 0xd8, 0x67, 0x10, 0xaa, 0x9d, 0xe3, 0xd4, 0x6e, + 0xa6, 0x5c, 0x47, 0xaf, 0x6e, 0xe8, 0xde, 0x45, + 0x0c, 0xce, 0xb8, 0xb1, 0x1b, 0x04, 0x5f, 0x50 }, + { 0xc0, 0x21, 0xbc, 0x5f, 0x09, 0x54, 0xfe, 0xe9, + 0x4f, 0x46, 0xea, 0x09, 0x48, 0x7e, 0x10, 0xa8, + 0x48, 0x40, 0xd0, 0x2f, 0x64, 0x81, 0x0b, 0xc0, + 0x8d, 0x9e, 0x55, 0x1f, 0x7d, 0x41, 0x68, 0x14 }, + { 0x20, 0x30, 0x51, 0x6e, 0x8a, 0x5f, 0xe1, 0x9a, + 0xe7, 0x9c, 0x33, 0x6f, 0xce, 0x26, 0x38, 0x2a, + 0x74, 0x9d, 0x3f, 0xd0, 0xec, 0x91, 0xe5, 0x37, + 0xd4, 0xbd, 0x23, 0x58, 0xc1, 0x2d, 0xfb, 0x22 }, + { 0x55, 0x66, 0x98, 0xda, 0xc8, 0x31, 0x7f, 0xd3, + 0x6d, 0xfb, 0xdf, 0x25, 0xa7, 0x9c, 0xb1, 0x12, + 0xd5, 0x42, 0x58, 0x60, 0x60, 0x5c, 0xba, 0xf5, + 0x07, 0xf2, 0x3b, 0xf7, 0xe9, 0xf4, 0x2a, 0xfe }, + { 0x2f, 0x86, 0x7b, 0xa6, 0x77, 0x73, 0xfd, 0xc3, + 0xe9, 0x2f, 0xce, 0xd9, 0x9a, 0x64, 0x09, 0xad, + 0x39, 0xd0, 0xb8, 0x80, 0xfd, 0xe8, 0xf1, 0x09, + 0xa8, 0x17, 0x30, 0xc4, 0x45, 0x1d, 0x01, 0x78 }, + { 0x17, 0x2e, 0xc2, 0x18, 0xf1, 0x19, 0xdf, 0xae, + 0x98, 0x89, 0x6d, 0xff, 0x29, 0xdd, 0x98, 0x76, + 0xc9, 0x4a, 0xf8, 0x74, 0x17, 0xf9, 0xae, 0x4c, + 0x70, 0x14, 0xbb, 0x4e, 0x4b, 0x96, 0xaf, 0xc7 }, + { 0x3f, 0x85, 0x81, 0x4a, 0x18, 0x19, 0x5f, 0x87, + 0x9a, 0xa9, 0x62, 0xf9, 0x5d, 0x26, 0xbd, 0x82, + 0xa2, 0x78, 0xf2, 0xb8, 0x23, 0x20, 0x21, 0x8f, + 0x6b, 0x3b, 0xd6, 0xf7, 0xf6, 0x67, 0xa6, 0xd9 }, + { 0x1b, 0x61, 0x8f, 0xba, 0xa5, 0x66, 0xb3, 0xd4, + 0x98, 0xc1, 0x2e, 0x98, 0x2c, 0x9e, 0xc5, 0x2e, + 0x4d, 0xa8, 0x5a, 0x8c, 0x54, 0xf3, 0x8f, 0x34, + 0xc0, 0x90, 0x39, 0x4f, 0x23, 0xc1, 0x84, 0xc1 }, + { 0x0c, 0x75, 0x8f, 0xb5, 0x69, 0x2f, 0xfd, 0x41, + 0xa3, 0x57, 0x5d, 0x0a, 0xf0, 0x0c, 0xc7, 0xfb, + 0xf2, 0xcb, 0xe5, 0x90, 0x5a, 0x58, 0x32, 0x3a, + 0x88, 0xae, 0x42, 0x44, 0xf6, 0xe4, 0xc9, 0x93 }, + { 0xa9, 0x31, 0x36, 0x0c, 0xad, 0x62, 0x8c, 0x7f, + 0x12, 0xa6, 0xc1, 0xc4, 0xb7, 0x53, 0xb0, 0xf4, + 0x06, 0x2a, 0xef, 0x3c, 0xe6, 0x5a, 0x1a, 0xe3, + 0xf1, 0x93, 0x69, 0xda, 0xdf, 0x3a, 0xe2, 0x3d }, + { 0xcb, 0xac, 0x7d, 0x77, 0x3b, 0x1e, 0x3b, 0x3c, + 0x66, 0x91, 0xd7, 0xab, 0xb7, 0xe9, 0xdf, 0x04, + 0x5c, 0x8b, 0xa1, 0x92, 0x68, 0xde, 0xd1, 0x53, + 0x20, 0x7f, 0x5e, 0x80, 0x43, 0x52, 0xec, 0x5d }, + { 0x23, 0xa1, 0x96, 0xd3, 0x80, 0x2e, 0xd3, 0xc1, + 0xb3, 0x84, 0x01, 0x9a, 0x82, 0x32, 0x58, 0x40, + 0xd3, 0x2f, 0x71, 0x95, 0x0c, 0x45, 0x80, 0xb0, + 0x34, 0x45, 0xe0, 0x89, 0x8e, 0x14, 0x05, 0x3c }, + { 0xf4, 0x49, 0x54, 0x70, 0xf2, 0x26, 0xc8, 0xc2, + 0x14, 0xbe, 0x08, 0xfd, 0xfa, 0xd4, 0xbc, 0x4a, + 0x2a, 0x9d, 0xbe, 0xa9, 0x13, 0x6a, 0x21, 0x0d, + 0xf0, 0xd4, 0xb6, 0x49, 0x29, 0xe6, 0xfc, 0x14 }, + { 0xe2, 0x90, 0xdd, 0x27, 0x0b, 0x46, 0x7f, 0x34, + 0xab, 0x1c, 0x00, 0x2d, 0x34, 0x0f, 0xa0, 0x16, + 0x25, 0x7f, 0xf1, 0x9e, 0x58, 0x33, 0xfd, 0xbb, + 0xf2, 0xcb, 0x40, 0x1c, 0x3b, 0x28, 0x17, 0xde }, + { 0x9f, 0xc7, 0xb5, 0xde, 0xd3, 0xc1, 0x50, 0x42, + 0xb2, 0xa6, 0x58, 0x2d, 0xc3, 0x9b, 0xe0, 0x16, + 0xd2, 0x4a, 0x68, 0x2d, 0x5e, 0x61, 0xad, 0x1e, + 0xff, 0x9c, 0x63, 0x30, 0x98, 0x48, 0xf7, 0x06 }, + { 0x8c, 0xca, 0x67, 0xa3, 0x6d, 0x17, 0xd5, 0xe6, + 0x34, 0x1c, 0xb5, 0x92, 0xfd, 0x7b, 0xef, 0x99, + 0x26, 0xc9, 0xe3, 0xaa, 0x10, 0x27, 0xea, 0x11, + 0xa7, 0xd8, 0xbd, 0x26, 0x0b, 0x57, 0x6e, 0x04 }, + { 0x40, 0x93, 0x92, 0xf5, 0x60, 0xf8, 0x68, 0x31, + 0xda, 0x43, 0x73, 0xee, 0x5e, 0x00, 0x74, 0x26, + 0x05, 0x95, 0xd7, 0xbc, 0x24, 0x18, 0x3b, 0x60, + 0xed, 0x70, 0x0d, 0x45, 0x83, 0xd3, 0xf6, 0xf0 }, + { 0x28, 0x02, 0x16, 0x5d, 0xe0, 0x90, 0x91, 0x55, + 0x46, 0xf3, 0x39, 0x8c, 0xd8, 0x49, 0x16, 0x4a, + 0x19, 0xf9, 0x2a, 0xdb, 0xc3, 0x61, 0xad, 0xc9, + 0x9b, 0x0f, 0x20, 0xc8, 0xea, 0x07, 0x10, 0x54 }, + { 0xad, 0x83, 0x91, 0x68, 0xd9, 0xf8, 0xa4, 0xbe, + 0x95, 0xba, 0x9e, 0xf9, 0xa6, 0x92, 0xf0, 0x72, + 0x56, 0xae, 0x43, 0xfe, 0x6f, 0x98, 0x64, 0xe2, + 0x90, 0x69, 0x1b, 0x02, 0x56, 0xce, 0x50, 0xa9 }, + { 0x75, 0xfd, 0xaa, 0x50, 0x38, 0xc2, 0x84, 0xb8, + 0x6d, 0x6e, 0x8a, 0xff, 0xe8, 0xb2, 0x80, 0x7e, + 0x46, 0x7b, 0x86, 0x60, 0x0e, 0x79, 0xaf, 0x36, + 0x89, 0xfb, 0xc0, 0x63, 0x28, 0xcb, 0xf8, 0x94 }, + { 0xe5, 0x7c, 0xb7, 0x94, 0x87, 0xdd, 0x57, 0x90, + 0x24, 0x32, 0xb2, 0x50, 0x73, 0x38, 0x13, 0xbd, + 0x96, 0xa8, 0x4e, 0xfc, 0xe5, 0x9f, 0x65, 0x0f, + 0xac, 0x26, 0xe6, 0x69, 0x6a, 0xef, 0xaf, 0xc3 }, + { 0x56, 0xf3, 0x4e, 0x8b, 0x96, 0x55, 0x7e, 0x90, + 0xc1, 0xf2, 0x4b, 0x52, 0xd0, 0xc8, 0x9d, 0x51, + 0x08, 0x6a, 0xcf, 0x1b, 0x00, 0xf6, 0x34, 0xcf, + 0x1d, 0xde, 0x92, 0x33, 0xb8, 0xea, 0xaa, 0x3e }, + { 0x1b, 0x53, 0xee, 0x94, 0xaa, 0xf3, 0x4e, 0x4b, + 0x15, 0x9d, 0x48, 0xde, 0x35, 0x2c, 0x7f, 0x06, + 0x61, 0xd0, 0xa4, 0x0e, 0xdf, 0xf9, 0x5a, 0x0b, + 0x16, 0x39, 0xb4, 0x09, 0x0e, 0x97, 0x44, 0x72 }, + { 0x05, 0x70, 0x5e, 0x2a, 0x81, 0x75, 0x7c, 0x14, + 0xbd, 0x38, 0x3e, 0xa9, 0x8d, 0xda, 0x54, 0x4e, + 0xb1, 0x0e, 0x6b, 0xc0, 0x7b, 0xae, 0x43, 0x5e, + 0x25, 0x18, 0xdb, 0xe1, 0x33, 0x52, 0x53, 0x75 }, + { 0xd8, 0xb2, 0x86, 0x6e, 0x8a, 0x30, 0x9d, 0xb5, + 0x3e, 0x52, 0x9e, 0xc3, 0x29, 0x11, 0xd8, 0x2f, + 0x5c, 0xa1, 0x6c, 0xff, 0x76, 0x21, 0x68, 0x91, + 0xa9, 0x67, 0x6a, 0xa3, 0x1a, 0xaa, 0x6c, 0x42 }, + { 0xf5, 0x04, 0x1c, 0x24, 0x12, 0x70, 0xeb, 0x04, + 0xc7, 0x1e, 0xc2, 0xc9, 0x5d, 0x4c, 0x38, 0xd8, + 0x03, 0xb1, 0x23, 0x7b, 0x0f, 0x29, 0xfd, 0x4d, + 0xb3, 0xeb, 0x39, 0x76, 0x69, 0xe8, 0x86, 0x99 }, + { 0x9a, 0x4c, 0xe0, 0x77, 0xc3, 0x49, 0x32, 0x2f, + 0x59, 0x5e, 0x0e, 0xe7, 0x9e, 0xd0, 0xda, 0x5f, + 0xab, 0x66, 0x75, 0x2c, 0xbf, 0xef, 0x8f, 0x87, + 0xd0, 0xe9, 0xd0, 0x72, 0x3c, 0x75, 0x30, 0xdd }, + { 0x65, 0x7b, 0x09, 0xf3, 0xd0, 0xf5, 0x2b, 0x5b, + 0x8f, 0x2f, 0x97, 0x16, 0x3a, 0x0e, 0xdf, 0x0c, + 0x04, 0xf0, 0x75, 0x40, 0x8a, 0x07, 0xbb, 0xeb, + 0x3a, 0x41, 0x01, 0xa8, 0x91, 0x99, 0x0d, 0x62 }, + { 0x1e, 0x3f, 0x7b, 0xd5, 0xa5, 0x8f, 0xa5, 0x33, + 0x34, 0x4a, 0xa8, 0xed, 0x3a, 0xc1, 0x22, 0xbb, + 0x9e, 0x70, 0xd4, 0xef, 0x50, 0xd0, 0x04, 0x53, + 0x08, 0x21, 0x94, 0x8f, 0x5f, 0xe6, 0x31, 0x5a }, + { 0x80, 0xdc, 0xcf, 0x3f, 0xd8, 0x3d, 0xfd, 0x0d, + 0x35, 0xaa, 0x28, 0x58, 0x59, 0x22, 0xab, 0x89, + 0xd5, 0x31, 0x39, 0x97, 0x67, 0x3e, 0xaf, 0x90, + 0x5c, 0xea, 0x9c, 0x0b, 0x22, 0x5c, 0x7b, 0x5f }, + { 0x8a, 0x0d, 0x0f, 0xbf, 0x63, 0x77, 0xd8, 0x3b, + 0xb0, 0x8b, 0x51, 0x4b, 0x4b, 0x1c, 0x43, 0xac, + 0xc9, 0x5d, 0x75, 0x17, 0x14, 0xf8, 0x92, 0x56, + 0x45, 0xcb, 0x6b, 0xc8, 0x56, 0xca, 0x15, 0x0a }, + { 0x9f, 0xa5, 0xb4, 0x87, 0x73, 0x8a, 0xd2, 0x84, + 0x4c, 0xc6, 0x34, 0x8a, 0x90, 0x19, 0x18, 0xf6, + 0x59, 0xa3, 0xb8, 0x9e, 0x9c, 0x0d, 0xfe, 0xea, + 0xd3, 0x0d, 0xd9, 0x4b, 0xcf, 0x42, 0xef, 0x8e }, + { 0x80, 0x83, 0x2c, 0x4a, 0x16, 0x77, 0xf5, 0xea, + 0x25, 0x60, 0xf6, 0x68, 0xe9, 0x35, 0x4d, 0xd3, + 0x69, 0x97, 0xf0, 0x37, 0x28, 0xcf, 0xa5, 0x5e, + 0x1b, 0x38, 0x33, 0x7c, 0x0c, 0x9e, 0xf8, 0x18 }, + { 0xab, 0x37, 0xdd, 0xb6, 0x83, 0x13, 0x7e, 0x74, + 0x08, 0x0d, 0x02, 0x6b, 0x59, 0x0b, 0x96, 0xae, + 0x9b, 0xb4, 0x47, 0x72, 0x2f, 0x30, 0x5a, 0x5a, + 0xc5, 0x70, 0xec, 0x1d, 0xf9, 0xb1, 0x74, 0x3c }, + { 0x3e, 0xe7, 0x35, 0xa6, 0x94, 0xc2, 0x55, 0x9b, + 0x69, 0x3a, 0xa6, 0x86, 0x29, 0x36, 0x1e, 0x15, + 0xd1, 0x22, 0x65, 0xad, 0x6a, 0x3d, 0xed, 0xf4, + 0x88, 0xb0, 0xb0, 0x0f, 0xac, 0x97, 0x54, 0xba }, + { 0xd6, 0xfc, 0xd2, 0x32, 0x19, 0xb6, 0x47, 0xe4, + 0xcb, 0xd5, 0xeb, 0x2d, 0x0a, 0xd0, 0x1e, 0xc8, + 0x83, 0x8a, 0x4b, 0x29, 0x01, 0xfc, 0x32, 0x5c, + 0xc3, 0x70, 0x19, 0x81, 0xca, 0x6c, 0x88, 0x8b }, + { 0x05, 0x20, 0xec, 0x2f, 0x5b, 0xf7, 0xa7, 0x55, + 0xda, 0xcb, 0x50, 0xc6, 0xbf, 0x23, 0x3e, 0x35, + 0x15, 0x43, 0x47, 0x63, 0xdb, 0x01, 0x39, 0xcc, + 0xd9, 0xfa, 0xef, 0xbb, 0x82, 0x07, 0x61, 0x2d }, + { 0xaf, 0xf3, 0xb7, 0x5f, 0x3f, 0x58, 0x12, 0x64, + 0xd7, 0x66, 0x16, 0x62, 0xb9, 0x2f, 0x5a, 0xd3, + 0x7c, 0x1d, 0x32, 0xbd, 0x45, 0xff, 0x81, 0xa4, + 0xed, 0x8a, 0xdc, 0x9e, 0xf3, 0x0d, 0xd9, 0x89 }, + { 0xd0, 0xdd, 0x65, 0x0b, 0xef, 0xd3, 0xba, 0x63, + 0xdc, 0x25, 0x10, 0x2c, 0x62, 0x7c, 0x92, 0x1b, + 0x9c, 0xbe, 0xb0, 0xb1, 0x30, 0x68, 0x69, 0x35, + 0xb5, 0xc9, 0x27, 0xcb, 0x7c, 0xcd, 0x5e, 0x3b }, + { 0xe1, 0x14, 0x98, 0x16, 0xb1, 0x0a, 0x85, 0x14, + 0xfb, 0x3e, 0x2c, 0xab, 0x2c, 0x08, 0xbe, 0xe9, + 0xf7, 0x3c, 0xe7, 0x62, 0x21, 0x70, 0x12, 0x46, + 0xa5, 0x89, 0xbb, 0xb6, 0x73, 0x02, 0xd8, 0xa9 }, + { 0x7d, 0xa3, 0xf4, 0x41, 0xde, 0x90, 0x54, 0x31, + 0x7e, 0x72, 0xb5, 0xdb, 0xf9, 0x79, 0xda, 0x01, + 0xe6, 0xbc, 0xee, 0xbb, 0x84, 0x78, 0xea, 0xe6, + 0xa2, 0x28, 0x49, 0xd9, 0x02, 0x92, 0x63, 0x5c }, + { 0x12, 0x30, 0xb1, 0xfc, 0x8a, 0x7d, 0x92, 0x15, + 0xed, 0xc2, 0xd4, 0xa2, 0xde, 0xcb, 0xdd, 0x0a, + 0x6e, 0x21, 0x6c, 0x92, 0x42, 0x78, 0xc9, 0x1f, + 0xc5, 0xd1, 0x0e, 0x7d, 0x60, 0x19, 0x2d, 0x94 }, + { 0x57, 0x50, 0xd7, 0x16, 0xb4, 0x80, 0x8f, 0x75, + 0x1f, 0xeb, 0xc3, 0x88, 0x06, 0xba, 0x17, 0x0b, + 0xf6, 0xd5, 0x19, 0x9a, 0x78, 0x16, 0xbe, 0x51, + 0x4e, 0x3f, 0x93, 0x2f, 0xbe, 0x0c, 0xb8, 0x71 }, + { 0x6f, 0xc5, 0x9b, 0x2f, 0x10, 0xfe, 0xba, 0x95, + 0x4a, 0xa6, 0x82, 0x0b, 0x3c, 0xa9, 0x87, 0xee, + 0x81, 0xd5, 0xcc, 0x1d, 0xa3, 0xc6, 0x3c, 0xe8, + 0x27, 0x30, 0x1c, 0x56, 0x9d, 0xfb, 0x39, 0xce }, + { 0xc7, 0xc3, 0xfe, 0x1e, 0xeb, 0xdc, 0x7b, 0x5a, + 0x93, 0x93, 0x26, 0xe8, 0xdd, 0xb8, 0x3e, 0x8b, + 0xf2, 0xb7, 0x80, 0xb6, 0x56, 0x78, 0xcb, 0x62, + 0xf2, 0x08, 0xb0, 0x40, 0xab, 0xdd, 0x35, 0xe2 }, + { 0x0c, 0x75, 0xc1, 0xa1, 0x5c, 0xf3, 0x4a, 0x31, + 0x4e, 0xe4, 0x78, 0xf4, 0xa5, 0xce, 0x0b, 0x8a, + 0x6b, 0x36, 0x52, 0x8e, 0xf7, 0xa8, 0x20, 0x69, + 0x6c, 0x3e, 0x42, 0x46, 0xc5, 0xa1, 0x58, 0x64 }, + { 0x21, 0x6d, 0xc1, 0x2a, 0x10, 0x85, 0x69, 0xa3, + 0xc7, 0xcd, 0xde, 0x4a, 0xed, 0x43, 0xa6, 0xc3, + 0x30, 0x13, 0x9d, 0xda, 0x3c, 0xcc, 0x4a, 0x10, + 0x89, 0x05, 0xdb, 0x38, 0x61, 0x89, 0x90, 0x50 }, + { 0xa5, 0x7b, 0xe6, 0xae, 0x67, 0x56, 0xf2, 0x8b, + 0x02, 0xf5, 0x9d, 0xad, 0xf7, 0xe0, 0xd7, 0xd8, + 0x80, 0x7f, 0x10, 0xfa, 0x15, 0xce, 0xd1, 0xad, + 0x35, 0x85, 0x52, 0x1a, 0x1d, 0x99, 0x5a, 0x89 }, + { 0x81, 0x6a, 0xef, 0x87, 0x59, 0x53, 0x71, 0x6c, + 0xd7, 0xa5, 0x81, 0xf7, 0x32, 0xf5, 0x3d, 0xd4, + 0x35, 0xda, 0xb6, 0x6d, 0x09, 0xc3, 0x61, 0xd2, + 0xd6, 0x59, 0x2d, 0xe1, 0x77, 0x55, 0xd8, 0xa8 }, + { 0x9a, 0x76, 0x89, 0x32, 0x26, 0x69, 0x3b, 0x6e, + 0xa9, 0x7e, 0x6a, 0x73, 0x8f, 0x9d, 0x10, 0xfb, + 0x3d, 0x0b, 0x43, 0xae, 0x0e, 0x8b, 0x7d, 0x81, + 0x23, 0xea, 0x76, 0xce, 0x97, 0x98, 0x9c, 0x7e }, + { 0x8d, 0xae, 0xdb, 0x9a, 0x27, 0x15, 0x29, 0xdb, + 0xb7, 0xdc, 0x3b, 0x60, 0x7f, 0xe5, 0xeb, 0x2d, + 0x32, 0x11, 0x77, 0x07, 0x58, 0xdd, 0x3b, 0x0a, + 0x35, 0x93, 0xd2, 0xd7, 0x95, 0x4e, 0x2d, 0x5b }, + { 0x16, 0xdb, 0xc0, 0xaa, 0x5d, 0xd2, 0xc7, 0x74, + 0xf5, 0x05, 0x10, 0x0f, 0x73, 0x37, 0x86, 0xd8, + 0xa1, 0x75, 0xfc, 0xbb, 0xb5, 0x9c, 0x43, 0xe1, + 0xfb, 0xff, 0x3e, 0x1e, 0xaf, 0x31, 0xcb, 0x4a }, + { 0x86, 0x06, 0xcb, 0x89, 0x9c, 0x6a, 0xea, 0xf5, + 0x1b, 0x9d, 0xb0, 0xfe, 0x49, 0x24, 0xa9, 0xfd, + 0x5d, 0xab, 0xc1, 0x9f, 0x88, 0x26, 0xf2, 0xbc, + 0x1c, 0x1d, 0x7d, 0xa1, 0x4d, 0x2c, 0x2c, 0x99 }, + { 0x84, 0x79, 0x73, 0x1a, 0xed, 0xa5, 0x7b, 0xd3, + 0x7e, 0xad, 0xb5, 0x1a, 0x50, 0x7e, 0x30, 0x7f, + 0x3b, 0xd9, 0x5e, 0x69, 0xdb, 0xca, 0x94, 0xf3, + 0xbc, 0x21, 0x72, 0x60, 0x66, 0xad, 0x6d, 0xfd }, + { 0x58, 0x47, 0x3a, 0x9e, 0xa8, 0x2e, 0xfa, 0x3f, + 0x3b, 0x3d, 0x8f, 0xc8, 0x3e, 0xd8, 0x86, 0x31, + 0x27, 0xb3, 0x3a, 0xe8, 0xde, 0xae, 0x63, 0x07, + 0x20, 0x1e, 0xdb, 0x6d, 0xde, 0x61, 0xde, 0x29 }, + { 0x9a, 0x92, 0x55, 0xd5, 0x3a, 0xf1, 0x16, 0xde, + 0x8b, 0xa2, 0x7c, 0xe3, 0x5b, 0x4c, 0x7e, 0x15, + 0x64, 0x06, 0x57, 0xa0, 0xfc, 0xb8, 0x88, 0xc7, + 0x0d, 0x95, 0x43, 0x1d, 0xac, 0xd8, 0xf8, 0x30 }, + { 0x9e, 0xb0, 0x5f, 0xfb, 0xa3, 0x9f, 0xd8, 0x59, + 0x6a, 0x45, 0x49, 0x3e, 0x18, 0xd2, 0x51, 0x0b, + 0xf3, 0xef, 0x06, 0x5c, 0x51, 0xd6, 0xe1, 0x3a, + 0xbe, 0x66, 0xaa, 0x57, 0xe0, 0x5c, 0xfd, 0xb7 }, + { 0x81, 0xdc, 0xc3, 0xa5, 0x05, 0xea, 0xce, 0x3f, + 0x87, 0x9d, 0x8f, 0x70, 0x27, 0x76, 0x77, 0x0f, + 0x9d, 0xf5, 0x0e, 0x52, 0x1d, 0x14, 0x28, 0xa8, + 0x5d, 0xaf, 0x04, 0xf9, 0xad, 0x21, 0x50, 0xe0 }, + { 0xe3, 0xe3, 0xc4, 0xaa, 0x3a, 0xcb, 0xbc, 0x85, + 0x33, 0x2a, 0xf9, 0xd5, 0x64, 0xbc, 0x24, 0x16, + 0x5e, 0x16, 0x87, 0xf6, 0xb1, 0xad, 0xcb, 0xfa, + 0xe7, 0x7a, 0x8f, 0x03, 0xc7, 0x2a, 0xc2, 0x8c }, + { 0x67, 0x46, 0xc8, 0x0b, 0x4e, 0xb5, 0x6a, 0xea, + 0x45, 0xe6, 0x4e, 0x72, 0x89, 0xbb, 0xa3, 0xed, + 0xbf, 0x45, 0xec, 0xf8, 0x20, 0x64, 0x81, 0xff, + 0x63, 0x02, 0x12, 0x29, 0x84, 0xcd, 0x52, 0x6a }, + { 0x2b, 0x62, 0x8e, 0x52, 0x76, 0x4d, 0x7d, 0x62, + 0xc0, 0x86, 0x8b, 0x21, 0x23, 0x57, 0xcd, 0xd1, + 0x2d, 0x91, 0x49, 0x82, 0x2f, 0x4e, 0x98, 0x45, + 0xd9, 0x18, 0xa0, 0x8d, 0x1a, 0xe9, 0x90, 0xc0 }, + { 0xe4, 0xbf, 0xe8, 0x0d, 0x58, 0xc9, 0x19, 0x94, + 0x61, 0x39, 0x09, 0xdc, 0x4b, 0x1a, 0x12, 0x49, + 0x68, 0x96, 0xc0, 0x04, 0xaf, 0x7b, 0x57, 0x01, + 0x48, 0x3d, 0xe4, 0x5d, 0x28, 0x23, 0xd7, 0x8e }, + { 0xeb, 0xb4, 0xba, 0x15, 0x0c, 0xef, 0x27, 0x34, + 0x34, 0x5b, 0x5d, 0x64, 0x1b, 0xbe, 0xd0, 0x3a, + 0x21, 0xea, 0xfa, 0xe9, 0x33, 0xc9, 0x9e, 0x00, + 0x92, 0x12, 0xef, 0x04, 0x57, 0x4a, 0x85, 0x30 }, + { 0x39, 0x66, 0xec, 0x73, 0xb1, 0x54, 0xac, 0xc6, + 0x97, 0xac, 0x5c, 0xf5, 0xb2, 0x4b, 0x40, 0xbd, + 0xb0, 0xdb, 0x9e, 0x39, 0x88, 0x36, 0xd7, 0x6d, + 0x4b, 0x88, 0x0e, 0x3b, 0x2a, 0xf1, 0xaa, 0x27 }, + { 0xef, 0x7e, 0x48, 0x31, 0xb3, 0xa8, 0x46, 0x36, + 0x51, 0x8d, 0x6e, 0x4b, 0xfc, 0xe6, 0x4a, 0x43, + 0xdb, 0x2a, 0x5d, 0xda, 0x9c, 0xca, 0x2b, 0x44, + 0xf3, 0x90, 0x33, 0xbd, 0xc4, 0x0d, 0x62, 0x43 }, + { 0x7a, 0xbf, 0x6a, 0xcf, 0x5c, 0x8e, 0x54, 0x9d, + 0xdb, 0xb1, 0x5a, 0xe8, 0xd8, 0xb3, 0x88, 0xc1, + 0xc1, 0x97, 0xe6, 0x98, 0x73, 0x7c, 0x97, 0x85, + 0x50, 0x1e, 0xd1, 0xf9, 0x49, 0x30, 0xb7, 0xd9 }, + { 0x88, 0x01, 0x8d, 0xed, 0x66, 0x81, 0x3f, 0x0c, + 0xa9, 0x5d, 0xef, 0x47, 0x4c, 0x63, 0x06, 0x92, + 0x01, 0x99, 0x67, 0xb9, 0xe3, 0x68, 0x88, 0xda, + 0xdd, 0x94, 0x12, 0x47, 0x19, 0xb6, 0x82, 0xf6 }, + { 0x39, 0x30, 0x87, 0x6b, 0x9f, 0xc7, 0x52, 0x90, + 0x36, 0xb0, 0x08, 0xb1, 0xb8, 0xbb, 0x99, 0x75, + 0x22, 0xa4, 0x41, 0x63, 0x5a, 0x0c, 0x25, 0xec, + 0x02, 0xfb, 0x6d, 0x90, 0x26, 0xe5, 0x5a, 0x97 }, + { 0x0a, 0x40, 0x49, 0xd5, 0x7e, 0x83, 0x3b, 0x56, + 0x95, 0xfa, 0xc9, 0x3d, 0xd1, 0xfb, 0xef, 0x31, + 0x66, 0xb4, 0x4b, 0x12, 0xad, 0x11, 0x24, 0x86, + 0x62, 0x38, 0x3a, 0xe0, 0x51, 0xe1, 0x58, 0x27 }, + { 0x81, 0xdc, 0xc0, 0x67, 0x8b, 0xb6, 0xa7, 0x65, + 0xe4, 0x8c, 0x32, 0x09, 0x65, 0x4f, 0xe9, 0x00, + 0x89, 0xce, 0x44, 0xff, 0x56, 0x18, 0x47, 0x7e, + 0x39, 0xab, 0x28, 0x64, 0x76, 0xdf, 0x05, 0x2b }, + { 0xe6, 0x9b, 0x3a, 0x36, 0xa4, 0x46, 0x19, 0x12, + 0xdc, 0x08, 0x34, 0x6b, 0x11, 0xdd, 0xcb, 0x9d, + 0xb7, 0x96, 0xf8, 0x85, 0xfd, 0x01, 0x93, 0x6e, + 0x66, 0x2f, 0xe2, 0x92, 0x97, 0xb0, 0x99, 0xa4 }, + { 0x5a, 0xc6, 0x50, 0x3b, 0x0d, 0x8d, 0xa6, 0x91, + 0x76, 0x46, 0xe6, 0xdc, 0xc8, 0x7e, 0xdc, 0x58, + 0xe9, 0x42, 0x45, 0x32, 0x4c, 0xc2, 0x04, 0xf4, + 0xdd, 0x4a, 0xf0, 0x15, 0x63, 0xac, 0xd4, 0x27 }, + { 0xdf, 0x6d, 0xda, 0x21, 0x35, 0x9a, 0x30, 0xbc, + 0x27, 0x17, 0x80, 0x97, 0x1c, 0x1a, 0xbd, 0x56, + 0xa6, 0xef, 0x16, 0x7e, 0x48, 0x08, 0x87, 0x88, + 0x8e, 0x73, 0xa8, 0x6d, 0x3b, 0xf6, 0x05, 0xe9 }, + { 0xe8, 0xe6, 0xe4, 0x70, 0x71, 0xe7, 0xb7, 0xdf, + 0x25, 0x80, 0xf2, 0x25, 0xcf, 0xbb, 0xed, 0xf8, + 0x4c, 0xe6, 0x77, 0x46, 0x62, 0x66, 0x28, 0xd3, + 0x30, 0x97, 0xe4, 0xb7, 0xdc, 0x57, 0x11, 0x07 }, + { 0x53, 0xe4, 0x0e, 0xad, 0x62, 0x05, 0x1e, 0x19, + 0xcb, 0x9b, 0xa8, 0x13, 0x3e, 0x3e, 0x5c, 0x1c, + 0xe0, 0x0d, 0xdc, 0xad, 0x8a, 0xcf, 0x34, 0x2a, + 0x22, 0x43, 0x60, 0xb0, 0xac, 0xc1, 0x47, 0x77 }, + { 0x9c, 0xcd, 0x53, 0xfe, 0x80, 0xbe, 0x78, 0x6a, + 0xa9, 0x84, 0x63, 0x84, 0x62, 0xfb, 0x28, 0xaf, + 0xdf, 0x12, 0x2b, 0x34, 0xd7, 0x8f, 0x46, 0x87, + 0xec, 0x63, 0x2b, 0xb1, 0x9d, 0xe2, 0x37, 0x1a }, + { 0xcb, 0xd4, 0x80, 0x52, 0xc4, 0x8d, 0x78, 0x84, + 0x66, 0xa3, 0xe8, 0x11, 0x8c, 0x56, 0xc9, 0x7f, + 0xe1, 0x46, 0xe5, 0x54, 0x6f, 0xaa, 0xf9, 0x3e, + 0x2b, 0xc3, 0xc4, 0x7e, 0x45, 0x93, 0x97, 0x53 }, + { 0x25, 0x68, 0x83, 0xb1, 0x4e, 0x2a, 0xf4, 0x4d, + 0xad, 0xb2, 0x8e, 0x1b, 0x34, 0xb2, 0xac, 0x0f, + 0x0f, 0x4c, 0x91, 0xc3, 0x4e, 0xc9, 0x16, 0x9e, + 0x29, 0x03, 0x61, 0x58, 0xac, 0xaa, 0x95, 0xb9 }, + { 0x44, 0x71, 0xb9, 0x1a, 0xb4, 0x2d, 0xb7, 0xc4, + 0xdd, 0x84, 0x90, 0xab, 0x95, 0xa2, 0xee, 0x8d, + 0x04, 0xe3, 0xef, 0x5c, 0x3d, 0x6f, 0xc7, 0x1a, + 0xc7, 0x4b, 0x2b, 0x26, 0x91, 0x4d, 0x16, 0x41 }, + { 0xa5, 0xeb, 0x08, 0x03, 0x8f, 0x8f, 0x11, 0x55, + 0xed, 0x86, 0xe6, 0x31, 0x90, 0x6f, 0xc1, 0x30, + 0x95, 0xf6, 0xbb, 0xa4, 0x1d, 0xe5, 0xd4, 0xe7, + 0x95, 0x75, 0x8e, 0xc8, 0xc8, 0xdf, 0x8a, 0xf1 }, + { 0xdc, 0x1d, 0xb6, 0x4e, 0xd8, 0xb4, 0x8a, 0x91, + 0x0e, 0x06, 0x0a, 0x6b, 0x86, 0x63, 0x74, 0xc5, + 0x78, 0x78, 0x4e, 0x9a, 0xc4, 0x9a, 0xb2, 0x77, + 0x40, 0x92, 0xac, 0x71, 0x50, 0x19, 0x34, 0xac }, + { 0x28, 0x54, 0x13, 0xb2, 0xf2, 0xee, 0x87, 0x3d, + 0x34, 0x31, 0x9e, 0xe0, 0xbb, 0xfb, 0xb9, 0x0f, + 0x32, 0xda, 0x43, 0x4c, 0xc8, 0x7e, 0x3d, 0xb5, + 0xed, 0x12, 0x1b, 0xb3, 0x98, 0xed, 0x96, 0x4b }, + { 0x02, 0x16, 0xe0, 0xf8, 0x1f, 0x75, 0x0f, 0x26, + 0xf1, 0x99, 0x8b, 0xc3, 0x93, 0x4e, 0x3e, 0x12, + 0x4c, 0x99, 0x45, 0xe6, 0x85, 0xa6, 0x0b, 0x25, + 0xe8, 0xfb, 0xd9, 0x62, 0x5a, 0xb6, 0xb5, 0x99 }, + { 0x38, 0xc4, 0x10, 0xf5, 0xb9, 0xd4, 0x07, 0x20, + 0x50, 0x75, 0x5b, 0x31, 0xdc, 0xa8, 0x9f, 0xd5, + 0x39, 0x5c, 0x67, 0x85, 0xee, 0xb3, 0xd7, 0x90, + 0xf3, 0x20, 0xff, 0x94, 0x1c, 0x5a, 0x93, 0xbf }, + { 0xf1, 0x84, 0x17, 0xb3, 0x9d, 0x61, 0x7a, 0xb1, + 0xc1, 0x8f, 0xdf, 0x91, 0xeb, 0xd0, 0xfc, 0x6d, + 0x55, 0x16, 0xbb, 0x34, 0xcf, 0x39, 0x36, 0x40, + 0x37, 0xbc, 0xe8, 0x1f, 0xa0, 0x4c, 0xec, 0xb1 }, + { 0x1f, 0xa8, 0x77, 0xde, 0x67, 0x25, 0x9d, 0x19, + 0x86, 0x3a, 0x2a, 0x34, 0xbc, 0xc6, 0x96, 0x2a, + 0x2b, 0x25, 0xfc, 0xbf, 0x5c, 0xbe, 0xcd, 0x7e, + 0xde, 0x8f, 0x1f, 0xa3, 0x66, 0x88, 0xa7, 0x96 }, + { 0x5b, 0xd1, 0x69, 0xe6, 0x7c, 0x82, 0xc2, 0xc2, + 0xe9, 0x8e, 0xf7, 0x00, 0x8b, 0xdf, 0x26, 0x1f, + 0x2d, 0xdf, 0x30, 0xb1, 0xc0, 0x0f, 0x9e, 0x7f, + 0x27, 0x5b, 0xb3, 0xe8, 0xa2, 0x8d, 0xc9, 0xa2 }, + { 0xc8, 0x0a, 0xbe, 0xeb, 0xb6, 0x69, 0xad, 0x5d, + 0xee, 0xb5, 0xf5, 0xec, 0x8e, 0xa6, 0xb7, 0xa0, + 0x5d, 0xdf, 0x7d, 0x31, 0xec, 0x4c, 0x0a, 0x2e, + 0xe2, 0x0b, 0x0b, 0x98, 0xca, 0xec, 0x67, 0x46 }, + { 0xe7, 0x6d, 0x3f, 0xbd, 0xa5, 0xba, 0x37, 0x4e, + 0x6b, 0xf8, 0xe5, 0x0f, 0xad, 0xc3, 0xbb, 0xb9, + 0xba, 0x5c, 0x20, 0x6e, 0xbd, 0xec, 0x89, 0xa3, + 0xa5, 0x4c, 0xf3, 0xdd, 0x84, 0xa0, 0x70, 0x16 }, + { 0x7b, 0xba, 0x9d, 0xc5, 0xb5, 0xdb, 0x20, 0x71, + 0xd1, 0x77, 0x52, 0xb1, 0x04, 0x4c, 0x1e, 0xce, + 0xd9, 0x6a, 0xaf, 0x2d, 0xd4, 0x6e, 0x9b, 0x43, + 0x37, 0x50, 0xe8, 0xea, 0x0d, 0xcc, 0x18, 0x70 }, + { 0xf2, 0x9b, 0x1b, 0x1a, 0xb9, 0xba, 0xb1, 0x63, + 0x01, 0x8e, 0xe3, 0xda, 0x15, 0x23, 0x2c, 0xca, + 0x78, 0xec, 0x52, 0xdb, 0xc3, 0x4e, 0xda, 0x5b, + 0x82, 0x2e, 0xc1, 0xd8, 0x0f, 0xc2, 0x1b, 0xd0 }, + { 0x9e, 0xe3, 0xe3, 0xe7, 0xe9, 0x00, 0xf1, 0xe1, + 0x1d, 0x30, 0x8c, 0x4b, 0x2b, 0x30, 0x76, 0xd2, + 0x72, 0xcf, 0x70, 0x12, 0x4f, 0x9f, 0x51, 0xe1, + 0xda, 0x60, 0xf3, 0x78, 0x46, 0xcd, 0xd2, 0xf4 }, + { 0x70, 0xea, 0x3b, 0x01, 0x76, 0x92, 0x7d, 0x90, + 0x96, 0xa1, 0x85, 0x08, 0xcd, 0x12, 0x3a, 0x29, + 0x03, 0x25, 0x92, 0x0a, 0x9d, 0x00, 0xa8, 0x9b, + 0x5d, 0xe0, 0x42, 0x73, 0xfb, 0xc7, 0x6b, 0x85 }, + { 0x67, 0xde, 0x25, 0xc0, 0x2a, 0x4a, 0xab, 0xa2, + 0x3b, 0xdc, 0x97, 0x3c, 0x8b, 0xb0, 0xb5, 0x79, + 0x6d, 0x47, 0xcc, 0x06, 0x59, 0xd4, 0x3d, 0xff, + 0x1f, 0x97, 0xde, 0x17, 0x49, 0x63, 0xb6, 0x8e }, + { 0xb2, 0x16, 0x8e, 0x4e, 0x0f, 0x18, 0xb0, 0xe6, + 0x41, 0x00, 0xb5, 0x17, 0xed, 0x95, 0x25, 0x7d, + 0x73, 0xf0, 0x62, 0x0d, 0xf8, 0x85, 0xc1, 0x3d, + 0x2e, 0xcf, 0x79, 0x36, 0x7b, 0x38, 0x4c, 0xee }, + { 0x2e, 0x7d, 0xec, 0x24, 0x28, 0x85, 0x3b, 0x2c, + 0x71, 0x76, 0x07, 0x45, 0x54, 0x1f, 0x7a, 0xfe, + 0x98, 0x25, 0xb5, 0xdd, 0x77, 0xdf, 0x06, 0x51, + 0x1d, 0x84, 0x41, 0xa9, 0x4b, 0xac, 0xc9, 0x27 }, + { 0xca, 0x9f, 0xfa, 0xc4, 0xc4, 0x3f, 0x0b, 0x48, + 0x46, 0x1d, 0xc5, 0xc2, 0x63, 0xbe, 0xa3, 0xf6, + 0xf0, 0x06, 0x11, 0xce, 0xac, 0xab, 0xf6, 0xf8, + 0x95, 0xba, 0x2b, 0x01, 0x01, 0xdb, 0xb6, 0x8d }, + { 0x74, 0x10, 0xd4, 0x2d, 0x8f, 0xd1, 0xd5, 0xe9, + 0xd2, 0xf5, 0x81, 0x5c, 0xb9, 0x34, 0x17, 0x99, + 0x88, 0x28, 0xef, 0x3c, 0x42, 0x30, 0xbf, 0xbd, + 0x41, 0x2d, 0xf0, 0xa4, 0xa7, 0xa2, 0x50, 0x7a }, + { 0x50, 0x10, 0xf6, 0x84, 0x51, 0x6d, 0xcc, 0xd0, + 0xb6, 0xee, 0x08, 0x52, 0xc2, 0x51, 0x2b, 0x4d, + 0xc0, 0x06, 0x6c, 0xf0, 0xd5, 0x6f, 0x35, 0x30, + 0x29, 0x78, 0xdb, 0x8a, 0xe3, 0x2c, 0x6a, 0x81 }, + { 0xac, 0xaa, 0xb5, 0x85, 0xf7, 0xb7, 0x9b, 0x71, + 0x99, 0x35, 0xce, 0xb8, 0x95, 0x23, 0xdd, 0xc5, + 0x48, 0x27, 0xf7, 0x5c, 0x56, 0x88, 0x38, 0x56, + 0x15, 0x4a, 0x56, 0xcd, 0xcd, 0x5e, 0xe9, 0x88 }, + { 0x66, 0x6d, 0xe5, 0xd1, 0x44, 0x0f, 0xee, 0x73, + 0x31, 0xaa, 0xf0, 0x12, 0x3a, 0x62, 0xef, 0x2d, + 0x8b, 0xa5, 0x74, 0x53, 0xa0, 0x76, 0x96, 0x35, + 0xac, 0x6c, 0xd0, 0x1e, 0x63, 0x3f, 0x77, 0x12 }, + { 0xa6, 0xf9, 0x86, 0x58, 0xf6, 0xea, 0xba, 0xf9, + 0x02, 0xd8, 0xb3, 0x87, 0x1a, 0x4b, 0x10, 0x1d, + 0x16, 0x19, 0x6e, 0x8a, 0x4b, 0x24, 0x1e, 0x15, + 0x58, 0xfe, 0x29, 0x96, 0x6e, 0x10, 0x3e, 0x8d }, + { 0x89, 0x15, 0x46, 0xa8, 0xb2, 0x9f, 0x30, 0x47, + 0xdd, 0xcf, 0xe5, 0xb0, 0x0e, 0x45, 0xfd, 0x55, + 0x75, 0x63, 0x73, 0x10, 0x5e, 0xa8, 0x63, 0x7d, + 0xfc, 0xff, 0x54, 0x7b, 0x6e, 0xa9, 0x53, 0x5f }, + { 0x18, 0xdf, 0xbc, 0x1a, 0xc5, 0xd2, 0x5b, 0x07, + 0x61, 0x13, 0x7d, 0xbd, 0x22, 0xc1, 0x7c, 0x82, + 0x9d, 0x0f, 0x0e, 0xf1, 0xd8, 0x23, 0x44, 0xe9, + 0xc8, 0x9c, 0x28, 0x66, 0x94, 0xda, 0x24, 0xe8 }, + { 0xb5, 0x4b, 0x9b, 0x67, 0xf8, 0xfe, 0xd5, 0x4b, + 0xbf, 0x5a, 0x26, 0x66, 0xdb, 0xdf, 0x4b, 0x23, + 0xcf, 0xf1, 0xd1, 0xb6, 0xf4, 0xaf, 0xc9, 0x85, + 0xb2, 0xe6, 0xd3, 0x30, 0x5a, 0x9f, 0xf8, 0x0f }, + { 0x7d, 0xb4, 0x42, 0xe1, 0x32, 0xba, 0x59, 0xbc, + 0x12, 0x89, 0xaa, 0x98, 0xb0, 0xd3, 0xe8, 0x06, + 0x00, 0x4f, 0x8e, 0xc1, 0x28, 0x11, 0xaf, 0x1e, + 0x2e, 0x33, 0xc6, 0x9b, 0xfd, 0xe7, 0x29, 0xe1 }, + { 0x25, 0x0f, 0x37, 0xcd, 0xc1, 0x5e, 0x81, 0x7d, + 0x2f, 0x16, 0x0d, 0x99, 0x56, 0xc7, 0x1f, 0xe3, + 0xeb, 0x5d, 0xb7, 0x45, 0x56, 0xe4, 0xad, 0xf9, + 0xa4, 0xff, 0xaf, 0xba, 0x74, 0x01, 0x03, 0x96 }, + { 0x4a, 0xb8, 0xa3, 0xdd, 0x1d, 0xdf, 0x8a, 0xd4, + 0x3d, 0xab, 0x13, 0xa2, 0x7f, 0x66, 0xa6, 0x54, + 0x4f, 0x29, 0x05, 0x97, 0xfa, 0x96, 0x04, 0x0e, + 0x0e, 0x1d, 0xb9, 0x26, 0x3a, 0xa4, 0x79, 0xf8 }, + { 0xee, 0x61, 0x72, 0x7a, 0x07, 0x66, 0xdf, 0x93, + 0x9c, 0xcd, 0xc8, 0x60, 0x33, 0x40, 0x44, 0xc7, + 0x9a, 0x3c, 0x9b, 0x15, 0x62, 0x00, 0xbc, 0x3a, + 0xa3, 0x29, 0x73, 0x48, 0x3d, 0x83, 0x41, 0xae }, + { 0x3f, 0x68, 0xc7, 0xec, 0x63, 0xac, 0x11, 0xeb, + 0xb9, 0x8f, 0x94, 0xb3, 0x39, 0xb0, 0x5c, 0x10, + 0x49, 0x84, 0xfd, 0xa5, 0x01, 0x03, 0x06, 0x01, + 0x44, 0xe5, 0xa2, 0xbf, 0xcc, 0xc9, 0xda, 0x95 }, + { 0x05, 0x6f, 0x29, 0x81, 0x6b, 0x8a, 0xf8, 0xf5, + 0x66, 0x82, 0xbc, 0x4d, 0x7c, 0xf0, 0x94, 0x11, + 0x1d, 0xa7, 0x73, 0x3e, 0x72, 0x6c, 0xd1, 0x3d, + 0x6b, 0x3e, 0x8e, 0xa0, 0x3e, 0x92, 0xa0, 0xd5 }, + { 0xf5, 0xec, 0x43, 0xa2, 0x8a, 0xcb, 0xef, 0xf1, + 0xf3, 0x31, 0x8a, 0x5b, 0xca, 0xc7, 0xc6, 0x6d, + 0xdb, 0x52, 0x30, 0xb7, 0x9d, 0xb2, 0xd1, 0x05, + 0xbc, 0xbe, 0x15, 0xf3, 0xc1, 0x14, 0x8d, 0x69 }, + { 0x2a, 0x69, 0x60, 0xad, 0x1d, 0x8d, 0xd5, 0x47, + 0x55, 0x5c, 0xfb, 0xd5, 0xe4, 0x60, 0x0f, 0x1e, + 0xaa, 0x1c, 0x8e, 0xda, 0x34, 0xde, 0x03, 0x74, + 0xec, 0x4a, 0x26, 0xea, 0xaa, 0xa3, 0x3b, 0x4e }, + { 0xdc, 0xc1, 0xea, 0x7b, 0xaa, 0xb9, 0x33, 0x84, + 0xf7, 0x6b, 0x79, 0x68, 0x66, 0x19, 0x97, 0x54, + 0x74, 0x2f, 0x7b, 0x96, 0xd6, 0xb4, 0xc1, 0x20, + 0x16, 0x5c, 0x04, 0xa6, 0xc4, 0xf5, 0xce, 0x10 }, + { 0x13, 0xd5, 0xdf, 0x17, 0x92, 0x21, 0x37, 0x9c, + 0x6a, 0x78, 0xc0, 0x7c, 0x79, 0x3f, 0xf5, 0x34, + 0x87, 0xca, 0xe6, 0xbf, 0x9f, 0xe8, 0x82, 0x54, + 0x1a, 0xb0, 0xe7, 0x35, 0xe3, 0xea, 0xda, 0x3b }, + { 0x8c, 0x59, 0xe4, 0x40, 0x76, 0x41, 0xa0, 0x1e, + 0x8f, 0xf9, 0x1f, 0x99, 0x80, 0xdc, 0x23, 0x6f, + 0x4e, 0xcd, 0x6f, 0xcf, 0x52, 0x58, 0x9a, 0x09, + 0x9a, 0x96, 0x16, 0x33, 0x96, 0x77, 0x14, 0xe1 }, + { 0x83, 0x3b, 0x1a, 0xc6, 0xa2, 0x51, 0xfd, 0x08, + 0xfd, 0x6d, 0x90, 0x8f, 0xea, 0x2a, 0x4e, 0xe1, + 0xe0, 0x40, 0xbc, 0xa9, 0x3f, 0xc1, 0xa3, 0x8e, + 0xc3, 0x82, 0x0e, 0x0c, 0x10, 0xbd, 0x82, 0xea }, + { 0xa2, 0x44, 0xf9, 0x27, 0xf3, 0xb4, 0x0b, 0x8f, + 0x6c, 0x39, 0x15, 0x70, 0xc7, 0x65, 0x41, 0x8f, + 0x2f, 0x6e, 0x70, 0x8e, 0xac, 0x90, 0x06, 0xc5, + 0x1a, 0x7f, 0xef, 0xf4, 0xaf, 0x3b, 0x2b, 0x9e }, + { 0x3d, 0x99, 0xed, 0x95, 0x50, 0xcf, 0x11, 0x96, + 0xe6, 0xc4, 0xd2, 0x0c, 0x25, 0x96, 0x20, 0xf8, + 0x58, 0xc3, 0xd7, 0x03, 0x37, 0x4c, 0x12, 0x8c, + 0xe7, 0xb5, 0x90, 0x31, 0x0c, 0x83, 0x04, 0x6d }, + { 0x2b, 0x35, 0xc4, 0x7d, 0x7b, 0x87, 0x76, 0x1f, + 0x0a, 0xe4, 0x3a, 0xc5, 0x6a, 0xc2, 0x7b, 0x9f, + 0x25, 0x83, 0x03, 0x67, 0xb5, 0x95, 0xbe, 0x8c, + 0x24, 0x0e, 0x94, 0x60, 0x0c, 0x6e, 0x33, 0x12 }, + { 0x5d, 0x11, 0xed, 0x37, 0xd2, 0x4d, 0xc7, 0x67, + 0x30, 0x5c, 0xb7, 0xe1, 0x46, 0x7d, 0x87, 0xc0, + 0x65, 0xac, 0x4b, 0xc8, 0xa4, 0x26, 0xde, 0x38, + 0x99, 0x1f, 0xf5, 0x9a, 0xa8, 0x73, 0x5d, 0x02 }, + { 0xb8, 0x36, 0x47, 0x8e, 0x1c, 0xa0, 0x64, 0x0d, + 0xce, 0x6f, 0xd9, 0x10, 0xa5, 0x09, 0x62, 0x72, + 0xc8, 0x33, 0x09, 0x90, 0xcd, 0x97, 0x86, 0x4a, + 0xc2, 0xbf, 0x14, 0xef, 0x6b, 0x23, 0x91, 0x4a }, + { 0x91, 0x00, 0xf9, 0x46, 0xd6, 0xcc, 0xde, 0x3a, + 0x59, 0x7f, 0x90, 0xd3, 0x9f, 0xc1, 0x21, 0x5b, + 0xad, 0xdc, 0x74, 0x13, 0x64, 0x3d, 0x85, 0xc2, + 0x1c, 0x3e, 0xee, 0x5d, 0x2d, 0xd3, 0x28, 0x94 }, + { 0xda, 0x70, 0xee, 0xdd, 0x23, 0xe6, 0x63, 0xaa, + 0x1a, 0x74, 0xb9, 0x76, 0x69, 0x35, 0xb4, 0x79, + 0x22, 0x2a, 0x72, 0xaf, 0xba, 0x5c, 0x79, 0x51, + 0x58, 0xda, 0xd4, 0x1a, 0x3b, 0xd7, 0x7e, 0x40 }, + { 0xf0, 0x67, 0xed, 0x6a, 0x0d, 0xbd, 0x43, 0xaa, + 0x0a, 0x92, 0x54, 0xe6, 0x9f, 0xd6, 0x6b, 0xdd, + 0x8a, 0xcb, 0x87, 0xde, 0x93, 0x6c, 0x25, 0x8c, + 0xfb, 0x02, 0x28, 0x5f, 0x2c, 0x11, 0xfa, 0x79 }, + { 0x71, 0x5c, 0x99, 0xc7, 0xd5, 0x75, 0x80, 0xcf, + 0x97, 0x53, 0xb4, 0xc1, 0xd7, 0x95, 0xe4, 0x5a, + 0x83, 0xfb, 0xb2, 0x28, 0xc0, 0xd3, 0x6f, 0xbe, + 0x20, 0xfa, 0xf3, 0x9b, 0xdd, 0x6d, 0x4e, 0x85 }, + { 0xe4, 0x57, 0xd6, 0xad, 0x1e, 0x67, 0xcb, 0x9b, + 0xbd, 0x17, 0xcb, 0xd6, 0x98, 0xfa, 0x6d, 0x7d, + 0xae, 0x0c, 0x9b, 0x7a, 0xd6, 0xcb, 0xd6, 0x53, + 0x96, 0x34, 0xe3, 0x2a, 0x71, 0x9c, 0x84, 0x92 }, + { 0xec, 0xe3, 0xea, 0x81, 0x03, 0xe0, 0x24, 0x83, + 0xc6, 0x4a, 0x70, 0xa4, 0xbd, 0xce, 0xe8, 0xce, + 0xb6, 0x27, 0x8f, 0x25, 0x33, 0xf3, 0xf4, 0x8d, + 0xbe, 0xed, 0xfb, 0xa9, 0x45, 0x31, 0xd4, 0xae }, + { 0x38, 0x8a, 0xa5, 0xd3, 0x66, 0x7a, 0x97, 0xc6, + 0x8d, 0x3d, 0x56, 0xf8, 0xf3, 0xee, 0x8d, 0x3d, + 0x36, 0x09, 0x1f, 0x17, 0xfe, 0x5d, 0x1b, 0x0d, + 0x5d, 0x84, 0xc9, 0x3b, 0x2f, 0xfe, 0x40, 0xbd }, + { 0x8b, 0x6b, 0x31, 0xb9, 0xad, 0x7c, 0x3d, 0x5c, + 0xd8, 0x4b, 0xf9, 0x89, 0x47, 0xb9, 0xcd, 0xb5, + 0x9d, 0xf8, 0xa2, 0x5f, 0xf7, 0x38, 0x10, 0x10, + 0x13, 0xbe, 0x4f, 0xd6, 0x5e, 0x1d, 0xd1, 0xa3 }, + { 0x06, 0x62, 0x91, 0xf6, 0xbb, 0xd2, 0x5f, 0x3c, + 0x85, 0x3d, 0xb7, 0xd8, 0xb9, 0x5c, 0x9a, 0x1c, + 0xfb, 0x9b, 0xf1, 0xc1, 0xc9, 0x9f, 0xb9, 0x5a, + 0x9b, 0x78, 0x69, 0xd9, 0x0f, 0x1c, 0x29, 0x03 }, + { 0xa7, 0x07, 0xef, 0xbc, 0xcd, 0xce, 0xed, 0x42, + 0x96, 0x7a, 0x66, 0xf5, 0x53, 0x9b, 0x93, 0xed, + 0x75, 0x60, 0xd4, 0x67, 0x30, 0x40, 0x16, 0xc4, + 0x78, 0x0d, 0x77, 0x55, 0xa5, 0x65, 0xd4, 0xc4 }, + { 0x38, 0xc5, 0x3d, 0xfb, 0x70, 0xbe, 0x7e, 0x79, + 0x2b, 0x07, 0xa6, 0xa3, 0x5b, 0x8a, 0x6a, 0x0a, + 0xba, 0x02, 0xc5, 0xc5, 0xf3, 0x8b, 0xaf, 0x5c, + 0x82, 0x3f, 0xdf, 0xd9, 0xe4, 0x2d, 0x65, 0x7e }, + { 0xf2, 0x91, 0x13, 0x86, 0x50, 0x1d, 0x9a, 0xb9, + 0xd7, 0x20, 0xcf, 0x8a, 0xd1, 0x05, 0x03, 0xd5, + 0x63, 0x4b, 0xf4, 0xb7, 0xd1, 0x2b, 0x56, 0xdf, + 0xb7, 0x4f, 0xec, 0xc6, 0xe4, 0x09, 0x3f, 0x68 }, + { 0xc6, 0xf2, 0xbd, 0xd5, 0x2b, 0x81, 0xe6, 0xe4, + 0xf6, 0x59, 0x5a, 0xbd, 0x4d, 0x7f, 0xb3, 0x1f, + 0x65, 0x11, 0x69, 0xd0, 0x0f, 0xf3, 0x26, 0x92, + 0x6b, 0x34, 0x94, 0x7b, 0x28, 0xa8, 0x39, 0x59 }, + { 0x29, 0x3d, 0x94, 0xb1, 0x8c, 0x98, 0xbb, 0x32, + 0x23, 0x36, 0x6b, 0x8c, 0xe7, 0x4c, 0x28, 0xfb, + 0xdf, 0x28, 0xe1, 0xf8, 0x4a, 0x33, 0x50, 0xb0, + 0xeb, 0x2d, 0x18, 0x04, 0xa5, 0x77, 0x57, 0x9b }, + { 0x2c, 0x2f, 0xa5, 0xc0, 0xb5, 0x15, 0x33, 0x16, + 0x5b, 0xc3, 0x75, 0xc2, 0x2e, 0x27, 0x81, 0x76, + 0x82, 0x70, 0xa3, 0x83, 0x98, 0x5d, 0x13, 0xbd, + 0x6b, 0x67, 0xb6, 0xfd, 0x67, 0xf8, 0x89, 0xeb }, + { 0xca, 0xa0, 0x9b, 0x82, 0xb7, 0x25, 0x62, 0xe4, + 0x3f, 0x4b, 0x22, 0x75, 0xc0, 0x91, 0x91, 0x8e, + 0x62, 0x4d, 0x91, 0x16, 0x61, 0xcc, 0x81, 0x1b, + 0xb5, 0xfa, 0xec, 0x51, 0xf6, 0x08, 0x8e, 0xf7 }, + { 0x24, 0x76, 0x1e, 0x45, 0xe6, 0x74, 0x39, 0x53, + 0x79, 0xfb, 0x17, 0x72, 0x9c, 0x78, 0xcb, 0x93, + 0x9e, 0x6f, 0x74, 0xc5, 0xdf, 0xfb, 0x9c, 0x96, + 0x1f, 0x49, 0x59, 0x82, 0xc3, 0xed, 0x1f, 0xe3 }, + { 0x55, 0xb7, 0x0a, 0x82, 0x13, 0x1e, 0xc9, 0x48, + 0x88, 0xd7, 0xab, 0x54, 0xa7, 0xc5, 0x15, 0x25, + 0x5c, 0x39, 0x38, 0xbb, 0x10, 0xbc, 0x78, 0x4d, + 0xc9, 0xb6, 0x7f, 0x07, 0x6e, 0x34, 0x1a, 0x73 }, + { 0x6a, 0xb9, 0x05, 0x7b, 0x97, 0x7e, 0xbc, 0x3c, + 0xa4, 0xd4, 0xce, 0x74, 0x50, 0x6c, 0x25, 0xcc, + 0xcd, 0xc5, 0x66, 0x49, 0x7c, 0x45, 0x0b, 0x54, + 0x15, 0xa3, 0x94, 0x86, 0xf8, 0x65, 0x7a, 0x03 }, + { 0x24, 0x06, 0x6d, 0xee, 0xe0, 0xec, 0xee, 0x15, + 0xa4, 0x5f, 0x0a, 0x32, 0x6d, 0x0f, 0x8d, 0xbc, + 0x79, 0x76, 0x1e, 0xbb, 0x93, 0xcf, 0x8c, 0x03, + 0x77, 0xaf, 0x44, 0x09, 0x78, 0xfc, 0xf9, 0x94 }, + { 0x20, 0x00, 0x0d, 0x3f, 0x66, 0xba, 0x76, 0x86, + 0x0d, 0x5a, 0x95, 0x06, 0x88, 0xb9, 0xaa, 0x0d, + 0x76, 0xcf, 0xea, 0x59, 0xb0, 0x05, 0xd8, 0x59, + 0x91, 0x4b, 0x1a, 0x46, 0x65, 0x3a, 0x93, 0x9b }, + { 0xb9, 0x2d, 0xaa, 0x79, 0x60, 0x3e, 0x3b, 0xdb, + 0xc3, 0xbf, 0xe0, 0xf4, 0x19, 0xe4, 0x09, 0xb2, + 0xea, 0x10, 0xdc, 0x43, 0x5b, 0xee, 0xfe, 0x29, + 0x59, 0xda, 0x16, 0x89, 0x5d, 0x5d, 0xca, 0x1c }, + { 0xe9, 0x47, 0x94, 0x87, 0x05, 0xb2, 0x06, 0xd5, + 0x72, 0xb0, 0xe8, 0xf6, 0x2f, 0x66, 0xa6, 0x55, + 0x1c, 0xbd, 0x6b, 0xc3, 0x05, 0xd2, 0x6c, 0xe7, + 0x53, 0x9a, 0x12, 0xf9, 0xaa, 0xdf, 0x75, 0x71 }, + { 0x3d, 0x67, 0xc1, 0xb3, 0xf9, 0xb2, 0x39, 0x10, + 0xe3, 0xd3, 0x5e, 0x6b, 0x0f, 0x2c, 0xcf, 0x44, + 0xa0, 0xb5, 0x40, 0xa4, 0x5c, 0x18, 0xba, 0x3c, + 0x36, 0x26, 0x4d, 0xd4, 0x8e, 0x96, 0xaf, 0x6a }, + { 0xc7, 0x55, 0x8b, 0xab, 0xda, 0x04, 0xbc, 0xcb, + 0x76, 0x4d, 0x0b, 0xbf, 0x33, 0x58, 0x42, 0x51, + 0x41, 0x90, 0x2d, 0x22, 0x39, 0x1d, 0x9f, 0x8c, + 0x59, 0x15, 0x9f, 0xec, 0x9e, 0x49, 0xb1, 0x51 }, + { 0x0b, 0x73, 0x2b, 0xb0, 0x35, 0x67, 0x5a, 0x50, + 0xff, 0x58, 0xf2, 0xc2, 0x42, 0xe4, 0x71, 0x0a, + 0xec, 0xe6, 0x46, 0x70, 0x07, 0x9c, 0x13, 0x04, + 0x4c, 0x79, 0xc9, 0xb7, 0x49, 0x1f, 0x70, 0x00 }, + { 0xd1, 0x20, 0xb5, 0xef, 0x6d, 0x57, 0xeb, 0xf0, + 0x6e, 0xaf, 0x96, 0xbc, 0x93, 0x3c, 0x96, 0x7b, + 0x16, 0xcb, 0xe6, 0xe2, 0xbf, 0x00, 0x74, 0x1c, + 0x30, 0xaa, 0x1c, 0x54, 0xba, 0x64, 0x80, 0x1f }, + { 0x58, 0xd2, 0x12, 0xad, 0x6f, 0x58, 0xae, 0xf0, + 0xf8, 0x01, 0x16, 0xb4, 0x41, 0xe5, 0x7f, 0x61, + 0x95, 0xbf, 0xef, 0x26, 0xb6, 0x14, 0x63, 0xed, + 0xec, 0x11, 0x83, 0xcd, 0xb0, 0x4f, 0xe7, 0x6d }, + { 0xb8, 0x83, 0x6f, 0x51, 0xd1, 0xe2, 0x9b, 0xdf, + 0xdb, 0xa3, 0x25, 0x56, 0x53, 0x60, 0x26, 0x8b, + 0x8f, 0xad, 0x62, 0x74, 0x73, 0xed, 0xec, 0xef, + 0x7e, 0xae, 0xfe, 0xe8, 0x37, 0xc7, 0x40, 0x03 }, + { 0xc5, 0x47, 0xa3, 0xc1, 0x24, 0xae, 0x56, 0x85, + 0xff, 0xa7, 0xb8, 0xed, 0xaf, 0x96, 0xec, 0x86, + 0xf8, 0xb2, 0xd0, 0xd5, 0x0c, 0xee, 0x8b, 0xe3, + 0xb1, 0xf0, 0xc7, 0x67, 0x63, 0x06, 0x9d, 0x9c }, + { 0x5d, 0x16, 0x8b, 0x76, 0x9a, 0x2f, 0x67, 0x85, + 0x3d, 0x62, 0x95, 0xf7, 0x56, 0x8b, 0xe4, 0x0b, + 0xb7, 0xa1, 0x6b, 0x8d, 0x65, 0xba, 0x87, 0x63, + 0x5d, 0x19, 0x78, 0xd2, 0xab, 0x11, 0xba, 0x2a }, + { 0xa2, 0xf6, 0x75, 0xdc, 0x73, 0x02, 0x63, 0x8c, + 0xb6, 0x02, 0x01, 0x06, 0x4c, 0xa5, 0x50, 0x77, + 0x71, 0x4d, 0x71, 0xfe, 0x09, 0x6a, 0x31, 0x5f, + 0x2f, 0xe7, 0x40, 0x12, 0x77, 0xca, 0xa5, 0xaf }, + { 0xc8, 0xaa, 0xb5, 0xcd, 0x01, 0x60, 0xae, 0x78, + 0xcd, 0x2e, 0x8a, 0xc5, 0xfb, 0x0e, 0x09, 0x3c, + 0xdb, 0x5c, 0x4b, 0x60, 0x52, 0xa0, 0xa9, 0x7b, + 0xb0, 0x42, 0x16, 0x82, 0x6f, 0xa7, 0xa4, 0x37 }, + { 0xff, 0x68, 0xca, 0x40, 0x35, 0xbf, 0xeb, 0x43, + 0xfb, 0xf1, 0x45, 0xfd, 0xdd, 0x5e, 0x43, 0xf1, + 0xce, 0xa5, 0x4f, 0x11, 0xf7, 0xbe, 0xe1, 0x30, + 0x58, 0xf0, 0x27, 0x32, 0x9a, 0x4a, 0x5f, 0xa4 }, + { 0x1d, 0x4e, 0x54, 0x87, 0xae, 0x3c, 0x74, 0x0f, + 0x2b, 0xa6, 0xe5, 0x41, 0xac, 0x91, 0xbc, 0x2b, + 0xfc, 0xd2, 0x99, 0x9c, 0x51, 0x8d, 0x80, 0x7b, + 0x42, 0x67, 0x48, 0x80, 0x3a, 0x35, 0x0f, 0xd4 }, + { 0x6d, 0x24, 0x4e, 0x1a, 0x06, 0xce, 0x4e, 0xf5, + 0x78, 0xdd, 0x0f, 0x63, 0xaf, 0xf0, 0x93, 0x67, + 0x06, 0x73, 0x51, 0x19, 0xca, 0x9c, 0x8d, 0x22, + 0xd8, 0x6c, 0x80, 0x14, 0x14, 0xab, 0x97, 0x41 }, + { 0xde, 0xcf, 0x73, 0x29, 0xdb, 0xcc, 0x82, 0x7b, + 0x8f, 0xc5, 0x24, 0xc9, 0x43, 0x1e, 0x89, 0x98, + 0x02, 0x9e, 0xce, 0x12, 0xce, 0x93, 0xb7, 0xb2, + 0xf3, 0xe7, 0x69, 0xa9, 0x41, 0xfb, 0x8c, 0xea }, + { 0x2f, 0xaf, 0xcc, 0x0f, 0x2e, 0x63, 0xcb, 0xd0, + 0x77, 0x55, 0xbe, 0x7b, 0x75, 0xec, 0xea, 0x0a, + 0xdf, 0xf9, 0xaa, 0x5e, 0xde, 0x2a, 0x52, 0xfd, + 0xab, 0x4d, 0xfd, 0x03, 0x74, 0xcd, 0x48, 0x3f }, + { 0xaa, 0x85, 0x01, 0x0d, 0xd4, 0x6a, 0x54, 0x6b, + 0x53, 0x5e, 0xf4, 0xcf, 0x5f, 0x07, 0xd6, 0x51, + 0x61, 0xe8, 0x98, 0x28, 0xf3, 0xa7, 0x7d, 0xb7, + 0xb9, 0xb5, 0x6f, 0x0d, 0xf5, 0x9a, 0xae, 0x45 }, + { 0x07, 0xe8, 0xe1, 0xee, 0x73, 0x2c, 0xb0, 0xd3, + 0x56, 0xc9, 0xc0, 0xd1, 0x06, 0x9c, 0x89, 0xd1, + 0x7a, 0xdf, 0x6a, 0x9a, 0x33, 0x4f, 0x74, 0x5e, + 0xc7, 0x86, 0x73, 0x32, 0x54, 0x8c, 0xa8, 0xe9 }, + { 0x0e, 0x01, 0xe8, 0x1c, 0xad, 0xa8, 0x16, 0x2b, + 0xfd, 0x5f, 0x8a, 0x8c, 0x81, 0x8a, 0x6c, 0x69, + 0xfe, 0xdf, 0x02, 0xce, 0xb5, 0x20, 0x85, 0x23, + 0xcb, 0xe5, 0x31, 0x3b, 0x89, 0xca, 0x10, 0x53 }, + { 0x6b, 0xb6, 0xc6, 0x47, 0x26, 0x55, 0x08, 0x43, + 0x99, 0x85, 0x2e, 0x00, 0x24, 0x9f, 0x8c, 0xb2, + 0x47, 0x89, 0x6d, 0x39, 0x2b, 0x02, 0xd7, 0x3b, + 0x7f, 0x0d, 0xd8, 0x18, 0xe1, 0xe2, 0x9b, 0x07 }, + { 0x42, 0xd4, 0x63, 0x6e, 0x20, 0x60, 0xf0, 0x8f, + 0x41, 0xc8, 0x82, 0xe7, 0x6b, 0x39, 0x6b, 0x11, + 0x2e, 0xf6, 0x27, 0xcc, 0x24, 0xc4, 0x3d, 0xd5, + 0xf8, 0x3a, 0x1d, 0x1a, 0x7e, 0xad, 0x71, 0x1a }, + { 0x48, 0x58, 0xc9, 0xa1, 0x88, 0xb0, 0x23, 0x4f, + 0xb9, 0xa8, 0xd4, 0x7d, 0x0b, 0x41, 0x33, 0x65, + 0x0a, 0x03, 0x0b, 0xd0, 0x61, 0x1b, 0x87, 0xc3, + 0x89, 0x2e, 0x94, 0x95, 0x1f, 0x8d, 0xf8, 0x52 }, + { 0x3f, 0xab, 0x3e, 0x36, 0x98, 0x8d, 0x44, 0x5a, + 0x51, 0xc8, 0x78, 0x3e, 0x53, 0x1b, 0xe3, 0xa0, + 0x2b, 0xe4, 0x0c, 0xd0, 0x47, 0x96, 0xcf, 0xb6, + 0x1d, 0x40, 0x34, 0x74, 0x42, 0xd3, 0xf7, 0x94 }, + { 0xeb, 0xab, 0xc4, 0x96, 0x36, 0xbd, 0x43, 0x3d, + 0x2e, 0xc8, 0xf0, 0xe5, 0x18, 0x73, 0x2e, 0xf8, + 0xfa, 0x21, 0xd4, 0xd0, 0x71, 0xcc, 0x3b, 0xc4, + 0x6c, 0xd7, 0x9f, 0xa3, 0x8a, 0x28, 0xb8, 0x10 }, + { 0xa1, 0xd0, 0x34, 0x35, 0x23, 0xb8, 0x93, 0xfc, + 0xa8, 0x4f, 0x47, 0xfe, 0xb4, 0xa6, 0x4d, 0x35, + 0x0a, 0x17, 0xd8, 0xee, 0xf5, 0x49, 0x7e, 0xce, + 0x69, 0x7d, 0x02, 0xd7, 0x91, 0x78, 0xb5, 0x91 }, + { 0x26, 0x2e, 0xbf, 0xd9, 0x13, 0x0b, 0x7d, 0x28, + 0x76, 0x0d, 0x08, 0xef, 0x8b, 0xfd, 0x3b, 0x86, + 0xcd, 0xd3, 0xb2, 0x11, 0x3d, 0x2c, 0xae, 0xf7, + 0xea, 0x95, 0x1a, 0x30, 0x3d, 0xfa, 0x38, 0x46 }, + { 0xf7, 0x61, 0x58, 0xed, 0xd5, 0x0a, 0x15, 0x4f, + 0xa7, 0x82, 0x03, 0xed, 0x23, 0x62, 0x93, 0x2f, + 0xcb, 0x82, 0x53, 0xaa, 0xe3, 0x78, 0x90, 0x3e, + 0xde, 0xd1, 0xe0, 0x3f, 0x70, 0x21, 0xa2, 0x57 }, + { 0x26, 0x17, 0x8e, 0x95, 0x0a, 0xc7, 0x22, 0xf6, + 0x7a, 0xe5, 0x6e, 0x57, 0x1b, 0x28, 0x4c, 0x02, + 0x07, 0x68, 0x4a, 0x63, 0x34, 0xa1, 0x77, 0x48, + 0xa9, 0x4d, 0x26, 0x0b, 0xc5, 0xf5, 0x52, 0x74 }, + { 0xc3, 0x78, 0xd1, 0xe4, 0x93, 0xb4, 0x0e, 0xf1, + 0x1f, 0xe6, 0xa1, 0x5d, 0x9c, 0x27, 0x37, 0xa3, + 0x78, 0x09, 0x63, 0x4c, 0x5a, 0xba, 0xd5, 0xb3, + 0x3d, 0x7e, 0x39, 0x3b, 0x4a, 0xe0, 0x5d, 0x03 }, + { 0x98, 0x4b, 0xd8, 0x37, 0x91, 0x01, 0xbe, 0x8f, + 0xd8, 0x06, 0x12, 0xd8, 0xea, 0x29, 0x59, 0xa7, + 0x86, 0x5e, 0xc9, 0x71, 0x85, 0x23, 0x55, 0x01, + 0x07, 0xae, 0x39, 0x38, 0xdf, 0x32, 0x01, 0x1b }, + { 0xc6, 0xf2, 0x5a, 0x81, 0x2a, 0x14, 0x48, 0x58, + 0xac, 0x5c, 0xed, 0x37, 0xa9, 0x3a, 0x9f, 0x47, + 0x59, 0xba, 0x0b, 0x1c, 0x0f, 0xdc, 0x43, 0x1d, + 0xce, 0x35, 0xf9, 0xec, 0x1f, 0x1f, 0x4a, 0x99 }, + { 0x92, 0x4c, 0x75, 0xc9, 0x44, 0x24, 0xff, 0x75, + 0xe7, 0x4b, 0x8b, 0x4e, 0x94, 0x35, 0x89, 0x58, + 0xb0, 0x27, 0xb1, 0x71, 0xdf, 0x5e, 0x57, 0x89, + 0x9a, 0xd0, 0xd4, 0xda, 0xc3, 0x73, 0x53, 0xb6 }, + { 0x0a, 0xf3, 0x58, 0x92, 0xa6, 0x3f, 0x45, 0x93, + 0x1f, 0x68, 0x46, 0xed, 0x19, 0x03, 0x61, 0xcd, + 0x07, 0x30, 0x89, 0xe0, 0x77, 0x16, 0x57, 0x14, + 0xb5, 0x0b, 0x81, 0xa2, 0xe3, 0xdd, 0x9b, 0xa1 }, + { 0xcc, 0x80, 0xce, 0xfb, 0x26, 0xc3, 0xb2, 0xb0, + 0xda, 0xef, 0x23, 0x3e, 0x60, 0x6d, 0x5f, 0xfc, + 0x80, 0xfa, 0x17, 0x42, 0x7d, 0x18, 0xe3, 0x04, + 0x89, 0x67, 0x3e, 0x06, 0xef, 0x4b, 0x87, 0xf7 }, + { 0xc2, 0xf8, 0xc8, 0x11, 0x74, 0x47, 0xf3, 0x97, + 0x8b, 0x08, 0x18, 0xdc, 0xf6, 0xf7, 0x01, 0x16, + 0xac, 0x56, 0xfd, 0x18, 0x4d, 0xd1, 0x27, 0x84, + 0x94, 0xe1, 0x03, 0xfc, 0x6d, 0x74, 0xa8, 0x87 }, + { 0xbd, 0xec, 0xf6, 0xbf, 0xc1, 0xba, 0x0d, 0xf6, + 0xe8, 0x62, 0xc8, 0x31, 0x99, 0x22, 0x07, 0x79, + 0x6a, 0xcc, 0x79, 0x79, 0x68, 0x35, 0x88, 0x28, + 0xc0, 0x6e, 0x7a, 0x51, 0xe0, 0x90, 0x09, 0x8f }, + { 0x24, 0xd1, 0xa2, 0x6e, 0x3d, 0xab, 0x02, 0xfe, + 0x45, 0x72, 0xd2, 0xaa, 0x7d, 0xbd, 0x3e, 0xc3, + 0x0f, 0x06, 0x93, 0xdb, 0x26, 0xf2, 0x73, 0xd0, + 0xab, 0x2c, 0xb0, 0xc1, 0x3b, 0x5e, 0x64, 0x51 }, + { 0xec, 0x56, 0xf5, 0x8b, 0x09, 0x29, 0x9a, 0x30, + 0x0b, 0x14, 0x05, 0x65, 0xd7, 0xd3, 0xe6, 0x87, + 0x82, 0xb6, 0xe2, 0xfb, 0xeb, 0x4b, 0x7e, 0xa9, + 0x7a, 0xc0, 0x57, 0x98, 0x90, 0x61, 0xdd, 0x3f }, + { 0x11, 0xa4, 0x37, 0xc1, 0xab, 0xa3, 0xc1, 0x19, + 0xdd, 0xfa, 0xb3, 0x1b, 0x3e, 0x8c, 0x84, 0x1d, + 0xee, 0xeb, 0x91, 0x3e, 0xf5, 0x7f, 0x7e, 0x48, + 0xf2, 0xc9, 0xcf, 0x5a, 0x28, 0xfa, 0x42, 0xbc }, + { 0x53, 0xc7, 0xe6, 0x11, 0x4b, 0x85, 0x0a, 0x2c, + 0xb4, 0x96, 0xc9, 0xb3, 0xc6, 0x9a, 0x62, 0x3e, + 0xae, 0xa2, 0xcb, 0x1d, 0x33, 0xdd, 0x81, 0x7e, + 0x47, 0x65, 0xed, 0xaa, 0x68, 0x23, 0xc2, 0x28 }, + { 0x15, 0x4c, 0x3e, 0x96, 0xfe, 0xe5, 0xdb, 0x14, + 0xf8, 0x77, 0x3e, 0x18, 0xaf, 0x14, 0x85, 0x79, + 0x13, 0x50, 0x9d, 0xa9, 0x99, 0xb4, 0x6c, 0xdd, + 0x3d, 0x4c, 0x16, 0x97, 0x60, 0xc8, 0x3a, 0xd2 }, + { 0x40, 0xb9, 0x91, 0x6f, 0x09, 0x3e, 0x02, 0x7a, + 0x87, 0x86, 0x64, 0x18, 0x18, 0x92, 0x06, 0x20, + 0x47, 0x2f, 0xbc, 0xf6, 0x8f, 0x70, 0x1d, 0x1b, + 0x68, 0x06, 0x32, 0xe6, 0x99, 0x6b, 0xde, 0xd3 }, + { 0x24, 0xc4, 0xcb, 0xba, 0x07, 0x11, 0x98, 0x31, + 0xa7, 0x26, 0xb0, 0x53, 0x05, 0xd9, 0x6d, 0xa0, + 0x2f, 0xf8, 0xb1, 0x48, 0xf0, 0xda, 0x44, 0x0f, + 0xe2, 0x33, 0xbc, 0xaa, 0x32, 0xc7, 0x2f, 0x6f }, + { 0x5d, 0x20, 0x15, 0x10, 0x25, 0x00, 0x20, 0xb7, + 0x83, 0x68, 0x96, 0x88, 0xab, 0xbf, 0x8e, 0xcf, + 0x25, 0x94, 0xa9, 0x6a, 0x08, 0xf2, 0xbf, 0xec, + 0x6c, 0xe0, 0x57, 0x44, 0x65, 0xdd, 0xed, 0x71 }, + { 0x04, 0x3b, 0x97, 0xe3, 0x36, 0xee, 0x6f, 0xdb, + 0xbe, 0x2b, 0x50, 0xf2, 0x2a, 0xf8, 0x32, 0x75, + 0xa4, 0x08, 0x48, 0x05, 0xd2, 0xd5, 0x64, 0x59, + 0x62, 0x45, 0x4b, 0x6c, 0x9b, 0x80, 0x53, 0xa0 }, + { 0x56, 0x48, 0x35, 0xcb, 0xae, 0xa7, 0x74, 0x94, + 0x85, 0x68, 0xbe, 0x36, 0xcf, 0x52, 0xfc, 0xdd, + 0x83, 0x93, 0x4e, 0xb0, 0xa2, 0x75, 0x12, 0xdb, + 0xe3, 0xe2, 0xdb, 0x47, 0xb9, 0xe6, 0x63, 0x5a }, + { 0xf2, 0x1c, 0x33, 0xf4, 0x7b, 0xde, 0x40, 0xa2, + 0xa1, 0x01, 0xc9, 0xcd, 0xe8, 0x02, 0x7a, 0xaf, + 0x61, 0xa3, 0x13, 0x7d, 0xe2, 0x42, 0x2b, 0x30, + 0x03, 0x5a, 0x04, 0xc2, 0x70, 0x89, 0x41, 0x83 }, + { 0x9d, 0xb0, 0xef, 0x74, 0xe6, 0x6c, 0xbb, 0x84, + 0x2e, 0xb0, 0xe0, 0x73, 0x43, 0xa0, 0x3c, 0x5c, + 0x56, 0x7e, 0x37, 0x2b, 0x3f, 0x23, 0xb9, 0x43, + 0xc7, 0x88, 0xa4, 0xf2, 0x50, 0xf6, 0x78, 0x91 }, + { 0xab, 0x8d, 0x08, 0x65, 0x5f, 0xf1, 0xd3, 0xfe, + 0x87, 0x58, 0xd5, 0x62, 0x23, 0x5f, 0xd2, 0x3e, + 0x7c, 0xf9, 0xdc, 0xaa, 0xd6, 0x58, 0x87, 0x2a, + 0x49, 0xe5, 0xd3, 0x18, 0x3b, 0x6c, 0xce, 0xbd }, + { 0x6f, 0x27, 0xf7, 0x7e, 0x7b, 0xcf, 0x46, 0xa1, + 0xe9, 0x63, 0xad, 0xe0, 0x30, 0x97, 0x33, 0x54, + 0x30, 0x31, 0xdc, 0xcd, 0xd4, 0x7c, 0xaa, 0xc1, + 0x74, 0xd7, 0xd2, 0x7c, 0xe8, 0x07, 0x7e, 0x8b }, + { 0xe3, 0xcd, 0x54, 0xda, 0x7e, 0x44, 0x4c, 0xaa, + 0x62, 0x07, 0x56, 0x95, 0x25, 0xa6, 0x70, 0xeb, + 0xae, 0x12, 0x78, 0xde, 0x4e, 0x3f, 0xe2, 0x68, + 0x4b, 0x3e, 0x33, 0xf5, 0xef, 0x90, 0xcc, 0x1b }, + { 0xb2, 0xc3, 0xe3, 0x3a, 0x51, 0xd2, 0x2c, 0x4c, + 0x08, 0xfc, 0x09, 0x89, 0xc8, 0x73, 0xc9, 0xcc, + 0x41, 0x50, 0x57, 0x9b, 0x1e, 0x61, 0x63, 0xfa, + 0x69, 0x4a, 0xd5, 0x1d, 0x53, 0xd7, 0x12, 0xdc }, + { 0xbe, 0x7f, 0xda, 0x98, 0x3e, 0x13, 0x18, 0x9b, + 0x4c, 0x77, 0xe0, 0xa8, 0x09, 0x20, 0xb6, 0xe0, + 0xe0, 0xea, 0x80, 0xc3, 0xb8, 0x4d, 0xbe, 0x7e, + 0x71, 0x17, 0xd2, 0x53, 0xf4, 0x81, 0x12, 0xf4 }, + { 0xb6, 0x00, 0x8c, 0x28, 0xfa, 0xe0, 0x8a, 0xa4, + 0x27, 0xe5, 0xbd, 0x3a, 0xad, 0x36, 0xf1, 0x00, + 0x21, 0xf1, 0x6c, 0x77, 0xcf, 0xea, 0xbe, 0xd0, + 0x7f, 0x97, 0xcc, 0x7d, 0xc1, 0xf1, 0x28, 0x4a }, + { 0x6e, 0x4e, 0x67, 0x60, 0xc5, 0x38, 0xf2, 0xe9, + 0x7b, 0x3a, 0xdb, 0xfb, 0xbc, 0xde, 0x57, 0xf8, + 0x96, 0x6b, 0x7e, 0xa8, 0xfc, 0xb5, 0xbf, 0x7e, + 0xfe, 0xc9, 0x13, 0xfd, 0x2a, 0x2b, 0x0c, 0x55 }, + { 0x4a, 0xe5, 0x1f, 0xd1, 0x83, 0x4a, 0xa5, 0xbd, + 0x9a, 0x6f, 0x7e, 0xc3, 0x9f, 0xc6, 0x63, 0x33, + 0x8d, 0xc5, 0xd2, 0xe2, 0x07, 0x61, 0x56, 0x6d, + 0x90, 0xcc, 0x68, 0xb1, 0xcb, 0x87, 0x5e, 0xd8 }, + { 0xb6, 0x73, 0xaa, 0xd7, 0x5a, 0xb1, 0xfd, 0xb5, + 0x40, 0x1a, 0xbf, 0xa1, 0xbf, 0x89, 0xf3, 0xad, + 0xd2, 0xeb, 0xc4, 0x68, 0xdf, 0x36, 0x24, 0xa4, + 0x78, 0xf4, 0xfe, 0x85, 0x9d, 0x8d, 0x55, 0xe2 }, + { 0x13, 0xc9, 0x47, 0x1a, 0x98, 0x55, 0x91, 0x35, + 0x39, 0x83, 0x66, 0x60, 0x39, 0x8d, 0xa0, 0xf3, + 0xf9, 0x9a, 0xda, 0x08, 0x47, 0x9c, 0x69, 0xd1, + 0xb7, 0xfc, 0xaa, 0x34, 0x61, 0xdd, 0x7e, 0x59 }, + { 0x2c, 0x11, 0xf4, 0xa7, 0xf9, 0x9a, 0x1d, 0x23, + 0xa5, 0x8b, 0xb6, 0x36, 0x35, 0x0f, 0xe8, 0x49, + 0xf2, 0x9c, 0xba, 0xc1, 0xb2, 0xa1, 0x11, 0x2d, + 0x9f, 0x1e, 0xd5, 0xbc, 0x5b, 0x31, 0x3c, 0xcd }, + { 0xc7, 0xd3, 0xc0, 0x70, 0x6b, 0x11, 0xae, 0x74, + 0x1c, 0x05, 0xa1, 0xef, 0x15, 0x0d, 0xd6, 0x5b, + 0x54, 0x94, 0xd6, 0xd5, 0x4c, 0x9a, 0x86, 0xe2, + 0x61, 0x78, 0x54, 0xe6, 0xae, 0xee, 0xbb, 0xd9 }, + { 0x19, 0x4e, 0x10, 0xc9, 0x38, 0x93, 0xaf, 0xa0, + 0x64, 0xc3, 0xac, 0x04, 0xc0, 0xdd, 0x80, 0x8d, + 0x79, 0x1c, 0x3d, 0x4b, 0x75, 0x56, 0xe8, 0x9d, + 0x8d, 0x9c, 0xb2, 0x25, 0xc4, 0xb3, 0x33, 0x39 }, + { 0x6f, 0xc4, 0x98, 0x8b, 0x8f, 0x78, 0x54, 0x6b, + 0x16, 0x88, 0x99, 0x18, 0x45, 0x90, 0x8f, 0x13, + 0x4b, 0x6a, 0x48, 0x2e, 0x69, 0x94, 0xb3, 0xd4, + 0x83, 0x17, 0xbf, 0x08, 0xdb, 0x29, 0x21, 0x85 }, + { 0x56, 0x65, 0xbe, 0xb8, 0xb0, 0x95, 0x55, 0x25, + 0x81, 0x3b, 0x59, 0x81, 0xcd, 0x14, 0x2e, 0xd4, + 0xd0, 0x3f, 0xba, 0x38, 0xa6, 0xf3, 0xe5, 0xad, + 0x26, 0x8e, 0x0c, 0xc2, 0x70, 0xd1, 0xcd, 0x11 }, + { 0xb8, 0x83, 0xd6, 0x8f, 0x5f, 0xe5, 0x19, 0x36, + 0x43, 0x1b, 0xa4, 0x25, 0x67, 0x38, 0x05, 0x3b, + 0x1d, 0x04, 0x26, 0xd4, 0xcb, 0x64, 0xb1, 0x6e, + 0x83, 0xba, 0xdc, 0x5e, 0x9f, 0xbe, 0x3b, 0x81 }, + { 0x53, 0xe7, 0xb2, 0x7e, 0xa5, 0x9c, 0x2f, 0x6d, + 0xbb, 0x50, 0x76, 0x9e, 0x43, 0x55, 0x4d, 0xf3, + 0x5a, 0xf8, 0x9f, 0x48, 0x22, 0xd0, 0x46, 0x6b, + 0x00, 0x7d, 0xd6, 0xf6, 0xde, 0xaf, 0xff, 0x02 }, + { 0x1f, 0x1a, 0x02, 0x29, 0xd4, 0x64, 0x0f, 0x01, + 0x90, 0x15, 0x88, 0xd9, 0xde, 0xc2, 0x2d, 0x13, + 0xfc, 0x3e, 0xb3, 0x4a, 0x61, 0xb3, 0x29, 0x38, + 0xef, 0xbf, 0x53, 0x34, 0xb2, 0x80, 0x0a, 0xfa }, + { 0xc2, 0xb4, 0x05, 0xaf, 0xa0, 0xfa, 0x66, 0x68, + 0x85, 0x2a, 0xee, 0x4d, 0x88, 0x04, 0x08, 0x53, + 0xfa, 0xb8, 0x00, 0xe7, 0x2b, 0x57, 0x58, 0x14, + 0x18, 0xe5, 0x50, 0x6f, 0x21, 0x4c, 0x7d, 0x1f }, + { 0xc0, 0x8a, 0xa1, 0xc2, 0x86, 0xd7, 0x09, 0xfd, + 0xc7, 0x47, 0x37, 0x44, 0x97, 0x71, 0x88, 0xc8, + 0x95, 0xba, 0x01, 0x10, 0x14, 0x24, 0x7e, 0x4e, + 0xfa, 0x8d, 0x07, 0xe7, 0x8f, 0xec, 0x69, 0x5c }, + { 0xf0, 0x3f, 0x57, 0x89, 0xd3, 0x33, 0x6b, 0x80, + 0xd0, 0x02, 0xd5, 0x9f, 0xdf, 0x91, 0x8b, 0xdb, + 0x77, 0x5b, 0x00, 0x95, 0x6e, 0xd5, 0x52, 0x8e, + 0x86, 0xaa, 0x99, 0x4a, 0xcb, 0x38, 0xfe, 0x2d } +}; + +static const u8 blake2s_keyed_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b, + 0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b, + 0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a, + 0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49 }, + { 0x40, 0xd1, 0x5f, 0xee, 0x7c, 0x32, 0x88, 0x30, + 0x16, 0x6a, 0xc3, 0xf9, 0x18, 0x65, 0x0f, 0x80, + 0x7e, 0x7e, 0x01, 0xe1, 0x77, 0x25, 0x8c, 0xdc, + 0x0a, 0x39, 0xb1, 0x1f, 0x59, 0x80, 0x66, 0xf1 }, + { 0x6b, 0xb7, 0x13, 0x00, 0x64, 0x4c, 0xd3, 0x99, + 0x1b, 0x26, 0xcc, 0xd4, 0xd2, 0x74, 0xac, 0xd1, + 0xad, 0xea, 0xb8, 0xb1, 0xd7, 0x91, 0x45, 0x46, + 0xc1, 0x19, 0x8b, 0xbe, 0x9f, 0xc9, 0xd8, 0x03 }, + { 0x1d, 0x22, 0x0d, 0xbe, 0x2e, 0xe1, 0x34, 0x66, + 0x1f, 0xdf, 0x6d, 0x9e, 0x74, 0xb4, 0x17, 0x04, + 0x71, 0x05, 0x56, 0xf2, 0xf6, 0xe5, 0xa0, 0x91, + 0xb2, 0x27, 0x69, 0x74, 0x45, 0xdb, 0xea, 0x6b }, + { 0xf6, 0xc3, 0xfb, 0xad, 0xb4, 0xcc, 0x68, 0x7a, + 0x00, 0x64, 0xa5, 0xbe, 0x6e, 0x79, 0x1b, 0xec, + 0x63, 0xb8, 0x68, 0xad, 0x62, 0xfb, 0xa6, 0x1b, + 0x37, 0x57, 0xef, 0x9c, 0xa5, 0x2e, 0x05, 0xb2 }, + { 0x49, 0xc1, 0xf2, 0x11, 0x88, 0xdf, 0xd7, 0x69, + 0xae, 0xa0, 0xe9, 0x11, 0xdd, 0x6b, 0x41, 0xf1, + 0x4d, 0xab, 0x10, 0x9d, 0x2b, 0x85, 0x97, 0x7a, + 0xa3, 0x08, 0x8b, 0x5c, 0x70, 0x7e, 0x85, 0x98 }, + { 0xfd, 0xd8, 0x99, 0x3d, 0xcd, 0x43, 0xf6, 0x96, + 0xd4, 0x4f, 0x3c, 0xea, 0x0f, 0xf3, 0x53, 0x45, + 0x23, 0x4e, 0xc8, 0xee, 0x08, 0x3e, 0xb3, 0xca, + 0xda, 0x01, 0x7c, 0x7f, 0x78, 0xc1, 0x71, 0x43 }, + { 0xe6, 0xc8, 0x12, 0x56, 0x37, 0x43, 0x8d, 0x09, + 0x05, 0xb7, 0x49, 0xf4, 0x65, 0x60, 0xac, 0x89, + 0xfd, 0x47, 0x1c, 0xf8, 0x69, 0x2e, 0x28, 0xfa, + 0xb9, 0x82, 0xf7, 0x3f, 0x01, 0x9b, 0x83, 0xa9 }, + { 0x19, 0xfc, 0x8c, 0xa6, 0x97, 0x9d, 0x60, 0xe6, + 0xed, 0xd3, 0xb4, 0x54, 0x1e, 0x2f, 0x96, 0x7c, + 0xed, 0x74, 0x0d, 0xf6, 0xec, 0x1e, 0xae, 0xbb, + 0xfe, 0x81, 0x38, 0x32, 0xe9, 0x6b, 0x29, 0x74 }, + { 0xa6, 0xad, 0x77, 0x7c, 0xe8, 0x81, 0xb5, 0x2b, + 0xb5, 0xa4, 0x42, 0x1a, 0xb6, 0xcd, 0xd2, 0xdf, + 0xba, 0x13, 0xe9, 0x63, 0x65, 0x2d, 0x4d, 0x6d, + 0x12, 0x2a, 0xee, 0x46, 0x54, 0x8c, 0x14, 0xa7 }, + { 0xf5, 0xc4, 0xb2, 0xba, 0x1a, 0x00, 0x78, 0x1b, + 0x13, 0xab, 0xa0, 0x42, 0x52, 0x42, 0xc6, 0x9c, + 0xb1, 0x55, 0x2f, 0x3f, 0x71, 0xa9, 0xa3, 0xbb, + 0x22, 0xb4, 0xa6, 0xb4, 0x27, 0x7b, 0x46, 0xdd }, + { 0xe3, 0x3c, 0x4c, 0x9b, 0xd0, 0xcc, 0x7e, 0x45, + 0xc8, 0x0e, 0x65, 0xc7, 0x7f, 0xa5, 0x99, 0x7f, + 0xec, 0x70, 0x02, 0x73, 0x85, 0x41, 0x50, 0x9e, + 0x68, 0xa9, 0x42, 0x38, 0x91, 0xe8, 0x22, 0xa3 }, + { 0xfb, 0xa1, 0x61, 0x69, 0xb2, 0xc3, 0xee, 0x10, + 0x5b, 0xe6, 0xe1, 0xe6, 0x50, 0xe5, 0xcb, 0xf4, + 0x07, 0x46, 0xb6, 0x75, 0x3d, 0x03, 0x6a, 0xb5, + 0x51, 0x79, 0x01, 0x4a, 0xd7, 0xef, 0x66, 0x51 }, + { 0xf5, 0xc4, 0xbe, 0xc6, 0xd6, 0x2f, 0xc6, 0x08, + 0xbf, 0x41, 0xcc, 0x11, 0x5f, 0x16, 0xd6, 0x1c, + 0x7e, 0xfd, 0x3f, 0xf6, 0xc6, 0x56, 0x92, 0xbb, + 0xe0, 0xaf, 0xff, 0xb1, 0xfe, 0xde, 0x74, 0x75 }, + { 0xa4, 0x86, 0x2e, 0x76, 0xdb, 0x84, 0x7f, 0x05, + 0xba, 0x17, 0xed, 0xe5, 0xda, 0x4e, 0x7f, 0x91, + 0xb5, 0x92, 0x5c, 0xf1, 0xad, 0x4b, 0xa1, 0x27, + 0x32, 0xc3, 0x99, 0x57, 0x42, 0xa5, 0xcd, 0x6e }, + { 0x65, 0xf4, 0xb8, 0x60, 0xcd, 0x15, 0xb3, 0x8e, + 0xf8, 0x14, 0xa1, 0xa8, 0x04, 0x31, 0x4a, 0x55, + 0xbe, 0x95, 0x3c, 0xaa, 0x65, 0xfd, 0x75, 0x8a, + 0xd9, 0x89, 0xff, 0x34, 0xa4, 0x1c, 0x1e, 0xea }, + { 0x19, 0xba, 0x23, 0x4f, 0x0a, 0x4f, 0x38, 0x63, + 0x7d, 0x18, 0x39, 0xf9, 0xd9, 0xf7, 0x6a, 0xd9, + 0x1c, 0x85, 0x22, 0x30, 0x71, 0x43, 0xc9, 0x7d, + 0x5f, 0x93, 0xf6, 0x92, 0x74, 0xce, 0xc9, 0xa7 }, + { 0x1a, 0x67, 0x18, 0x6c, 0xa4, 0xa5, 0xcb, 0x8e, + 0x65, 0xfc, 0xa0, 0xe2, 0xec, 0xbc, 0x5d, 0xdc, + 0x14, 0xae, 0x38, 0x1b, 0xb8, 0xbf, 0xfe, 0xb9, + 0xe0, 0xa1, 0x03, 0x44, 0x9e, 0x3e, 0xf0, 0x3c }, + { 0xaf, 0xbe, 0xa3, 0x17, 0xb5, 0xa2, 0xe8, 0x9c, + 0x0b, 0xd9, 0x0c, 0xcf, 0x5d, 0x7f, 0xd0, 0xed, + 0x57, 0xfe, 0x58, 0x5e, 0x4b, 0xe3, 0x27, 0x1b, + 0x0a, 0x6b, 0xf0, 0xf5, 0x78, 0x6b, 0x0f, 0x26 }, + { 0xf1, 0xb0, 0x15, 0x58, 0xce, 0x54, 0x12, 0x62, + 0xf5, 0xec, 0x34, 0x29, 0x9d, 0x6f, 0xb4, 0x09, + 0x00, 0x09, 0xe3, 0x43, 0x4b, 0xe2, 0xf4, 0x91, + 0x05, 0xcf, 0x46, 0xaf, 0x4d, 0x2d, 0x41, 0x24 }, + { 0x13, 0xa0, 0xa0, 0xc8, 0x63, 0x35, 0x63, 0x5e, + 0xaa, 0x74, 0xca, 0x2d, 0x5d, 0x48, 0x8c, 0x79, + 0x7b, 0xbb, 0x4f, 0x47, 0xdc, 0x07, 0x10, 0x50, + 0x15, 0xed, 0x6a, 0x1f, 0x33, 0x09, 0xef, 0xce }, + { 0x15, 0x80, 0xaf, 0xee, 0xbe, 0xbb, 0x34, 0x6f, + 0x94, 0xd5, 0x9f, 0xe6, 0x2d, 0xa0, 0xb7, 0x92, + 0x37, 0xea, 0xd7, 0xb1, 0x49, 0x1f, 0x56, 0x67, + 0xa9, 0x0e, 0x45, 0xed, 0xf6, 0xca, 0x8b, 0x03 }, + { 0x20, 0xbe, 0x1a, 0x87, 0x5b, 0x38, 0xc5, 0x73, + 0xdd, 0x7f, 0xaa, 0xa0, 0xde, 0x48, 0x9d, 0x65, + 0x5c, 0x11, 0xef, 0xb6, 0xa5, 0x52, 0x69, 0x8e, + 0x07, 0xa2, 0xd3, 0x31, 0xb5, 0xf6, 0x55, 0xc3 }, + { 0xbe, 0x1f, 0xe3, 0xc4, 0xc0, 0x40, 0x18, 0xc5, + 0x4c, 0x4a, 0x0f, 0x6b, 0x9a, 0x2e, 0xd3, 0xc5, + 0x3a, 0xbe, 0x3a, 0x9f, 0x76, 0xb4, 0xd2, 0x6d, + 0xe5, 0x6f, 0xc9, 0xae, 0x95, 0x05, 0x9a, 0x99 }, + { 0xe3, 0xe3, 0xac, 0xe5, 0x37, 0xeb, 0x3e, 0xdd, + 0x84, 0x63, 0xd9, 0xad, 0x35, 0x82, 0xe1, 0x3c, + 0xf8, 0x65, 0x33, 0xff, 0xde, 0x43, 0xd6, 0x68, + 0xdd, 0x2e, 0x93, 0xbb, 0xdb, 0xd7, 0x19, 0x5a }, + { 0x11, 0x0c, 0x50, 0xc0, 0xbf, 0x2c, 0x6e, 0x7a, + 0xeb, 0x7e, 0x43, 0x5d, 0x92, 0xd1, 0x32, 0xab, + 0x66, 0x55, 0x16, 0x8e, 0x78, 0xa2, 0xde, 0xcd, + 0xec, 0x33, 0x30, 0x77, 0x76, 0x84, 0xd9, 0xc1 }, + { 0xe9, 0xba, 0x8f, 0x50, 0x5c, 0x9c, 0x80, 0xc0, + 0x86, 0x66, 0xa7, 0x01, 0xf3, 0x36, 0x7e, 0x6c, + 0xc6, 0x65, 0xf3, 0x4b, 0x22, 0xe7, 0x3c, 0x3c, + 0x04, 0x17, 0xeb, 0x1c, 0x22, 0x06, 0x08, 0x2f }, + { 0x26, 0xcd, 0x66, 0xfc, 0xa0, 0x23, 0x79, 0xc7, + 0x6d, 0xf1, 0x23, 0x17, 0x05, 0x2b, 0xca, 0xfd, + 0x6c, 0xd8, 0xc3, 0xa7, 0xb8, 0x90, 0xd8, 0x05, + 0xf3, 0x6c, 0x49, 0x98, 0x97, 0x82, 0x43, 0x3a }, + { 0x21, 0x3f, 0x35, 0x96, 0xd6, 0xe3, 0xa5, 0xd0, + 0xe9, 0x93, 0x2c, 0xd2, 0x15, 0x91, 0x46, 0x01, + 0x5e, 0x2a, 0xbc, 0x94, 0x9f, 0x47, 0x29, 0xee, + 0x26, 0x32, 0xfe, 0x1e, 0xdb, 0x78, 0xd3, 0x37 }, + { 0x10, 0x15, 0xd7, 0x01, 0x08, 0xe0, 0x3b, 0xe1, + 0xc7, 0x02, 0xfe, 0x97, 0x25, 0x36, 0x07, 0xd1, + 0x4a, 0xee, 0x59, 0x1f, 0x24, 0x13, 0xea, 0x67, + 0x87, 0x42, 0x7b, 0x64, 0x59, 0xff, 0x21, 0x9a }, + { 0x3c, 0xa9, 0x89, 0xde, 0x10, 0xcf, 0xe6, 0x09, + 0x90, 0x94, 0x72, 0xc8, 0xd3, 0x56, 0x10, 0x80, + 0x5b, 0x2f, 0x97, 0x77, 0x34, 0xcf, 0x65, 0x2c, + 0xc6, 0x4b, 0x3b, 0xfc, 0x88, 0x2d, 0x5d, 0x89 }, + { 0xb6, 0x15, 0x6f, 0x72, 0xd3, 0x80, 0xee, 0x9e, + 0xa6, 0xac, 0xd1, 0x90, 0x46, 0x4f, 0x23, 0x07, + 0xa5, 0xc1, 0x79, 0xef, 0x01, 0xfd, 0x71, 0xf9, + 0x9f, 0x2d, 0x0f, 0x7a, 0x57, 0x36, 0x0a, 0xea }, + { 0xc0, 0x3b, 0xc6, 0x42, 0xb2, 0x09, 0x59, 0xcb, + 0xe1, 0x33, 0xa0, 0x30, 0x3e, 0x0c, 0x1a, 0xbf, + 0xf3, 0xe3, 0x1e, 0xc8, 0xe1, 0xa3, 0x28, 0xec, + 0x85, 0x65, 0xc3, 0x6d, 0xec, 0xff, 0x52, 0x65 }, + { 0x2c, 0x3e, 0x08, 0x17, 0x6f, 0x76, 0x0c, 0x62, + 0x64, 0xc3, 0xa2, 0xcd, 0x66, 0xfe, 0xc6, 0xc3, + 0xd7, 0x8d, 0xe4, 0x3f, 0xc1, 0x92, 0x45, 0x7b, + 0x2a, 0x4a, 0x66, 0x0a, 0x1e, 0x0e, 0xb2, 0x2b }, + { 0xf7, 0x38, 0xc0, 0x2f, 0x3c, 0x1b, 0x19, 0x0c, + 0x51, 0x2b, 0x1a, 0x32, 0xde, 0xab, 0xf3, 0x53, + 0x72, 0x8e, 0x0e, 0x9a, 0xb0, 0x34, 0x49, 0x0e, + 0x3c, 0x34, 0x09, 0x94, 0x6a, 0x97, 0xae, 0xec }, + { 0x8b, 0x18, 0x80, 0xdf, 0x30, 0x1c, 0xc9, 0x63, + 0x41, 0x88, 0x11, 0x08, 0x89, 0x64, 0x83, 0x92, + 0x87, 0xff, 0x7f, 0xe3, 0x1c, 0x49, 0xea, 0x6e, + 0xbd, 0x9e, 0x48, 0xbd, 0xee, 0xe4, 0x97, 0xc5 }, + { 0x1e, 0x75, 0xcb, 0x21, 0xc6, 0x09, 0x89, 0x02, + 0x03, 0x75, 0xf1, 0xa7, 0xa2, 0x42, 0x83, 0x9f, + 0x0b, 0x0b, 0x68, 0x97, 0x3a, 0x4c, 0x2a, 0x05, + 0xcf, 0x75, 0x55, 0xed, 0x5a, 0xae, 0xc4, 0xc1 }, + { 0x62, 0xbf, 0x8a, 0x9c, 0x32, 0xa5, 0xbc, 0xcf, + 0x29, 0x0b, 0x6c, 0x47, 0x4d, 0x75, 0xb2, 0xa2, + 0xa4, 0x09, 0x3f, 0x1a, 0x9e, 0x27, 0x13, 0x94, + 0x33, 0xa8, 0xf2, 0xb3, 0xbc, 0xe7, 0xb8, 0xd7 }, + { 0x16, 0x6c, 0x83, 0x50, 0xd3, 0x17, 0x3b, 0x5e, + 0x70, 0x2b, 0x78, 0x3d, 0xfd, 0x33, 0xc6, 0x6e, + 0xe0, 0x43, 0x27, 0x42, 0xe9, 0xb9, 0x2b, 0x99, + 0x7f, 0xd2, 0x3c, 0x60, 0xdc, 0x67, 0x56, 0xca }, + { 0x04, 0x4a, 0x14, 0xd8, 0x22, 0xa9, 0x0c, 0xac, + 0xf2, 0xf5, 0xa1, 0x01, 0x42, 0x8a, 0xdc, 0x8f, + 0x41, 0x09, 0x38, 0x6c, 0xcb, 0x15, 0x8b, 0xf9, + 0x05, 0xc8, 0x61, 0x8b, 0x8e, 0xe2, 0x4e, 0xc3 }, + { 0x38, 0x7d, 0x39, 0x7e, 0xa4, 0x3a, 0x99, 0x4b, + 0xe8, 0x4d, 0x2d, 0x54, 0x4a, 0xfb, 0xe4, 0x81, + 0xa2, 0x00, 0x0f, 0x55, 0x25, 0x26, 0x96, 0xbb, + 0xa2, 0xc5, 0x0c, 0x8e, 0xbd, 0x10, 0x13, 0x47 }, + { 0x56, 0xf8, 0xcc, 0xf1, 0xf8, 0x64, 0x09, 0xb4, + 0x6c, 0xe3, 0x61, 0x66, 0xae, 0x91, 0x65, 0x13, + 0x84, 0x41, 0x57, 0x75, 0x89, 0xdb, 0x08, 0xcb, + 0xc5, 0xf6, 0x6c, 0xa2, 0x97, 0x43, 0xb9, 0xfd }, + { 0x97, 0x06, 0xc0, 0x92, 0xb0, 0x4d, 0x91, 0xf5, + 0x3d, 0xff, 0x91, 0xfa, 0x37, 0xb7, 0x49, 0x3d, + 0x28, 0xb5, 0x76, 0xb5, 0xd7, 0x10, 0x46, 0x9d, + 0xf7, 0x94, 0x01, 0x66, 0x22, 0x36, 0xfc, 0x03 }, + { 0x87, 0x79, 0x68, 0x68, 0x6c, 0x06, 0x8c, 0xe2, + 0xf7, 0xe2, 0xad, 0xcf, 0xf6, 0x8b, 0xf8, 0x74, + 0x8e, 0xdf, 0x3c, 0xf8, 0x62, 0xcf, 0xb4, 0xd3, + 0x94, 0x7a, 0x31, 0x06, 0x95, 0x80, 0x54, 0xe3 }, + { 0x88, 0x17, 0xe5, 0x71, 0x98, 0x79, 0xac, 0xf7, + 0x02, 0x47, 0x87, 0xec, 0xcd, 0xb2, 0x71, 0x03, + 0x55, 0x66, 0xcf, 0xa3, 0x33, 0xe0, 0x49, 0x40, + 0x7c, 0x01, 0x78, 0xcc, 0xc5, 0x7a, 0x5b, 0x9f }, + { 0x89, 0x38, 0x24, 0x9e, 0x4b, 0x50, 0xca, 0xda, + 0xcc, 0xdf, 0x5b, 0x18, 0x62, 0x13, 0x26, 0xcb, + 0xb1, 0x52, 0x53, 0xe3, 0x3a, 0x20, 0xf5, 0x63, + 0x6e, 0x99, 0x5d, 0x72, 0x47, 0x8d, 0xe4, 0x72 }, + { 0xf1, 0x64, 0xab, 0xba, 0x49, 0x63, 0xa4, 0x4d, + 0x10, 0x72, 0x57, 0xe3, 0x23, 0x2d, 0x90, 0xac, + 0xa5, 0xe6, 0x6a, 0x14, 0x08, 0x24, 0x8c, 0x51, + 0x74, 0x1e, 0x99, 0x1d, 0xb5, 0x22, 0x77, 0x56 }, + { 0xd0, 0x55, 0x63, 0xe2, 0xb1, 0xcb, 0xa0, 0xc4, + 0xa2, 0xa1, 0xe8, 0xbd, 0xe3, 0xa1, 0xa0, 0xd9, + 0xf5, 0xb4, 0x0c, 0x85, 0xa0, 0x70, 0xd6, 0xf5, + 0xfb, 0x21, 0x06, 0x6e, 0xad, 0x5d, 0x06, 0x01 }, + { 0x03, 0xfb, 0xb1, 0x63, 0x84, 0xf0, 0xa3, 0x86, + 0x6f, 0x4c, 0x31, 0x17, 0x87, 0x76, 0x66, 0xef, + 0xbf, 0x12, 0x45, 0x97, 0x56, 0x4b, 0x29, 0x3d, + 0x4a, 0xab, 0x0d, 0x26, 0x9f, 0xab, 0xdd, 0xfa }, + { 0x5f, 0xa8, 0x48, 0x6a, 0xc0, 0xe5, 0x29, 0x64, + 0xd1, 0x88, 0x1b, 0xbe, 0x33, 0x8e, 0xb5, 0x4b, + 0xe2, 0xf7, 0x19, 0x54, 0x92, 0x24, 0x89, 0x20, + 0x57, 0xb4, 0xda, 0x04, 0xba, 0x8b, 0x34, 0x75 }, + { 0xcd, 0xfa, 0xbc, 0xee, 0x46, 0x91, 0x11, 0x11, + 0x23, 0x6a, 0x31, 0x70, 0x8b, 0x25, 0x39, 0xd7, + 0x1f, 0xc2, 0x11, 0xd9, 0xb0, 0x9c, 0x0d, 0x85, + 0x30, 0xa1, 0x1e, 0x1d, 0xbf, 0x6e, 0xed, 0x01 }, + { 0x4f, 0x82, 0xde, 0x03, 0xb9, 0x50, 0x47, 0x93, + 0xb8, 0x2a, 0x07, 0xa0, 0xbd, 0xcd, 0xff, 0x31, + 0x4d, 0x75, 0x9e, 0x7b, 0x62, 0xd2, 0x6b, 0x78, + 0x49, 0x46, 0xb0, 0xd3, 0x6f, 0x91, 0x6f, 0x52 }, + { 0x25, 0x9e, 0xc7, 0xf1, 0x73, 0xbc, 0xc7, 0x6a, + 0x09, 0x94, 0xc9, 0x67, 0xb4, 0xf5, 0xf0, 0x24, + 0xc5, 0x60, 0x57, 0xfb, 0x79, 0xc9, 0x65, 0xc4, + 0xfa, 0xe4, 0x18, 0x75, 0xf0, 0x6a, 0x0e, 0x4c }, + { 0x19, 0x3c, 0xc8, 0xe7, 0xc3, 0xe0, 0x8b, 0xb3, + 0x0f, 0x54, 0x37, 0xaa, 0x27, 0xad, 0xe1, 0xf1, + 0x42, 0x36, 0x9b, 0x24, 0x6a, 0x67, 0x5b, 0x23, + 0x83, 0xe6, 0xda, 0x9b, 0x49, 0xa9, 0x80, 0x9e }, + { 0x5c, 0x10, 0x89, 0x6f, 0x0e, 0x28, 0x56, 0xb2, + 0xa2, 0xee, 0xe0, 0xfe, 0x4a, 0x2c, 0x16, 0x33, + 0x56, 0x5d, 0x18, 0xf0, 0xe9, 0x3e, 0x1f, 0xab, + 0x26, 0xc3, 0x73, 0xe8, 0xf8, 0x29, 0x65, 0x4d }, + { 0xf1, 0x60, 0x12, 0xd9, 0x3f, 0x28, 0x85, 0x1a, + 0x1e, 0xb9, 0x89, 0xf5, 0xd0, 0xb4, 0x3f, 0x3f, + 0x39, 0xca, 0x73, 0xc9, 0xa6, 0x2d, 0x51, 0x81, + 0xbf, 0xf2, 0x37, 0x53, 0x6b, 0xd3, 0x48, 0xc3 }, + { 0x29, 0x66, 0xb3, 0xcf, 0xae, 0x1e, 0x44, 0xea, + 0x99, 0x6d, 0xc5, 0xd6, 0x86, 0xcf, 0x25, 0xfa, + 0x05, 0x3f, 0xb6, 0xf6, 0x72, 0x01, 0xb9, 0xe4, + 0x6e, 0xad, 0xe8, 0x5d, 0x0a, 0xd6, 0xb8, 0x06 }, + { 0xdd, 0xb8, 0x78, 0x24, 0x85, 0xe9, 0x00, 0xbc, + 0x60, 0xbc, 0xf4, 0xc3, 0x3a, 0x6f, 0xd5, 0x85, + 0x68, 0x0c, 0xc6, 0x83, 0xd5, 0x16, 0xef, 0xa0, + 0x3e, 0xb9, 0x98, 0x5f, 0xad, 0x87, 0x15, 0xfb }, + { 0x4c, 0x4d, 0x6e, 0x71, 0xae, 0xa0, 0x57, 0x86, + 0x41, 0x31, 0x48, 0xfc, 0x7a, 0x78, 0x6b, 0x0e, + 0xca, 0xf5, 0x82, 0xcf, 0xf1, 0x20, 0x9f, 0x5a, + 0x80, 0x9f, 0xba, 0x85, 0x04, 0xce, 0x66, 0x2c }, + { 0xfb, 0x4c, 0x5e, 0x86, 0xd7, 0xb2, 0x22, 0x9b, + 0x99, 0xb8, 0xba, 0x6d, 0x94, 0xc2, 0x47, 0xef, + 0x96, 0x4a, 0xa3, 0xa2, 0xba, 0xe8, 0xed, 0xc7, + 0x75, 0x69, 0xf2, 0x8d, 0xbb, 0xff, 0x2d, 0x4e }, + { 0xe9, 0x4f, 0x52, 0x6d, 0xe9, 0x01, 0x96, 0x33, + 0xec, 0xd5, 0x4a, 0xc6, 0x12, 0x0f, 0x23, 0x95, + 0x8d, 0x77, 0x18, 0xf1, 0xe7, 0x71, 0x7b, 0xf3, + 0x29, 0x21, 0x1a, 0x4f, 0xae, 0xed, 0x4e, 0x6d }, + { 0xcb, 0xd6, 0x66, 0x0a, 0x10, 0xdb, 0x3f, 0x23, + 0xf7, 0xa0, 0x3d, 0x4b, 0x9d, 0x40, 0x44, 0xc7, + 0x93, 0x2b, 0x28, 0x01, 0xac, 0x89, 0xd6, 0x0b, + 0xc9, 0xeb, 0x92, 0xd6, 0x5a, 0x46, 0xc2, 0xa0 }, + { 0x88, 0x18, 0xbb, 0xd3, 0xdb, 0x4d, 0xc1, 0x23, + 0xb2, 0x5c, 0xbb, 0xa5, 0xf5, 0x4c, 0x2b, 0xc4, + 0xb3, 0xfc, 0xf9, 0xbf, 0x7d, 0x7a, 0x77, 0x09, + 0xf4, 0xae, 0x58, 0x8b, 0x26, 0x7c, 0x4e, 0xce }, + { 0xc6, 0x53, 0x82, 0x51, 0x3f, 0x07, 0x46, 0x0d, + 0xa3, 0x98, 0x33, 0xcb, 0x66, 0x6c, 0x5e, 0xd8, + 0x2e, 0x61, 0xb9, 0xe9, 0x98, 0xf4, 0xb0, 0xc4, + 0x28, 0x7c, 0xee, 0x56, 0xc3, 0xcc, 0x9b, 0xcd }, + { 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66, + 0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26, + 0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab, + 0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4 }, + { 0x21, 0xfe, 0x0c, 0xeb, 0x00, 0x52, 0xbe, 0x7f, + 0xb0, 0xf0, 0x04, 0x18, 0x7c, 0xac, 0xd7, 0xde, + 0x67, 0xfa, 0x6e, 0xb0, 0x93, 0x8d, 0x92, 0x76, + 0x77, 0xf2, 0x39, 0x8c, 0x13, 0x23, 0x17, 0xa8 }, + { 0x2e, 0xf7, 0x3f, 0x3c, 0x26, 0xf1, 0x2d, 0x93, + 0x88, 0x9f, 0x3c, 0x78, 0xb6, 0xa6, 0x6c, 0x1d, + 0x52, 0xb6, 0x49, 0xdc, 0x9e, 0x85, 0x6e, 0x2c, + 0x17, 0x2e, 0xa7, 0xc5, 0x8a, 0xc2, 0xb5, 0xe3 }, + { 0x38, 0x8a, 0x3c, 0xd5, 0x6d, 0x73, 0x86, 0x7a, + 0xbb, 0x5f, 0x84, 0x01, 0x49, 0x2b, 0x6e, 0x26, + 0x81, 0xeb, 0x69, 0x85, 0x1e, 0x76, 0x7f, 0xd8, + 0x42, 0x10, 0xa5, 0x60, 0x76, 0xfb, 0x3d, 0xd3 }, + { 0xaf, 0x53, 0x3e, 0x02, 0x2f, 0xc9, 0x43, 0x9e, + 0x4e, 0x3c, 0xb8, 0x38, 0xec, 0xd1, 0x86, 0x92, + 0x23, 0x2a, 0xdf, 0x6f, 0xe9, 0x83, 0x95, 0x26, + 0xd3, 0xc3, 0xdd, 0x1b, 0x71, 0x91, 0x0b, 0x1a }, + { 0x75, 0x1c, 0x09, 0xd4, 0x1a, 0x93, 0x43, 0x88, + 0x2a, 0x81, 0xcd, 0x13, 0xee, 0x40, 0x81, 0x8d, + 0x12, 0xeb, 0x44, 0xc6, 0xc7, 0xf4, 0x0d, 0xf1, + 0x6e, 0x4a, 0xea, 0x8f, 0xab, 0x91, 0x97, 0x2a }, + { 0x5b, 0x73, 0xdd, 0xb6, 0x8d, 0x9d, 0x2b, 0x0a, + 0xa2, 0x65, 0xa0, 0x79, 0x88, 0xd6, 0xb8, 0x8a, + 0xe9, 0xaa, 0xc5, 0x82, 0xaf, 0x83, 0x03, 0x2f, + 0x8a, 0x9b, 0x21, 0xa2, 0xe1, 0xb7, 0xbf, 0x18 }, + { 0x3d, 0xa2, 0x91, 0x26, 0xc7, 0xc5, 0xd7, 0xf4, + 0x3e, 0x64, 0x24, 0x2a, 0x79, 0xfe, 0xaa, 0x4e, + 0xf3, 0x45, 0x9c, 0xde, 0xcc, 0xc8, 0x98, 0xed, + 0x59, 0xa9, 0x7f, 0x6e, 0xc9, 0x3b, 0x9d, 0xab }, + { 0x56, 0x6d, 0xc9, 0x20, 0x29, 0x3d, 0xa5, 0xcb, + 0x4f, 0xe0, 0xaa, 0x8a, 0xbd, 0xa8, 0xbb, 0xf5, + 0x6f, 0x55, 0x23, 0x13, 0xbf, 0xf1, 0x90, 0x46, + 0x64, 0x1e, 0x36, 0x15, 0xc1, 0xe3, 0xed, 0x3f }, + { 0x41, 0x15, 0xbe, 0xa0, 0x2f, 0x73, 0xf9, 0x7f, + 0x62, 0x9e, 0x5c, 0x55, 0x90, 0x72, 0x0c, 0x01, + 0xe7, 0xe4, 0x49, 0xae, 0x2a, 0x66, 0x97, 0xd4, + 0xd2, 0x78, 0x33, 0x21, 0x30, 0x36, 0x92, 0xf9 }, + { 0x4c, 0xe0, 0x8f, 0x47, 0x62, 0x46, 0x8a, 0x76, + 0x70, 0x01, 0x21, 0x64, 0x87, 0x8d, 0x68, 0x34, + 0x0c, 0x52, 0xa3, 0x5e, 0x66, 0xc1, 0x88, 0x4d, + 0x5c, 0x86, 0x48, 0x89, 0xab, 0xc9, 0x66, 0x77 }, + { 0x81, 0xea, 0x0b, 0x78, 0x04, 0x12, 0x4e, 0x0c, + 0x22, 0xea, 0x5f, 0xc7, 0x11, 0x04, 0xa2, 0xaf, + 0xcb, 0x52, 0xa1, 0xfa, 0x81, 0x6f, 0x3e, 0xcb, + 0x7d, 0xcb, 0x5d, 0x9d, 0xea, 0x17, 0x86, 0xd0 }, + { 0xfe, 0x36, 0x27, 0x33, 0xb0, 0x5f, 0x6b, 0xed, + 0xaf, 0x93, 0x79, 0xd7, 0xf7, 0x93, 0x6e, 0xde, + 0x20, 0x9b, 0x1f, 0x83, 0x23, 0xc3, 0x92, 0x25, + 0x49, 0xd9, 0xe7, 0x36, 0x81, 0xb5, 0xdb, 0x7b }, + { 0xef, 0xf3, 0x7d, 0x30, 0xdf, 0xd2, 0x03, 0x59, + 0xbe, 0x4e, 0x73, 0xfd, 0xf4, 0x0d, 0x27, 0x73, + 0x4b, 0x3d, 0xf9, 0x0a, 0x97, 0xa5, 0x5e, 0xd7, + 0x45, 0x29, 0x72, 0x94, 0xca, 0x85, 0xd0, 0x9f }, + { 0x17, 0x2f, 0xfc, 0x67, 0x15, 0x3d, 0x12, 0xe0, + 0xca, 0x76, 0xa8, 0xb6, 0xcd, 0x5d, 0x47, 0x31, + 0x88, 0x5b, 0x39, 0xce, 0x0c, 0xac, 0x93, 0xa8, + 0x97, 0x2a, 0x18, 0x00, 0x6c, 0x8b, 0x8b, 0xaf }, + { 0xc4, 0x79, 0x57, 0xf1, 0xcc, 0x88, 0xe8, 0x3e, + 0xf9, 0x44, 0x58, 0x39, 0x70, 0x9a, 0x48, 0x0a, + 0x03, 0x6b, 0xed, 0x5f, 0x88, 0xac, 0x0f, 0xcc, + 0x8e, 0x1e, 0x70, 0x3f, 0xfa, 0xac, 0x13, 0x2c }, + { 0x30, 0xf3, 0x54, 0x83, 0x70, 0xcf, 0xdc, 0xed, + 0xa5, 0xc3, 0x7b, 0x56, 0x9b, 0x61, 0x75, 0xe7, + 0x99, 0xee, 0xf1, 0xa6, 0x2a, 0xaa, 0x94, 0x32, + 0x45, 0xae, 0x76, 0x69, 0xc2, 0x27, 0xa7, 0xb5 }, + { 0xc9, 0x5d, 0xcb, 0x3c, 0xf1, 0xf2, 0x7d, 0x0e, + 0xef, 0x2f, 0x25, 0xd2, 0x41, 0x38, 0x70, 0x90, + 0x4a, 0x87, 0x7c, 0x4a, 0x56, 0xc2, 0xde, 0x1e, + 0x83, 0xe2, 0xbc, 0x2a, 0xe2, 0xe4, 0x68, 0x21 }, + { 0xd5, 0xd0, 0xb5, 0xd7, 0x05, 0x43, 0x4c, 0xd4, + 0x6b, 0x18, 0x57, 0x49, 0xf6, 0x6b, 0xfb, 0x58, + 0x36, 0xdc, 0xdf, 0x6e, 0xe5, 0x49, 0xa2, 0xb7, + 0xa4, 0xae, 0xe7, 0xf5, 0x80, 0x07, 0xca, 0xaf }, + { 0xbb, 0xc1, 0x24, 0xa7, 0x12, 0xf1, 0x5d, 0x07, + 0xc3, 0x00, 0xe0, 0x5b, 0x66, 0x83, 0x89, 0xa4, + 0x39, 0xc9, 0x17, 0x77, 0xf7, 0x21, 0xf8, 0x32, + 0x0c, 0x1c, 0x90, 0x78, 0x06, 0x6d, 0x2c, 0x7e }, + { 0xa4, 0x51, 0xb4, 0x8c, 0x35, 0xa6, 0xc7, 0x85, + 0x4c, 0xfa, 0xae, 0x60, 0x26, 0x2e, 0x76, 0x99, + 0x08, 0x16, 0x38, 0x2a, 0xc0, 0x66, 0x7e, 0x5a, + 0x5c, 0x9e, 0x1b, 0x46, 0xc4, 0x34, 0x2d, 0xdf }, + { 0xb0, 0xd1, 0x50, 0xfb, 0x55, 0xe7, 0x78, 0xd0, + 0x11, 0x47, 0xf0, 0xb5, 0xd8, 0x9d, 0x99, 0xec, + 0xb2, 0x0f, 0xf0, 0x7e, 0x5e, 0x67, 0x60, 0xd6, + 0xb6, 0x45, 0xeb, 0x5b, 0x65, 0x4c, 0x62, 0x2b }, + { 0x34, 0xf7, 0x37, 0xc0, 0xab, 0x21, 0x99, 0x51, + 0xee, 0xe8, 0x9a, 0x9f, 0x8d, 0xac, 0x29, 0x9c, + 0x9d, 0x4c, 0x38, 0xf3, 0x3f, 0xa4, 0x94, 0xc5, + 0xc6, 0xee, 0xfc, 0x92, 0xb6, 0xdb, 0x08, 0xbc }, + { 0x1a, 0x62, 0xcc, 0x3a, 0x00, 0x80, 0x0d, 0xcb, + 0xd9, 0x98, 0x91, 0x08, 0x0c, 0x1e, 0x09, 0x84, + 0x58, 0x19, 0x3a, 0x8c, 0xc9, 0xf9, 0x70, 0xea, + 0x99, 0xfb, 0xef, 0xf0, 0x03, 0x18, 0xc2, 0x89 }, + { 0xcf, 0xce, 0x55, 0xeb, 0xaf, 0xc8, 0x40, 0xd7, + 0xae, 0x48, 0x28, 0x1c, 0x7f, 0xd5, 0x7e, 0xc8, + 0xb4, 0x82, 0xd4, 0xb7, 0x04, 0x43, 0x74, 0x95, + 0x49, 0x5a, 0xc4, 0x14, 0xcf, 0x4a, 0x37, 0x4b }, + { 0x67, 0x46, 0xfa, 0xcf, 0x71, 0x14, 0x6d, 0x99, + 0x9d, 0xab, 0xd0, 0x5d, 0x09, 0x3a, 0xe5, 0x86, + 0x64, 0x8d, 0x1e, 0xe2, 0x8e, 0x72, 0x61, 0x7b, + 0x99, 0xd0, 0xf0, 0x08, 0x6e, 0x1e, 0x45, 0xbf }, + { 0x57, 0x1c, 0xed, 0x28, 0x3b, 0x3f, 0x23, 0xb4, + 0xe7, 0x50, 0xbf, 0x12, 0xa2, 0xca, 0xf1, 0x78, + 0x18, 0x47, 0xbd, 0x89, 0x0e, 0x43, 0x60, 0x3c, + 0xdc, 0x59, 0x76, 0x10, 0x2b, 0x7b, 0xb1, 0x1b }, + { 0xcf, 0xcb, 0x76, 0x5b, 0x04, 0x8e, 0x35, 0x02, + 0x2c, 0x5d, 0x08, 0x9d, 0x26, 0xe8, 0x5a, 0x36, + 0xb0, 0x05, 0xa2, 0xb8, 0x04, 0x93, 0xd0, 0x3a, + 0x14, 0x4e, 0x09, 0xf4, 0x09, 0xb6, 0xaf, 0xd1 }, + { 0x40, 0x50, 0xc7, 0xa2, 0x77, 0x05, 0xbb, 0x27, + 0xf4, 0x20, 0x89, 0xb2, 0x99, 0xf3, 0xcb, 0xe5, + 0x05, 0x4e, 0xad, 0x68, 0x72, 0x7e, 0x8e, 0xf9, + 0x31, 0x8c, 0xe6, 0xf2, 0x5c, 0xd6, 0xf3, 0x1d }, + { 0x18, 0x40, 0x70, 0xbd, 0x5d, 0x26, 0x5f, 0xbd, + 0xc1, 0x42, 0xcd, 0x1c, 0x5c, 0xd0, 0xd7, 0xe4, + 0x14, 0xe7, 0x03, 0x69, 0xa2, 0x66, 0xd6, 0x27, + 0xc8, 0xfb, 0xa8, 0x4f, 0xa5, 0xe8, 0x4c, 0x34 }, + { 0x9e, 0xdd, 0xa9, 0xa4, 0x44, 0x39, 0x02, 0xa9, + 0x58, 0x8c, 0x0d, 0x0c, 0xcc, 0x62, 0xb9, 0x30, + 0x21, 0x84, 0x79, 0xa6, 0x84, 0x1e, 0x6f, 0xe7, + 0xd4, 0x30, 0x03, 0xf0, 0x4b, 0x1f, 0xd6, 0x43 }, + { 0xe4, 0x12, 0xfe, 0xef, 0x79, 0x08, 0x32, 0x4a, + 0x6d, 0xa1, 0x84, 0x16, 0x29, 0xf3, 0x5d, 0x3d, + 0x35, 0x86, 0x42, 0x01, 0x93, 0x10, 0xec, 0x57, + 0xc6, 0x14, 0x83, 0x6b, 0x63, 0xd3, 0x07, 0x63 }, + { 0x1a, 0x2b, 0x8e, 0xdf, 0xf3, 0xf9, 0xac, 0xc1, + 0x55, 0x4f, 0xcb, 0xae, 0x3c, 0xf1, 0xd6, 0x29, + 0x8c, 0x64, 0x62, 0xe2, 0x2e, 0x5e, 0xb0, 0x25, + 0x96, 0x84, 0xf8, 0x35, 0x01, 0x2b, 0xd1, 0x3f }, + { 0x28, 0x8c, 0x4a, 0xd9, 0xb9, 0x40, 0x97, 0x62, + 0xea, 0x07, 0xc2, 0x4a, 0x41, 0xf0, 0x4f, 0x69, + 0xa7, 0xd7, 0x4b, 0xee, 0x2d, 0x95, 0x43, 0x53, + 0x74, 0xbd, 0xe9, 0x46, 0xd7, 0x24, 0x1c, 0x7b }, + { 0x80, 0x56, 0x91, 0xbb, 0x28, 0x67, 0x48, 0xcf, + 0xb5, 0x91, 0xd3, 0xae, 0xbe, 0x7e, 0x6f, 0x4e, + 0x4d, 0xc6, 0xe2, 0x80, 0x8c, 0x65, 0x14, 0x3c, + 0xc0, 0x04, 0xe4, 0xeb, 0x6f, 0xd0, 0x9d, 0x43 }, + { 0xd4, 0xac, 0x8d, 0x3a, 0x0a, 0xfc, 0x6c, 0xfa, + 0x7b, 0x46, 0x0a, 0xe3, 0x00, 0x1b, 0xae, 0xb3, + 0x6d, 0xad, 0xb3, 0x7d, 0xa0, 0x7d, 0x2e, 0x8a, + 0xc9, 0x18, 0x22, 0xdf, 0x34, 0x8a, 0xed, 0x3d }, + { 0xc3, 0x76, 0x61, 0x70, 0x14, 0xd2, 0x01, 0x58, + 0xbc, 0xed, 0x3d, 0x3b, 0xa5, 0x52, 0xb6, 0xec, + 0xcf, 0x84, 0xe6, 0x2a, 0xa3, 0xeb, 0x65, 0x0e, + 0x90, 0x02, 0x9c, 0x84, 0xd1, 0x3e, 0xea, 0x69 }, + { 0xc4, 0x1f, 0x09, 0xf4, 0x3c, 0xec, 0xae, 0x72, + 0x93, 0xd6, 0x00, 0x7c, 0xa0, 0xa3, 0x57, 0x08, + 0x7d, 0x5a, 0xe5, 0x9b, 0xe5, 0x00, 0xc1, 0xcd, + 0x5b, 0x28, 0x9e, 0xe8, 0x10, 0xc7, 0xb0, 0x82 }, + { 0x03, 0xd1, 0xce, 0xd1, 0xfb, 0xa5, 0xc3, 0x91, + 0x55, 0xc4, 0x4b, 0x77, 0x65, 0xcb, 0x76, 0x0c, + 0x78, 0x70, 0x8d, 0xcf, 0xc8, 0x0b, 0x0b, 0xd8, + 0xad, 0xe3, 0xa5, 0x6d, 0xa8, 0x83, 0x0b, 0x29 }, + { 0x09, 0xbd, 0xe6, 0xf1, 0x52, 0x21, 0x8d, 0xc9, + 0x2c, 0x41, 0xd7, 0xf4, 0x53, 0x87, 0xe6, 0x3e, + 0x58, 0x69, 0xd8, 0x07, 0xec, 0x70, 0xb8, 0x21, + 0x40, 0x5d, 0xbd, 0x88, 0x4b, 0x7f, 0xcf, 0x4b }, + { 0x71, 0xc9, 0x03, 0x6e, 0x18, 0x17, 0x9b, 0x90, + 0xb3, 0x7d, 0x39, 0xe9, 0xf0, 0x5e, 0xb8, 0x9c, + 0xc5, 0xfc, 0x34, 0x1f, 0xd7, 0xc4, 0x77, 0xd0, + 0xd7, 0x49, 0x32, 0x85, 0xfa, 0xca, 0x08, 0xa4 }, + { 0x59, 0x16, 0x83, 0x3e, 0xbb, 0x05, 0xcd, 0x91, + 0x9c, 0xa7, 0xfe, 0x83, 0xb6, 0x92, 0xd3, 0x20, + 0x5b, 0xef, 0x72, 0x39, 0x2b, 0x2c, 0xf6, 0xbb, + 0x0a, 0x6d, 0x43, 0xf9, 0x94, 0xf9, 0x5f, 0x11 }, + { 0xf6, 0x3a, 0xab, 0x3e, 0xc6, 0x41, 0xb3, 0xb0, + 0x24, 0x96, 0x4c, 0x2b, 0x43, 0x7c, 0x04, 0xf6, + 0x04, 0x3c, 0x4c, 0x7e, 0x02, 0x79, 0x23, 0x99, + 0x95, 0x40, 0x19, 0x58, 0xf8, 0x6b, 0xbe, 0x54 }, + { 0xf1, 0x72, 0xb1, 0x80, 0xbf, 0xb0, 0x97, 0x40, + 0x49, 0x31, 0x20, 0xb6, 0x32, 0x6c, 0xbd, 0xc5, + 0x61, 0xe4, 0x77, 0xde, 0xf9, 0xbb, 0xcf, 0xd2, + 0x8c, 0xc8, 0xc1, 0xc5, 0xe3, 0x37, 0x9a, 0x31 }, + { 0xcb, 0x9b, 0x89, 0xcc, 0x18, 0x38, 0x1d, 0xd9, + 0x14, 0x1a, 0xde, 0x58, 0x86, 0x54, 0xd4, 0xe6, + 0xa2, 0x31, 0xd5, 0xbf, 0x49, 0xd4, 0xd5, 0x9a, + 0xc2, 0x7d, 0x86, 0x9c, 0xbe, 0x10, 0x0c, 0xf3 }, + { 0x7b, 0xd8, 0x81, 0x50, 0x46, 0xfd, 0xd8, 0x10, + 0xa9, 0x23, 0xe1, 0x98, 0x4a, 0xae, 0xbd, 0xcd, + 0xf8, 0x4d, 0x87, 0xc8, 0x99, 0x2d, 0x68, 0xb5, + 0xee, 0xb4, 0x60, 0xf9, 0x3e, 0xb3, 0xc8, 0xd7 }, + { 0x60, 0x7b, 0xe6, 0x68, 0x62, 0xfd, 0x08, 0xee, + 0x5b, 0x19, 0xfa, 0xca, 0xc0, 0x9d, 0xfd, 0xbc, + 0xd4, 0x0c, 0x31, 0x21, 0x01, 0xd6, 0x6e, 0x6e, + 0xbd, 0x2b, 0x84, 0x1f, 0x1b, 0x9a, 0x93, 0x25 }, + { 0x9f, 0xe0, 0x3b, 0xbe, 0x69, 0xab, 0x18, 0x34, + 0xf5, 0x21, 0x9b, 0x0d, 0xa8, 0x8a, 0x08, 0xb3, + 0x0a, 0x66, 0xc5, 0x91, 0x3f, 0x01, 0x51, 0x96, + 0x3c, 0x36, 0x05, 0x60, 0xdb, 0x03, 0x87, 0xb3 }, + { 0x90, 0xa8, 0x35, 0x85, 0x71, 0x7b, 0x75, 0xf0, + 0xe9, 0xb7, 0x25, 0xe0, 0x55, 0xee, 0xee, 0xb9, + 0xe7, 0xa0, 0x28, 0xea, 0x7e, 0x6c, 0xbc, 0x07, + 0xb2, 0x09, 0x17, 0xec, 0x03, 0x63, 0xe3, 0x8c }, + { 0x33, 0x6e, 0xa0, 0x53, 0x0f, 0x4a, 0x74, 0x69, + 0x12, 0x6e, 0x02, 0x18, 0x58, 0x7e, 0xbb, 0xde, + 0x33, 0x58, 0xa0, 0xb3, 0x1c, 0x29, 0xd2, 0x00, + 0xf7, 0xdc, 0x7e, 0xb1, 0x5c, 0x6a, 0xad, 0xd8 }, + { 0xa7, 0x9e, 0x76, 0xdc, 0x0a, 0xbc, 0xa4, 0x39, + 0x6f, 0x07, 0x47, 0xcd, 0x7b, 0x74, 0x8d, 0xf9, + 0x13, 0x00, 0x76, 0x26, 0xb1, 0xd6, 0x59, 0xda, + 0x0c, 0x1f, 0x78, 0xb9, 0x30, 0x3d, 0x01, 0xa3 }, + { 0x44, 0xe7, 0x8a, 0x77, 0x37, 0x56, 0xe0, 0x95, + 0x15, 0x19, 0x50, 0x4d, 0x70, 0x38, 0xd2, 0x8d, + 0x02, 0x13, 0xa3, 0x7e, 0x0c, 0xe3, 0x75, 0x37, + 0x17, 0x57, 0xbc, 0x99, 0x63, 0x11, 0xe3, 0xb8 }, + { 0x77, 0xac, 0x01, 0x2a, 0x3f, 0x75, 0x4d, 0xcf, + 0xea, 0xb5, 0xeb, 0x99, 0x6b, 0xe9, 0xcd, 0x2d, + 0x1f, 0x96, 0x11, 0x1b, 0x6e, 0x49, 0xf3, 0x99, + 0x4d, 0xf1, 0x81, 0xf2, 0x85, 0x69, 0xd8, 0x25 }, + { 0xce, 0x5a, 0x10, 0xdb, 0x6f, 0xcc, 0xda, 0xf1, + 0x40, 0xaa, 0xa4, 0xde, 0xd6, 0x25, 0x0a, 0x9c, + 0x06, 0xe9, 0x22, 0x2b, 0xc9, 0xf9, 0xf3, 0x65, + 0x8a, 0x4a, 0xff, 0x93, 0x5f, 0x2b, 0x9f, 0x3a }, + { 0xec, 0xc2, 0x03, 0xa7, 0xfe, 0x2b, 0xe4, 0xab, + 0xd5, 0x5b, 0xb5, 0x3e, 0x6e, 0x67, 0x35, 0x72, + 0xe0, 0x07, 0x8d, 0xa8, 0xcd, 0x37, 0x5e, 0xf4, + 0x30, 0xcc, 0x97, 0xf9, 0xf8, 0x00, 0x83, 0xaf }, + { 0x14, 0xa5, 0x18, 0x6d, 0xe9, 0xd7, 0xa1, 0x8b, + 0x04, 0x12, 0xb8, 0x56, 0x3e, 0x51, 0xcc, 0x54, + 0x33, 0x84, 0x0b, 0x4a, 0x12, 0x9a, 0x8f, 0xf9, + 0x63, 0xb3, 0x3a, 0x3c, 0x4a, 0xfe, 0x8e, 0xbb }, + { 0x13, 0xf8, 0xef, 0x95, 0xcb, 0x86, 0xe6, 0xa6, + 0x38, 0x93, 0x1c, 0x8e, 0x10, 0x76, 0x73, 0xeb, + 0x76, 0xba, 0x10, 0xd7, 0xc2, 0xcd, 0x70, 0xb9, + 0xd9, 0x92, 0x0b, 0xbe, 0xed, 0x92, 0x94, 0x09 }, + { 0x0b, 0x33, 0x8f, 0x4e, 0xe1, 0x2f, 0x2d, 0xfc, + 0xb7, 0x87, 0x13, 0x37, 0x79, 0x41, 0xe0, 0xb0, + 0x63, 0x21, 0x52, 0x58, 0x1d, 0x13, 0x32, 0x51, + 0x6e, 0x4a, 0x2c, 0xab, 0x19, 0x42, 0xcc, 0xa4 }, + { 0xea, 0xab, 0x0e, 0xc3, 0x7b, 0x3b, 0x8a, 0xb7, + 0x96, 0xe9, 0xf5, 0x72, 0x38, 0xde, 0x14, 0xa2, + 0x64, 0xa0, 0x76, 0xf3, 0x88, 0x7d, 0x86, 0xe2, + 0x9b, 0xb5, 0x90, 0x6d, 0xb5, 0xa0, 0x0e, 0x02 }, + { 0x23, 0xcb, 0x68, 0xb8, 0xc0, 0xe6, 0xdc, 0x26, + 0xdc, 0x27, 0x76, 0x6d, 0xdc, 0x0a, 0x13, 0xa9, + 0x94, 0x38, 0xfd, 0x55, 0x61, 0x7a, 0xa4, 0x09, + 0x5d, 0x8f, 0x96, 0x97, 0x20, 0xc8, 0x72, 0xdf }, + { 0x09, 0x1d, 0x8e, 0xe3, 0x0d, 0x6f, 0x29, 0x68, + 0xd4, 0x6b, 0x68, 0x7d, 0xd6, 0x52, 0x92, 0x66, + 0x57, 0x42, 0xde, 0x0b, 0xb8, 0x3d, 0xcc, 0x00, + 0x04, 0xc7, 0x2c, 0xe1, 0x00, 0x07, 0xa5, 0x49 }, + { 0x7f, 0x50, 0x7a, 0xbc, 0x6d, 0x19, 0xba, 0x00, + 0xc0, 0x65, 0xa8, 0x76, 0xec, 0x56, 0x57, 0x86, + 0x88, 0x82, 0xd1, 0x8a, 0x22, 0x1b, 0xc4, 0x6c, + 0x7a, 0x69, 0x12, 0x54, 0x1f, 0x5b, 0xc7, 0xba }, + { 0xa0, 0x60, 0x7c, 0x24, 0xe1, 0x4e, 0x8c, 0x22, + 0x3d, 0xb0, 0xd7, 0x0b, 0x4d, 0x30, 0xee, 0x88, + 0x01, 0x4d, 0x60, 0x3f, 0x43, 0x7e, 0x9e, 0x02, + 0xaa, 0x7d, 0xaf, 0xa3, 0xcd, 0xfb, 0xad, 0x94 }, + { 0xdd, 0xbf, 0xea, 0x75, 0xcc, 0x46, 0x78, 0x82, + 0xeb, 0x34, 0x83, 0xce, 0x5e, 0x2e, 0x75, 0x6a, + 0x4f, 0x47, 0x01, 0xb7, 0x6b, 0x44, 0x55, 0x19, + 0xe8, 0x9f, 0x22, 0xd6, 0x0f, 0xa8, 0x6e, 0x06 }, + { 0x0c, 0x31, 0x1f, 0x38, 0xc3, 0x5a, 0x4f, 0xb9, + 0x0d, 0x65, 0x1c, 0x28, 0x9d, 0x48, 0x68, 0x56, + 0xcd, 0x14, 0x13, 0xdf, 0x9b, 0x06, 0x77, 0xf5, + 0x3e, 0xce, 0x2c, 0xd9, 0xe4, 0x77, 0xc6, 0x0a }, + { 0x46, 0xa7, 0x3a, 0x8d, 0xd3, 0xe7, 0x0f, 0x59, + 0xd3, 0x94, 0x2c, 0x01, 0xdf, 0x59, 0x9d, 0xef, + 0x78, 0x3c, 0x9d, 0xa8, 0x2f, 0xd8, 0x32, 0x22, + 0xcd, 0x66, 0x2b, 0x53, 0xdc, 0xe7, 0xdb, 0xdf }, + { 0xad, 0x03, 0x8f, 0xf9, 0xb1, 0x4d, 0xe8, 0x4a, + 0x80, 0x1e, 0x4e, 0x62, 0x1c, 0xe5, 0xdf, 0x02, + 0x9d, 0xd9, 0x35, 0x20, 0xd0, 0xc2, 0xfa, 0x38, + 0xbf, 0xf1, 0x76, 0xa8, 0xb1, 0xd1, 0x69, 0x8c }, + { 0xab, 0x70, 0xc5, 0xdf, 0xbd, 0x1e, 0xa8, 0x17, + 0xfe, 0xd0, 0xcd, 0x06, 0x72, 0x93, 0xab, 0xf3, + 0x19, 0xe5, 0xd7, 0x90, 0x1c, 0x21, 0x41, 0xd5, + 0xd9, 0x9b, 0x23, 0xf0, 0x3a, 0x38, 0xe7, 0x48 }, + { 0x1f, 0xff, 0xda, 0x67, 0x93, 0x2b, 0x73, 0xc8, + 0xec, 0xaf, 0x00, 0x9a, 0x34, 0x91, 0xa0, 0x26, + 0x95, 0x3b, 0xab, 0xfe, 0x1f, 0x66, 0x3b, 0x06, + 0x97, 0xc3, 0xc4, 0xae, 0x8b, 0x2e, 0x7d, 0xcb }, + { 0xb0, 0xd2, 0xcc, 0x19, 0x47, 0x2d, 0xd5, 0x7f, + 0x2b, 0x17, 0xef, 0xc0, 0x3c, 0x8d, 0x58, 0xc2, + 0x28, 0x3d, 0xbb, 0x19, 0xda, 0x57, 0x2f, 0x77, + 0x55, 0x85, 0x5a, 0xa9, 0x79, 0x43, 0x17, 0xa0 }, + { 0xa0, 0xd1, 0x9a, 0x6e, 0xe3, 0x39, 0x79, 0xc3, + 0x25, 0x51, 0x0e, 0x27, 0x66, 0x22, 0xdf, 0x41, + 0xf7, 0x15, 0x83, 0xd0, 0x75, 0x01, 0xb8, 0x70, + 0x71, 0x12, 0x9a, 0x0a, 0xd9, 0x47, 0x32, 0xa5 }, + { 0x72, 0x46, 0x42, 0xa7, 0x03, 0x2d, 0x10, 0x62, + 0xb8, 0x9e, 0x52, 0xbe, 0xa3, 0x4b, 0x75, 0xdf, + 0x7d, 0x8f, 0xe7, 0x72, 0xd9, 0xfe, 0x3c, 0x93, + 0xdd, 0xf3, 0xc4, 0x54, 0x5a, 0xb5, 0xa9, 0x9b }, + { 0xad, 0xe5, 0xea, 0xa7, 0xe6, 0x1f, 0x67, 0x2d, + 0x58, 0x7e, 0xa0, 0x3d, 0xae, 0x7d, 0x7b, 0x55, + 0x22, 0x9c, 0x01, 0xd0, 0x6b, 0xc0, 0xa5, 0x70, + 0x14, 0x36, 0xcb, 0xd1, 0x83, 0x66, 0xa6, 0x26 }, + { 0x01, 0x3b, 0x31, 0xeb, 0xd2, 0x28, 0xfc, 0xdd, + 0xa5, 0x1f, 0xab, 0xb0, 0x3b, 0xb0, 0x2d, 0x60, + 0xac, 0x20, 0xca, 0x21, 0x5a, 0xaf, 0xa8, 0x3b, + 0xdd, 0x85, 0x5e, 0x37, 0x55, 0xa3, 0x5f, 0x0b }, + { 0x33, 0x2e, 0xd4, 0x0b, 0xb1, 0x0d, 0xde, 0x3c, + 0x95, 0x4a, 0x75, 0xd7, 0xb8, 0x99, 0x9d, 0x4b, + 0x26, 0xa1, 0xc0, 0x63, 0xc1, 0xdc, 0x6e, 0x32, + 0xc1, 0xd9, 0x1b, 0xab, 0x7b, 0xbb, 0x7d, 0x16 }, + { 0xc7, 0xa1, 0x97, 0xb3, 0xa0, 0x5b, 0x56, 0x6b, + 0xcc, 0x9f, 0xac, 0xd2, 0x0e, 0x44, 0x1d, 0x6f, + 0x6c, 0x28, 0x60, 0xac, 0x96, 0x51, 0xcd, 0x51, + 0xd6, 0xb9, 0xd2, 0xcd, 0xee, 0xea, 0x03, 0x90 }, + { 0xbd, 0x9c, 0xf6, 0x4e, 0xa8, 0x95, 0x3c, 0x03, + 0x71, 0x08, 0xe6, 0xf6, 0x54, 0x91, 0x4f, 0x39, + 0x58, 0xb6, 0x8e, 0x29, 0xc1, 0x67, 0x00, 0xdc, + 0x18, 0x4d, 0x94, 0xa2, 0x17, 0x08, 0xff, 0x60 }, + { 0x88, 0x35, 0xb0, 0xac, 0x02, 0x11, 0x51, 0xdf, + 0x71, 0x64, 0x74, 0xce, 0x27, 0xce, 0x4d, 0x3c, + 0x15, 0xf0, 0xb2, 0xda, 0xb4, 0x80, 0x03, 0xcf, + 0x3f, 0x3e, 0xfd, 0x09, 0x45, 0x10, 0x6b, 0x9a }, + { 0x3b, 0xfe, 0xfa, 0x33, 0x01, 0xaa, 0x55, 0xc0, + 0x80, 0x19, 0x0c, 0xff, 0xda, 0x8e, 0xae, 0x51, + 0xd9, 0xaf, 0x48, 0x8b, 0x4c, 0x1f, 0x24, 0xc3, + 0xd9, 0xa7, 0x52, 0x42, 0xfd, 0x8e, 0xa0, 0x1d }, + { 0x08, 0x28, 0x4d, 0x14, 0x99, 0x3c, 0xd4, 0x7d, + 0x53, 0xeb, 0xae, 0xcf, 0x0d, 0xf0, 0x47, 0x8c, + 0xc1, 0x82, 0xc8, 0x9c, 0x00, 0xe1, 0x85, 0x9c, + 0x84, 0x85, 0x16, 0x86, 0xdd, 0xf2, 0xc1, 0xb7 }, + { 0x1e, 0xd7, 0xef, 0x9f, 0x04, 0xc2, 0xac, 0x8d, + 0xb6, 0xa8, 0x64, 0xdb, 0x13, 0x10, 0x87, 0xf2, + 0x70, 0x65, 0x09, 0x8e, 0x69, 0xc3, 0xfe, 0x78, + 0x71, 0x8d, 0x9b, 0x94, 0x7f, 0x4a, 0x39, 0xd0 }, + { 0xc1, 0x61, 0xf2, 0xdc, 0xd5, 0x7e, 0x9c, 0x14, + 0x39, 0xb3, 0x1a, 0x9d, 0xd4, 0x3d, 0x8f, 0x3d, + 0x7d, 0xd8, 0xf0, 0xeb, 0x7c, 0xfa, 0xc6, 0xfb, + 0x25, 0xa0, 0xf2, 0x8e, 0x30, 0x6f, 0x06, 0x61 }, + { 0xc0, 0x19, 0x69, 0xad, 0x34, 0xc5, 0x2c, 0xaf, + 0x3d, 0xc4, 0xd8, 0x0d, 0x19, 0x73, 0x5c, 0x29, + 0x73, 0x1a, 0xc6, 0xe7, 0xa9, 0x20, 0x85, 0xab, + 0x92, 0x50, 0xc4, 0x8d, 0xea, 0x48, 0xa3, 0xfc }, + { 0x17, 0x20, 0xb3, 0x65, 0x56, 0x19, 0xd2, 0xa5, + 0x2b, 0x35, 0x21, 0xae, 0x0e, 0x49, 0xe3, 0x45, + 0xcb, 0x33, 0x89, 0xeb, 0xd6, 0x20, 0x8a, 0xca, + 0xf9, 0xf1, 0x3f, 0xda, 0xcc, 0xa8, 0xbe, 0x49 }, + { 0x75, 0x62, 0x88, 0x36, 0x1c, 0x83, 0xe2, 0x4c, + 0x61, 0x7c, 0xf9, 0x5c, 0x90, 0x5b, 0x22, 0xd0, + 0x17, 0xcd, 0xc8, 0x6f, 0x0b, 0xf1, 0xd6, 0x58, + 0xf4, 0x75, 0x6c, 0x73, 0x79, 0x87, 0x3b, 0x7f }, + { 0xe7, 0xd0, 0xed, 0xa3, 0x45, 0x26, 0x93, 0xb7, + 0x52, 0xab, 0xcd, 0xa1, 0xb5, 0x5e, 0x27, 0x6f, + 0x82, 0x69, 0x8f, 0x5f, 0x16, 0x05, 0x40, 0x3e, + 0xff, 0x83, 0x0b, 0xea, 0x00, 0x71, 0xa3, 0x94 }, + { 0x2c, 0x82, 0xec, 0xaa, 0x6b, 0x84, 0x80, 0x3e, + 0x04, 0x4a, 0xf6, 0x31, 0x18, 0xaf, 0xe5, 0x44, + 0x68, 0x7c, 0xb6, 0xe6, 0xc7, 0xdf, 0x49, 0xed, + 0x76, 0x2d, 0xfd, 0x7c, 0x86, 0x93, 0xa1, 0xbc }, + { 0x61, 0x36, 0xcb, 0xf4, 0xb4, 0x41, 0x05, 0x6f, + 0xa1, 0xe2, 0x72, 0x24, 0x98, 0x12, 0x5d, 0x6d, + 0xed, 0x45, 0xe1, 0x7b, 0x52, 0x14, 0x39, 0x59, + 0xc7, 0xf4, 0xd4, 0xe3, 0x95, 0x21, 0x8a, 0xc2 }, + { 0x72, 0x1d, 0x32, 0x45, 0xaa, 0xfe, 0xf2, 0x7f, + 0x6a, 0x62, 0x4f, 0x47, 0x95, 0x4b, 0x6c, 0x25, + 0x50, 0x79, 0x52, 0x6f, 0xfa, 0x25, 0xe9, 0xff, + 0x77, 0xe5, 0xdc, 0xff, 0x47, 0x3b, 0x15, 0x97 }, + { 0x9d, 0xd2, 0xfb, 0xd8, 0xce, 0xf1, 0x6c, 0x35, + 0x3c, 0x0a, 0xc2, 0x11, 0x91, 0xd5, 0x09, 0xeb, + 0x28, 0xdd, 0x9e, 0x3e, 0x0d, 0x8c, 0xea, 0x5d, + 0x26, 0xca, 0x83, 0x93, 0x93, 0x85, 0x1c, 0x3a }, + { 0xb2, 0x39, 0x4c, 0xea, 0xcd, 0xeb, 0xf2, 0x1b, + 0xf9, 0xdf, 0x2c, 0xed, 0x98, 0xe5, 0x8f, 0x1c, + 0x3a, 0x4b, 0xbb, 0xff, 0x66, 0x0d, 0xd9, 0x00, + 0xf6, 0x22, 0x02, 0xd6, 0x78, 0x5c, 0xc4, 0x6e }, + { 0x57, 0x08, 0x9f, 0x22, 0x27, 0x49, 0xad, 0x78, + 0x71, 0x76, 0x5f, 0x06, 0x2b, 0x11, 0x4f, 0x43, + 0xba, 0x20, 0xec, 0x56, 0x42, 0x2a, 0x8b, 0x1e, + 0x3f, 0x87, 0x19, 0x2c, 0x0e, 0xa7, 0x18, 0xc6 }, + { 0xe4, 0x9a, 0x94, 0x59, 0x96, 0x1c, 0xd3, 0x3c, + 0xdf, 0x4a, 0xae, 0x1b, 0x10, 0x78, 0xa5, 0xde, + 0xa7, 0xc0, 0x40, 0xe0, 0xfe, 0xa3, 0x40, 0xc9, + 0x3a, 0x72, 0x48, 0x72, 0xfc, 0x4a, 0xf8, 0x06 }, + { 0xed, 0xe6, 0x7f, 0x72, 0x0e, 0xff, 0xd2, 0xca, + 0x9c, 0x88, 0x99, 0x41, 0x52, 0xd0, 0x20, 0x1d, + 0xee, 0x6b, 0x0a, 0x2d, 0x2c, 0x07, 0x7a, 0xca, + 0x6d, 0xae, 0x29, 0xf7, 0x3f, 0x8b, 0x63, 0x09 }, + { 0xe0, 0xf4, 0x34, 0xbf, 0x22, 0xe3, 0x08, 0x80, + 0x39, 0xc2, 0x1f, 0x71, 0x9f, 0xfc, 0x67, 0xf0, + 0xf2, 0xcb, 0x5e, 0x98, 0xa7, 0xa0, 0x19, 0x4c, + 0x76, 0xe9, 0x6b, 0xf4, 0xe8, 0xe1, 0x7e, 0x61 }, + { 0x27, 0x7c, 0x04, 0xe2, 0x85, 0x34, 0x84, 0xa4, + 0xeb, 0xa9, 0x10, 0xad, 0x33, 0x6d, 0x01, 0xb4, + 0x77, 0xb6, 0x7c, 0xc2, 0x00, 0xc5, 0x9f, 0x3c, + 0x8d, 0x77, 0xee, 0xf8, 0x49, 0x4f, 0x29, 0xcd }, + { 0x15, 0x6d, 0x57, 0x47, 0xd0, 0xc9, 0x9c, 0x7f, + 0x27, 0x09, 0x7d, 0x7b, 0x7e, 0x00, 0x2b, 0x2e, + 0x18, 0x5c, 0xb7, 0x2d, 0x8d, 0xd7, 0xeb, 0x42, + 0x4a, 0x03, 0x21, 0x52, 0x81, 0x61, 0x21, 0x9f }, + { 0x20, 0xdd, 0xd1, 0xed, 0x9b, 0x1c, 0xa8, 0x03, + 0x94, 0x6d, 0x64, 0xa8, 0x3a, 0xe4, 0x65, 0x9d, + 0xa6, 0x7f, 0xba, 0x7a, 0x1a, 0x3e, 0xdd, 0xb1, + 0xe1, 0x03, 0xc0, 0xf5, 0xe0, 0x3e, 0x3a, 0x2c }, + { 0xf0, 0xaf, 0x60, 0x4d, 0x3d, 0xab, 0xbf, 0x9a, + 0x0f, 0x2a, 0x7d, 0x3d, 0xda, 0x6b, 0xd3, 0x8b, + 0xba, 0x72, 0xc6, 0xd0, 0x9b, 0xe4, 0x94, 0xfc, + 0xef, 0x71, 0x3f, 0xf1, 0x01, 0x89, 0xb6, 0xe6 }, + { 0x98, 0x02, 0xbb, 0x87, 0xde, 0xf4, 0xcc, 0x10, + 0xc4, 0xa5, 0xfd, 0x49, 0xaa, 0x58, 0xdf, 0xe2, + 0xf3, 0xfd, 0xdb, 0x46, 0xb4, 0x70, 0x88, 0x14, + 0xea, 0xd8, 0x1d, 0x23, 0xba, 0x95, 0x13, 0x9b }, + { 0x4f, 0x8c, 0xe1, 0xe5, 0x1d, 0x2f, 0xe7, 0xf2, + 0x40, 0x43, 0xa9, 0x04, 0xd8, 0x98, 0xeb, 0xfc, + 0x91, 0x97, 0x54, 0x18, 0x75, 0x34, 0x13, 0xaa, + 0x09, 0x9b, 0x79, 0x5e, 0xcb, 0x35, 0xce, 0xdb }, + { 0xbd, 0xdc, 0x65, 0x14, 0xd7, 0xee, 0x6a, 0xce, + 0x0a, 0x4a, 0xc1, 0xd0, 0xe0, 0x68, 0x11, 0x22, + 0x88, 0xcb, 0xcf, 0x56, 0x04, 0x54, 0x64, 0x27, + 0x05, 0x63, 0x01, 0x77, 0xcb, 0xa6, 0x08, 0xbd }, + { 0xd6, 0x35, 0x99, 0x4f, 0x62, 0x91, 0x51, 0x7b, + 0x02, 0x81, 0xff, 0xdd, 0x49, 0x6a, 0xfa, 0x86, + 0x27, 0x12, 0xe5, 0xb3, 0xc4, 0xe5, 0x2e, 0x4c, + 0xd5, 0xfd, 0xae, 0x8c, 0x0e, 0x72, 0xfb, 0x08 }, + { 0x87, 0x8d, 0x9c, 0xa6, 0x00, 0xcf, 0x87, 0xe7, + 0x69, 0xcc, 0x30, 0x5c, 0x1b, 0x35, 0x25, 0x51, + 0x86, 0x61, 0x5a, 0x73, 0xa0, 0xda, 0x61, 0x3b, + 0x5f, 0x1c, 0x98, 0xdb, 0xf8, 0x12, 0x83, 0xea }, + { 0xa6, 0x4e, 0xbe, 0x5d, 0xc1, 0x85, 0xde, 0x9f, + 0xdd, 0xe7, 0x60, 0x7b, 0x69, 0x98, 0x70, 0x2e, + 0xb2, 0x34, 0x56, 0x18, 0x49, 0x57, 0x30, 0x7d, + 0x2f, 0xa7, 0x2e, 0x87, 0xa4, 0x77, 0x02, 0xd6 }, + { 0xce, 0x50, 0xea, 0xb7, 0xb5, 0xeb, 0x52, 0xbd, + 0xc9, 0xad, 0x8e, 0x5a, 0x48, 0x0a, 0xb7, 0x80, + 0xca, 0x93, 0x20, 0xe4, 0x43, 0x60, 0xb1, 0xfe, + 0x37, 0xe0, 0x3f, 0x2f, 0x7a, 0xd7, 0xde, 0x01 }, + { 0xee, 0xdd, 0xb7, 0xc0, 0xdb, 0x6e, 0x30, 0xab, + 0xe6, 0x6d, 0x79, 0xe3, 0x27, 0x51, 0x1e, 0x61, + 0xfc, 0xeb, 0xbc, 0x29, 0xf1, 0x59, 0xb4, 0x0a, + 0x86, 0xb0, 0x46, 0xec, 0xf0, 0x51, 0x38, 0x23 }, + { 0x78, 0x7f, 0xc9, 0x34, 0x40, 0xc1, 0xec, 0x96, + 0xb5, 0xad, 0x01, 0xc1, 0x6c, 0xf7, 0x79, 0x16, + 0xa1, 0x40, 0x5f, 0x94, 0x26, 0x35, 0x6e, 0xc9, + 0x21, 0xd8, 0xdf, 0xf3, 0xea, 0x63, 0xb7, 0xe0 }, + { 0x7f, 0x0d, 0x5e, 0xab, 0x47, 0xee, 0xfd, 0xa6, + 0x96, 0xc0, 0xbf, 0x0f, 0xbf, 0x86, 0xab, 0x21, + 0x6f, 0xce, 0x46, 0x1e, 0x93, 0x03, 0xab, 0xa6, + 0xac, 0x37, 0x41, 0x20, 0xe8, 0x90, 0xe8, 0xdf }, + { 0xb6, 0x80, 0x04, 0xb4, 0x2f, 0x14, 0xad, 0x02, + 0x9f, 0x4c, 0x2e, 0x03, 0xb1, 0xd5, 0xeb, 0x76, + 0xd5, 0x71, 0x60, 0xe2, 0x64, 0x76, 0xd2, 0x11, + 0x31, 0xbe, 0xf2, 0x0a, 0xda, 0x7d, 0x27, 0xf4 }, + { 0xb0, 0xc4, 0xeb, 0x18, 0xae, 0x25, 0x0b, 0x51, + 0xa4, 0x13, 0x82, 0xea, 0xd9, 0x2d, 0x0d, 0xc7, + 0x45, 0x5f, 0x93, 0x79, 0xfc, 0x98, 0x84, 0x42, + 0x8e, 0x47, 0x70, 0x60, 0x8d, 0xb0, 0xfa, 0xec }, + { 0xf9, 0x2b, 0x7a, 0x87, 0x0c, 0x05, 0x9f, 0x4d, + 0x46, 0x46, 0x4c, 0x82, 0x4e, 0xc9, 0x63, 0x55, + 0x14, 0x0b, 0xdc, 0xe6, 0x81, 0x32, 0x2c, 0xc3, + 0xa9, 0x92, 0xff, 0x10, 0x3e, 0x3f, 0xea, 0x52 }, + { 0x53, 0x64, 0x31, 0x26, 0x14, 0x81, 0x33, 0x98, + 0xcc, 0x52, 0x5d, 0x4c, 0x4e, 0x14, 0x6e, 0xde, + 0xb3, 0x71, 0x26, 0x5f, 0xba, 0x19, 0x13, 0x3a, + 0x2c, 0x3d, 0x21, 0x59, 0x29, 0x8a, 0x17, 0x42 }, + { 0xf6, 0x62, 0x0e, 0x68, 0xd3, 0x7f, 0xb2, 0xaf, + 0x50, 0x00, 0xfc, 0x28, 0xe2, 0x3b, 0x83, 0x22, + 0x97, 0xec, 0xd8, 0xbc, 0xe9, 0x9e, 0x8b, 0xe4, + 0xd0, 0x4e, 0x85, 0x30, 0x9e, 0x3d, 0x33, 0x74 }, + { 0x53, 0x16, 0xa2, 0x79, 0x69, 0xd7, 0xfe, 0x04, + 0xff, 0x27, 0xb2, 0x83, 0x96, 0x1b, 0xff, 0xc3, + 0xbf, 0x5d, 0xfb, 0x32, 0xfb, 0x6a, 0x89, 0xd1, + 0x01, 0xc6, 0xc3, 0xb1, 0x93, 0x7c, 0x28, 0x71 }, + { 0x81, 0xd1, 0x66, 0x4f, 0xdf, 0x3c, 0xb3, 0x3c, + 0x24, 0xee, 0xba, 0xc0, 0xbd, 0x64, 0x24, 0x4b, + 0x77, 0xc4, 0xab, 0xea, 0x90, 0xbb, 0xe8, 0xb5, + 0xee, 0x0b, 0x2a, 0xaf, 0xcf, 0x2d, 0x6a, 0x53 }, + { 0x34, 0x57, 0x82, 0xf2, 0x95, 0xb0, 0x88, 0x03, + 0x52, 0xe9, 0x24, 0xa0, 0x46, 0x7b, 0x5f, 0xbc, + 0x3e, 0x8f, 0x3b, 0xfb, 0xc3, 0xc7, 0xe4, 0x8b, + 0x67, 0x09, 0x1f, 0xb5, 0xe8, 0x0a, 0x94, 0x42 }, + { 0x79, 0x41, 0x11, 0xea, 0x6c, 0xd6, 0x5e, 0x31, + 0x1f, 0x74, 0xee, 0x41, 0xd4, 0x76, 0xcb, 0x63, + 0x2c, 0xe1, 0xe4, 0xb0, 0x51, 0xdc, 0x1d, 0x9e, + 0x9d, 0x06, 0x1a, 0x19, 0xe1, 0xd0, 0xbb, 0x49 }, + { 0x2a, 0x85, 0xda, 0xf6, 0x13, 0x88, 0x16, 0xb9, + 0x9b, 0xf8, 0xd0, 0x8b, 0xa2, 0x11, 0x4b, 0x7a, + 0xb0, 0x79, 0x75, 0xa7, 0x84, 0x20, 0xc1, 0xa3, + 0xb0, 0x6a, 0x77, 0x7c, 0x22, 0xdd, 0x8b, 0xcb }, + { 0x89, 0xb0, 0xd5, 0xf2, 0x89, 0xec, 0x16, 0x40, + 0x1a, 0x06, 0x9a, 0x96, 0x0d, 0x0b, 0x09, 0x3e, + 0x62, 0x5d, 0xa3, 0xcf, 0x41, 0xee, 0x29, 0xb5, + 0x9b, 0x93, 0x0c, 0x58, 0x20, 0x14, 0x54, 0x55 }, + { 0xd0, 0xfd, 0xcb, 0x54, 0x39, 0x43, 0xfc, 0x27, + 0xd2, 0x08, 0x64, 0xf5, 0x21, 0x81, 0x47, 0x1b, + 0x94, 0x2c, 0xc7, 0x7c, 0xa6, 0x75, 0xbc, 0xb3, + 0x0d, 0xf3, 0x1d, 0x35, 0x8e, 0xf7, 0xb1, 0xeb }, + { 0xb1, 0x7e, 0xa8, 0xd7, 0x70, 0x63, 0xc7, 0x09, + 0xd4, 0xdc, 0x6b, 0x87, 0x94, 0x13, 0xc3, 0x43, + 0xe3, 0x79, 0x0e, 0x9e, 0x62, 0xca, 0x85, 0xb7, + 0x90, 0x0b, 0x08, 0x6f, 0x6b, 0x75, 0xc6, 0x72 }, + { 0xe7, 0x1a, 0x3e, 0x2c, 0x27, 0x4d, 0xb8, 0x42, + 0xd9, 0x21, 0x14, 0xf2, 0x17, 0xe2, 0xc0, 0xea, + 0xc8, 0xb4, 0x50, 0x93, 0xfd, 0xfd, 0x9d, 0xf4, + 0xca, 0x71, 0x62, 0x39, 0x48, 0x62, 0xd5, 0x01 }, + { 0xc0, 0x47, 0x67, 0x59, 0xab, 0x7a, 0xa3, 0x33, + 0x23, 0x4f, 0x6b, 0x44, 0xf5, 0xfd, 0x85, 0x83, + 0x90, 0xec, 0x23, 0x69, 0x4c, 0x62, 0x2c, 0xb9, + 0x86, 0xe7, 0x69, 0xc7, 0x8e, 0xdd, 0x73, 0x3e }, + { 0x9a, 0xb8, 0xea, 0xbb, 0x14, 0x16, 0x43, 0x4d, + 0x85, 0x39, 0x13, 0x41, 0xd5, 0x69, 0x93, 0xc5, + 0x54, 0x58, 0x16, 0x7d, 0x44, 0x18, 0xb1, 0x9a, + 0x0f, 0x2a, 0xd8, 0xb7, 0x9a, 0x83, 0xa7, 0x5b }, + { 0x79, 0x92, 0xd0, 0xbb, 0xb1, 0x5e, 0x23, 0x82, + 0x6f, 0x44, 0x3e, 0x00, 0x50, 0x5d, 0x68, 0xd3, + 0xed, 0x73, 0x72, 0x99, 0x5a, 0x5c, 0x3e, 0x49, + 0x86, 0x54, 0x10, 0x2f, 0xbc, 0xd0, 0x96, 0x4e }, + { 0xc0, 0x21, 0xb3, 0x00, 0x85, 0x15, 0x14, 0x35, + 0xdf, 0x33, 0xb0, 0x07, 0xcc, 0xec, 0xc6, 0x9d, + 0xf1, 0x26, 0x9f, 0x39, 0xba, 0x25, 0x09, 0x2b, + 0xed, 0x59, 0xd9, 0x32, 0xac, 0x0f, 0xdc, 0x28 }, + { 0x91, 0xa2, 0x5e, 0xc0, 0xec, 0x0d, 0x9a, 0x56, + 0x7f, 0x89, 0xc4, 0xbf, 0xe1, 0xa6, 0x5a, 0x0e, + 0x43, 0x2d, 0x07, 0x06, 0x4b, 0x41, 0x90, 0xe2, + 0x7d, 0xfb, 0x81, 0x90, 0x1f, 0xd3, 0x13, 0x9b }, + { 0x59, 0x50, 0xd3, 0x9a, 0x23, 0xe1, 0x54, 0x5f, + 0x30, 0x12, 0x70, 0xaa, 0x1a, 0x12, 0xf2, 0xe6, + 0xc4, 0x53, 0x77, 0x6e, 0x4d, 0x63, 0x55, 0xde, + 0x42, 0x5c, 0xc1, 0x53, 0xf9, 0x81, 0x88, 0x67 }, + { 0xd7, 0x9f, 0x14, 0x72, 0x0c, 0x61, 0x0a, 0xf1, + 0x79, 0xa3, 0x76, 0x5d, 0x4b, 0x7c, 0x09, 0x68, + 0xf9, 0x77, 0x96, 0x2d, 0xbf, 0x65, 0x5b, 0x52, + 0x12, 0x72, 0xb6, 0xf1, 0xe1, 0x94, 0x48, 0x8e }, + { 0xe9, 0x53, 0x1b, 0xfc, 0x8b, 0x02, 0x99, 0x5a, + 0xea, 0xa7, 0x5b, 0xa2, 0x70, 0x31, 0xfa, 0xdb, + 0xcb, 0xf4, 0xa0, 0xda, 0xb8, 0x96, 0x1d, 0x92, + 0x96, 0xcd, 0x7e, 0x84, 0xd2, 0x5d, 0x60, 0x06 }, + { 0x34, 0xe9, 0xc2, 0x6a, 0x01, 0xd7, 0xf1, 0x61, + 0x81, 0xb4, 0x54, 0xa9, 0xd1, 0x62, 0x3c, 0x23, + 0x3c, 0xb9, 0x9d, 0x31, 0xc6, 0x94, 0x65, 0x6e, + 0x94, 0x13, 0xac, 0xa3, 0xe9, 0x18, 0x69, 0x2f }, + { 0xd9, 0xd7, 0x42, 0x2f, 0x43, 0x7b, 0xd4, 0x39, + 0xdd, 0xd4, 0xd8, 0x83, 0xda, 0xe2, 0xa0, 0x83, + 0x50, 0x17, 0x34, 0x14, 0xbe, 0x78, 0x15, 0x51, + 0x33, 0xff, 0xf1, 0x96, 0x4c, 0x3d, 0x79, 0x72 }, + { 0x4a, 0xee, 0x0c, 0x7a, 0xaf, 0x07, 0x54, 0x14, + 0xff, 0x17, 0x93, 0xea, 0xd7, 0xea, 0xca, 0x60, + 0x17, 0x75, 0xc6, 0x15, 0xdb, 0xd6, 0x0b, 0x64, + 0x0b, 0x0a, 0x9f, 0x0c, 0xe5, 0x05, 0xd4, 0x35 }, + { 0x6b, 0xfd, 0xd1, 0x54, 0x59, 0xc8, 0x3b, 0x99, + 0xf0, 0x96, 0xbf, 0xb4, 0x9e, 0xe8, 0x7b, 0x06, + 0x3d, 0x69, 0xc1, 0x97, 0x4c, 0x69, 0x28, 0xac, + 0xfc, 0xfb, 0x40, 0x99, 0xf8, 0xc4, 0xef, 0x67 }, + { 0x9f, 0xd1, 0xc4, 0x08, 0xfd, 0x75, 0xc3, 0x36, + 0x19, 0x3a, 0x2a, 0x14, 0xd9, 0x4f, 0x6a, 0xf5, + 0xad, 0xf0, 0x50, 0xb8, 0x03, 0x87, 0xb4, 0xb0, + 0x10, 0xfb, 0x29, 0xf4, 0xcc, 0x72, 0x70, 0x7c }, + { 0x13, 0xc8, 0x84, 0x80, 0xa5, 0xd0, 0x0d, 0x6c, + 0x8c, 0x7a, 0xd2, 0x11, 0x0d, 0x76, 0xa8, 0x2d, + 0x9b, 0x70, 0xf4, 0xfa, 0x66, 0x96, 0xd4, 0xe5, + 0xdd, 0x42, 0xa0, 0x66, 0xdc, 0xaf, 0x99, 0x20 }, + { 0x82, 0x0e, 0x72, 0x5e, 0xe2, 0x5f, 0xe8, 0xfd, + 0x3a, 0x8d, 0x5a, 0xbe, 0x4c, 0x46, 0xc3, 0xba, + 0x88, 0x9d, 0xe6, 0xfa, 0x91, 0x91, 0xaa, 0x22, + 0xba, 0x67, 0xd5, 0x70, 0x54, 0x21, 0x54, 0x2b }, + { 0x32, 0xd9, 0x3a, 0x0e, 0xb0, 0x2f, 0x42, 0xfb, + 0xbc, 0xaf, 0x2b, 0xad, 0x00, 0x85, 0xb2, 0x82, + 0xe4, 0x60, 0x46, 0xa4, 0xdf, 0x7a, 0xd1, 0x06, + 0x57, 0xc9, 0xd6, 0x47, 0x63, 0x75, 0xb9, 0x3e }, + { 0xad, 0xc5, 0x18, 0x79, 0x05, 0xb1, 0x66, 0x9c, + 0xd8, 0xec, 0x9c, 0x72, 0x1e, 0x19, 0x53, 0x78, + 0x6b, 0x9d, 0x89, 0xa9, 0xba, 0xe3, 0x07, 0x80, + 0xf1, 0xe1, 0xea, 0xb2, 0x4a, 0x00, 0x52, 0x3c }, + { 0xe9, 0x07, 0x56, 0xff, 0x7f, 0x9a, 0xd8, 0x10, + 0xb2, 0x39, 0xa1, 0x0c, 0xed, 0x2c, 0xf9, 0xb2, + 0x28, 0x43, 0x54, 0xc1, 0xf8, 0xc7, 0xe0, 0xac, + 0xcc, 0x24, 0x61, 0xdc, 0x79, 0x6d, 0x6e, 0x89 }, + { 0x12, 0x51, 0xf7, 0x6e, 0x56, 0x97, 0x84, 0x81, + 0x87, 0x53, 0x59, 0x80, 0x1d, 0xb5, 0x89, 0xa0, + 0xb2, 0x2f, 0x86, 0xd8, 0xd6, 0x34, 0xdc, 0x04, + 0x50, 0x6f, 0x32, 0x2e, 0xd7, 0x8f, 0x17, 0xe8 }, + { 0x3a, 0xfa, 0x89, 0x9f, 0xd9, 0x80, 0xe7, 0x3e, + 0xcb, 0x7f, 0x4d, 0x8b, 0x8f, 0x29, 0x1d, 0xc9, + 0xaf, 0x79, 0x6b, 0xc6, 0x5d, 0x27, 0xf9, 0x74, + 0xc6, 0xf1, 0x93, 0xc9, 0x19, 0x1a, 0x09, 0xfd }, + { 0xaa, 0x30, 0x5b, 0xe2, 0x6e, 0x5d, 0xed, 0xdc, + 0x3c, 0x10, 0x10, 0xcb, 0xc2, 0x13, 0xf9, 0x5f, + 0x05, 0x1c, 0x78, 0x5c, 0x5b, 0x43, 0x1e, 0x6a, + 0x7c, 0xd0, 0x48, 0xf1, 0x61, 0x78, 0x75, 0x28 }, + { 0x8e, 0xa1, 0x88, 0x4f, 0xf3, 0x2e, 0x9d, 0x10, + 0xf0, 0x39, 0xb4, 0x07, 0xd0, 0xd4, 0x4e, 0x7e, + 0x67, 0x0a, 0xbd, 0x88, 0x4a, 0xee, 0xe0, 0xfb, + 0x75, 0x7a, 0xe9, 0x4e, 0xaa, 0x97, 0x37, 0x3d }, + { 0xd4, 0x82, 0xb2, 0x15, 0x5d, 0x4d, 0xec, 0x6b, + 0x47, 0x36, 0xa1, 0xf1, 0x61, 0x7b, 0x53, 0xaa, + 0xa3, 0x73, 0x10, 0x27, 0x7d, 0x3f, 0xef, 0x0c, + 0x37, 0xad, 0x41, 0x76, 0x8f, 0xc2, 0x35, 0xb4 }, + { 0x4d, 0x41, 0x39, 0x71, 0x38, 0x7e, 0x7a, 0x88, + 0x98, 0xa8, 0xdc, 0x2a, 0x27, 0x50, 0x07, 0x78, + 0x53, 0x9e, 0xa2, 0x14, 0xa2, 0xdf, 0xe9, 0xb3, + 0xd7, 0xe8, 0xeb, 0xdc, 0xe5, 0xcf, 0x3d, 0xb3 }, + { 0x69, 0x6e, 0x5d, 0x46, 0xe6, 0xc5, 0x7e, 0x87, + 0x96, 0xe4, 0x73, 0x5d, 0x08, 0x91, 0x6e, 0x0b, + 0x79, 0x29, 0xb3, 0xcf, 0x29, 0x8c, 0x29, 0x6d, + 0x22, 0xe9, 0xd3, 0x01, 0x96, 0x53, 0x37, 0x1c }, + { 0x1f, 0x56, 0x47, 0xc1, 0xd3, 0xb0, 0x88, 0x22, + 0x88, 0x85, 0x86, 0x5c, 0x89, 0x40, 0x90, 0x8b, + 0xf4, 0x0d, 0x1a, 0x82, 0x72, 0x82, 0x19, 0x73, + 0xb1, 0x60, 0x00, 0x8e, 0x7a, 0x3c, 0xe2, 0xeb }, + { 0xb6, 0xe7, 0x6c, 0x33, 0x0f, 0x02, 0x1a, 0x5b, + 0xda, 0x65, 0x87, 0x50, 0x10, 0xb0, 0xed, 0xf0, + 0x91, 0x26, 0xc0, 0xf5, 0x10, 0xea, 0x84, 0x90, + 0x48, 0x19, 0x20, 0x03, 0xae, 0xf4, 0xc6, 0x1c }, + { 0x3c, 0xd9, 0x52, 0xa0, 0xbe, 0xad, 0xa4, 0x1a, + 0xbb, 0x42, 0x4c, 0xe4, 0x7f, 0x94, 0xb4, 0x2b, + 0xe6, 0x4e, 0x1f, 0xfb, 0x0f, 0xd0, 0x78, 0x22, + 0x76, 0x80, 0x79, 0x46, 0xd0, 0xd0, 0xbc, 0x55 }, + { 0x98, 0xd9, 0x26, 0x77, 0x43, 0x9b, 0x41, 0xb7, + 0xbb, 0x51, 0x33, 0x12, 0xaf, 0xb9, 0x2b, 0xcc, + 0x8e, 0xe9, 0x68, 0xb2, 0xe3, 0xb2, 0x38, 0xce, + 0xcb, 0x9b, 0x0f, 0x34, 0xc9, 0xbb, 0x63, 0xd0 }, + { 0xec, 0xbc, 0xa2, 0xcf, 0x08, 0xae, 0x57, 0xd5, + 0x17, 0xad, 0x16, 0x15, 0x8a, 0x32, 0xbf, 0xa7, + 0xdc, 0x03, 0x82, 0xea, 0xed, 0xa1, 0x28, 0xe9, + 0x18, 0x86, 0x73, 0x4c, 0x24, 0xa0, 0xb2, 0x9d }, + { 0x94, 0x2c, 0xc7, 0xc0, 0xb5, 0x2e, 0x2b, 0x16, + 0xa4, 0xb8, 0x9f, 0xa4, 0xfc, 0x7e, 0x0b, 0xf6, + 0x09, 0xe2, 0x9a, 0x08, 0xc1, 0xa8, 0x54, 0x34, + 0x52, 0xb7, 0x7c, 0x7b, 0xfd, 0x11, 0xbb, 0x28 }, + { 0x8a, 0x06, 0x5d, 0x8b, 0x61, 0xa0, 0xdf, 0xfb, + 0x17, 0x0d, 0x56, 0x27, 0x73, 0x5a, 0x76, 0xb0, + 0xe9, 0x50, 0x60, 0x37, 0x80, 0x8c, 0xba, 0x16, + 0xc3, 0x45, 0x00, 0x7c, 0x9f, 0x79, 0xcf, 0x8f }, + { 0x1b, 0x9f, 0xa1, 0x97, 0x14, 0x65, 0x9c, 0x78, + 0xff, 0x41, 0x38, 0x71, 0x84, 0x92, 0x15, 0x36, + 0x10, 0x29, 0xac, 0x80, 0x2b, 0x1c, 0xbc, 0xd5, + 0x4e, 0x40, 0x8b, 0xd8, 0x72, 0x87, 0xf8, 0x1f }, + { 0x8d, 0xab, 0x07, 0x1b, 0xcd, 0x6c, 0x72, 0x92, + 0xa9, 0xef, 0x72, 0x7b, 0x4a, 0xe0, 0xd8, 0x67, + 0x13, 0x30, 0x1d, 0xa8, 0x61, 0x8d, 0x9a, 0x48, + 0xad, 0xce, 0x55, 0xf3, 0x03, 0xa8, 0x69, 0xa1 }, + { 0x82, 0x53, 0xe3, 0xe7, 0xc7, 0xb6, 0x84, 0xb9, + 0xcb, 0x2b, 0xeb, 0x01, 0x4c, 0xe3, 0x30, 0xff, + 0x3d, 0x99, 0xd1, 0x7a, 0xbb, 0xdb, 0xab, 0xe4, + 0xf4, 0xd6, 0x74, 0xde, 0xd5, 0x3f, 0xfc, 0x6b }, + { 0xf1, 0x95, 0xf3, 0x21, 0xe9, 0xe3, 0xd6, 0xbd, + 0x7d, 0x07, 0x45, 0x04, 0xdd, 0x2a, 0xb0, 0xe6, + 0x24, 0x1f, 0x92, 0xe7, 0x84, 0xb1, 0xaa, 0x27, + 0x1f, 0xf6, 0x48, 0xb1, 0xca, 0xb6, 0xd7, 0xf6 }, + { 0x27, 0xe4, 0xcc, 0x72, 0x09, 0x0f, 0x24, 0x12, + 0x66, 0x47, 0x6a, 0x7c, 0x09, 0x49, 0x5f, 0x2d, + 0xb1, 0x53, 0xd5, 0xbc, 0xbd, 0x76, 0x19, 0x03, + 0xef, 0x79, 0x27, 0x5e, 0xc5, 0x6b, 0x2e, 0xd8 }, + { 0x89, 0x9c, 0x24, 0x05, 0x78, 0x8e, 0x25, 0xb9, + 0x9a, 0x18, 0x46, 0x35, 0x5e, 0x64, 0x6d, 0x77, + 0xcf, 0x40, 0x00, 0x83, 0x41, 0x5f, 0x7d, 0xc5, + 0xaf, 0xe6, 0x9d, 0x6e, 0x17, 0xc0, 0x00, 0x23 }, + { 0xa5, 0x9b, 0x78, 0xc4, 0x90, 0x57, 0x44, 0x07, + 0x6b, 0xfe, 0xe8, 0x94, 0xde, 0x70, 0x7d, 0x4f, + 0x12, 0x0b, 0x5c, 0x68, 0x93, 0xea, 0x04, 0x00, + 0x29, 0x7d, 0x0b, 0xb8, 0x34, 0x72, 0x76, 0x32 }, + { 0x59, 0xdc, 0x78, 0xb1, 0x05, 0x64, 0x97, 0x07, + 0xa2, 0xbb, 0x44, 0x19, 0xc4, 0x8f, 0x00, 0x54, + 0x00, 0xd3, 0x97, 0x3d, 0xe3, 0x73, 0x66, 0x10, + 0x23, 0x04, 0x35, 0xb1, 0x04, 0x24, 0xb2, 0x4f }, + { 0xc0, 0x14, 0x9d, 0x1d, 0x7e, 0x7a, 0x63, 0x53, + 0xa6, 0xd9, 0x06, 0xef, 0xe7, 0x28, 0xf2, 0xf3, + 0x29, 0xfe, 0x14, 0xa4, 0x14, 0x9a, 0x3e, 0xa7, + 0x76, 0x09, 0xbc, 0x42, 0xb9, 0x75, 0xdd, 0xfa }, + { 0xa3, 0x2f, 0x24, 0x14, 0x74, 0xa6, 0xc1, 0x69, + 0x32, 0xe9, 0x24, 0x3b, 0xe0, 0xcf, 0x09, 0xbc, + 0xdc, 0x7e, 0x0c, 0xa0, 0xe7, 0xa6, 0xa1, 0xb9, + 0xb1, 0xa0, 0xf0, 0x1e, 0x41, 0x50, 0x23, 0x77 }, + { 0xb2, 0x39, 0xb2, 0xe4, 0xf8, 0x18, 0x41, 0x36, + 0x1c, 0x13, 0x39, 0xf6, 0x8e, 0x2c, 0x35, 0x9f, + 0x92, 0x9a, 0xf9, 0xad, 0x9f, 0x34, 0xe0, 0x1a, + 0xab, 0x46, 0x31, 0xad, 0x6d, 0x55, 0x00, 0xb0 }, + { 0x85, 0xfb, 0x41, 0x9c, 0x70, 0x02, 0xa3, 0xe0, + 0xb4, 0xb6, 0xea, 0x09, 0x3b, 0x4c, 0x1a, 0xc6, + 0x93, 0x66, 0x45, 0xb6, 0x5d, 0xac, 0x5a, 0xc1, + 0x5a, 0x85, 0x28, 0xb7, 0xb9, 0x4c, 0x17, 0x54 }, + { 0x96, 0x19, 0x72, 0x06, 0x25, 0xf1, 0x90, 0xb9, + 0x3a, 0x3f, 0xad, 0x18, 0x6a, 0xb3, 0x14, 0x18, + 0x96, 0x33, 0xc0, 0xd3, 0xa0, 0x1e, 0x6f, 0x9b, + 0xc8, 0xc4, 0xa8, 0xf8, 0x2f, 0x38, 0x3d, 0xbf }, + { 0x7d, 0x62, 0x0d, 0x90, 0xfe, 0x69, 0xfa, 0x46, + 0x9a, 0x65, 0x38, 0x38, 0x89, 0x70, 0xa1, 0xaa, + 0x09, 0xbb, 0x48, 0xa2, 0xd5, 0x9b, 0x34, 0x7b, + 0x97, 0xe8, 0xce, 0x71, 0xf4, 0x8c, 0x7f, 0x46 }, + { 0x29, 0x43, 0x83, 0x56, 0x85, 0x96, 0xfb, 0x37, + 0xc7, 0x5b, 0xba, 0xcd, 0x97, 0x9c, 0x5f, 0xf6, + 0xf2, 0x0a, 0x55, 0x6b, 0xf8, 0x87, 0x9c, 0xc7, + 0x29, 0x24, 0x85, 0x5d, 0xf9, 0xb8, 0x24, 0x0e }, + { 0x16, 0xb1, 0x8a, 0xb3, 0x14, 0x35, 0x9c, 0x2b, + 0x83, 0x3c, 0x1c, 0x69, 0x86, 0xd4, 0x8c, 0x55, + 0xa9, 0xfc, 0x97, 0xcd, 0xe9, 0xa3, 0xc1, 0xf1, + 0x0a, 0x31, 0x77, 0x14, 0x0f, 0x73, 0xf7, 0x38 }, + { 0x8c, 0xbb, 0xdd, 0x14, 0xbc, 0x33, 0xf0, 0x4c, + 0xf4, 0x58, 0x13, 0xe4, 0xa1, 0x53, 0xa2, 0x73, + 0xd3, 0x6a, 0xda, 0xd5, 0xce, 0x71, 0xf4, 0x99, + 0xee, 0xb8, 0x7f, 0xb8, 0xac, 0x63, 0xb7, 0x29 }, + { 0x69, 0xc9, 0xa4, 0x98, 0xdb, 0x17, 0x4e, 0xca, + 0xef, 0xcc, 0x5a, 0x3a, 0xc9, 0xfd, 0xed, 0xf0, + 0xf8, 0x13, 0xa5, 0xbe, 0xc7, 0x27, 0xf1, 0xe7, + 0x75, 0xba, 0xbd, 0xec, 0x77, 0x18, 0x81, 0x6e }, + { 0xb4, 0x62, 0xc3, 0xbe, 0x40, 0x44, 0x8f, 0x1d, + 0x4f, 0x80, 0x62, 0x62, 0x54, 0xe5, 0x35, 0xb0, + 0x8b, 0xc9, 0xcd, 0xcf, 0xf5, 0x99, 0xa7, 0x68, + 0x57, 0x8d, 0x4b, 0x28, 0x81, 0xa8, 0xe3, 0xf0 }, + { 0x55, 0x3e, 0x9d, 0x9c, 0x5f, 0x36, 0x0a, 0xc0, + 0xb7, 0x4a, 0x7d, 0x44, 0xe5, 0xa3, 0x91, 0xda, + 0xd4, 0xce, 0xd0, 0x3e, 0x0c, 0x24, 0x18, 0x3b, + 0x7e, 0x8e, 0xca, 0xbd, 0xf1, 0x71, 0x5a, 0x64 }, + { 0x7a, 0x7c, 0x55, 0xa5, 0x6f, 0xa9, 0xae, 0x51, + 0xe6, 0x55, 0xe0, 0x19, 0x75, 0xd8, 0xa6, 0xff, + 0x4a, 0xe9, 0xe4, 0xb4, 0x86, 0xfc, 0xbe, 0x4e, + 0xac, 0x04, 0x45, 0x88, 0xf2, 0x45, 0xeb, 0xea }, + { 0x2a, 0xfd, 0xf3, 0xc8, 0x2a, 0xbc, 0x48, 0x67, + 0xf5, 0xde, 0x11, 0x12, 0x86, 0xc2, 0xb3, 0xbe, + 0x7d, 0x6e, 0x48, 0x65, 0x7b, 0xa9, 0x23, 0xcf, + 0xbf, 0x10, 0x1a, 0x6d, 0xfc, 0xf9, 0xdb, 0x9a }, + { 0x41, 0x03, 0x7d, 0x2e, 0xdc, 0xdc, 0xe0, 0xc4, + 0x9b, 0x7f, 0xb4, 0xa6, 0xaa, 0x09, 0x99, 0xca, + 0x66, 0x97, 0x6c, 0x74, 0x83, 0xaf, 0xe6, 0x31, + 0xd4, 0xed, 0xa2, 0x83, 0x14, 0x4f, 0x6d, 0xfc }, + { 0xc4, 0x46, 0x6f, 0x84, 0x97, 0xca, 0x2e, 0xeb, + 0x45, 0x83, 0xa0, 0xb0, 0x8e, 0x9d, 0x9a, 0xc7, + 0x43, 0x95, 0x70, 0x9f, 0xda, 0x10, 0x9d, 0x24, + 0xf2, 0xe4, 0x46, 0x21, 0x96, 0x77, 0x9c, 0x5d }, + { 0x75, 0xf6, 0x09, 0x33, 0x8a, 0xa6, 0x7d, 0x96, + 0x9a, 0x2a, 0xe2, 0xa2, 0x36, 0x2b, 0x2d, 0xa9, + 0xd7, 0x7c, 0x69, 0x5d, 0xfd, 0x1d, 0xf7, 0x22, + 0x4a, 0x69, 0x01, 0xdb, 0x93, 0x2c, 0x33, 0x64 }, + { 0x68, 0x60, 0x6c, 0xeb, 0x98, 0x9d, 0x54, 0x88, + 0xfc, 0x7c, 0xf6, 0x49, 0xf3, 0xd7, 0xc2, 0x72, + 0xef, 0x05, 0x5d, 0xa1, 0xa9, 0x3f, 0xae, 0xcd, + 0x55, 0xfe, 0x06, 0xf6, 0x96, 0x70, 0x98, 0xca }, + { 0x44, 0x34, 0x6b, 0xde, 0xb7, 0xe0, 0x52, 0xf6, + 0x25, 0x50, 0x48, 0xf0, 0xd9, 0xb4, 0x2c, 0x42, + 0x5b, 0xab, 0x9c, 0x3d, 0xd2, 0x41, 0x68, 0x21, + 0x2c, 0x3e, 0xcf, 0x1e, 0xbf, 0x34, 0xe6, 0xae }, + { 0x8e, 0x9c, 0xf6, 0xe1, 0xf3, 0x66, 0x47, 0x1f, + 0x2a, 0xc7, 0xd2, 0xee, 0x9b, 0x5e, 0x62, 0x66, + 0xfd, 0xa7, 0x1f, 0x8f, 0x2e, 0x41, 0x09, 0xf2, + 0x23, 0x7e, 0xd5, 0xf8, 0x81, 0x3f, 0xc7, 0x18 }, + { 0x84, 0xbb, 0xeb, 0x84, 0x06, 0xd2, 0x50, 0x95, + 0x1f, 0x8c, 0x1b, 0x3e, 0x86, 0xa7, 0xc0, 0x10, + 0x08, 0x29, 0x21, 0x83, 0x3d, 0xfd, 0x95, 0x55, + 0xa2, 0xf9, 0x09, 0xb1, 0x08, 0x6e, 0xb4, 0xb8 }, + { 0xee, 0x66, 0x6f, 0x3e, 0xef, 0x0f, 0x7e, 0x2a, + 0x9c, 0x22, 0x29, 0x58, 0xc9, 0x7e, 0xaf, 0x35, + 0xf5, 0x1c, 0xed, 0x39, 0x3d, 0x71, 0x44, 0x85, + 0xab, 0x09, 0xa0, 0x69, 0x34, 0x0f, 0xdf, 0x88 }, + { 0xc1, 0x53, 0xd3, 0x4a, 0x65, 0xc4, 0x7b, 0x4a, + 0x62, 0xc5, 0xca, 0xcf, 0x24, 0x01, 0x09, 0x75, + 0xd0, 0x35, 0x6b, 0x2f, 0x32, 0xc8, 0xf5, 0xda, + 0x53, 0x0d, 0x33, 0x88, 0x16, 0xad, 0x5d, 0xe6 }, + { 0x9f, 0xc5, 0x45, 0x01, 0x09, 0xe1, 0xb7, 0x79, + 0xf6, 0xc7, 0xae, 0x79, 0xd5, 0x6c, 0x27, 0x63, + 0x5c, 0x8d, 0xd4, 0x26, 0xc5, 0xa9, 0xd5, 0x4e, + 0x25, 0x78, 0xdb, 0x98, 0x9b, 0x8c, 0x3b, 0x4e }, + { 0xd1, 0x2b, 0xf3, 0x73, 0x2e, 0xf4, 0xaf, 0x5c, + 0x22, 0xfa, 0x90, 0x35, 0x6a, 0xf8, 0xfc, 0x50, + 0xfc, 0xb4, 0x0f, 0x8f, 0x2e, 0xa5, 0xc8, 0x59, + 0x47, 0x37, 0xa3, 0xb3, 0xd5, 0xab, 0xdb, 0xd7 }, + { 0x11, 0x03, 0x0b, 0x92, 0x89, 0xbb, 0xa5, 0xaf, + 0x65, 0x26, 0x06, 0x72, 0xab, 0x6f, 0xee, 0x88, + 0xb8, 0x74, 0x20, 0xac, 0xef, 0x4a, 0x17, 0x89, + 0xa2, 0x07, 0x3b, 0x7e, 0xc2, 0xf2, 0xa0, 0x9e }, + { 0x69, 0xcb, 0x19, 0x2b, 0x84, 0x44, 0x00, 0x5c, + 0x8c, 0x0c, 0xeb, 0x12, 0xc8, 0x46, 0x86, 0x07, + 0x68, 0x18, 0x8c, 0xda, 0x0a, 0xec, 0x27, 0xa9, + 0xc8, 0xa5, 0x5c, 0xde, 0xe2, 0x12, 0x36, 0x32 }, + { 0xdb, 0x44, 0x4c, 0x15, 0x59, 0x7b, 0x5f, 0x1a, + 0x03, 0xd1, 0xf9, 0xed, 0xd1, 0x6e, 0x4a, 0x9f, + 0x43, 0xa6, 0x67, 0xcc, 0x27, 0x51, 0x75, 0xdf, + 0xa2, 0xb7, 0x04, 0xe3, 0xbb, 0x1a, 0x9b, 0x83 }, + { 0x3f, 0xb7, 0x35, 0x06, 0x1a, 0xbc, 0x51, 0x9d, + 0xfe, 0x97, 0x9e, 0x54, 0xc1, 0xee, 0x5b, 0xfa, + 0xd0, 0xa9, 0xd8, 0x58, 0xb3, 0x31, 0x5b, 0xad, + 0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd } +}; + +static bool __init blake2s_selftest(void) +{ + u8 key[BLAKE2S_KEY_SIZE]; + u8 buf[ARRAY_SIZE(blake2s_testvecs)]; + u8 hash[BLAKE2S_HASH_SIZE]; + size_t i; + bool success = true; + + for (i = 0; i < BLAKE2S_KEY_SIZE; ++i) + key[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) + buf[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_keyed_testvecs); ++i) { + blake2s(hash, buf, key, BLAKE2S_HASH_SIZE, i, BLAKE2S_KEY_SIZE); + if (memcmp(hash, blake2s_keyed_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s keyed self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) { + blake2s(hash, buf, NULL, BLAKE2S_HASH_SIZE, i, 0); + if (memcmp(hash, blake2s_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s unkeyed self-test %zu: FAIL\n", i + i); + success = false; + } + } + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20.c b/net/wireguard/crypto/zinc/selftest/chacha20.c new file mode 100644 index 0000000000000..1a2390aaf6c23 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20.c @@ -0,0 +1,2698 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20_testvec { + const u8 *input, *output, *key; + u64 nonce; + size_t ilen; +}; + +struct hchacha20_testvec { + u8 key[HCHACHA20_KEY_SIZE]; + u8 nonce[HCHACHA20_NONCE_SIZE]; + u8 output[CHACHA20_KEY_SIZE]; +}; + +/* These test vectors are generated by reference implementations and are + * designed to check chacha20 implementation block handling, as well as from + * the draft-arciszewski-xchacha-01 document. + */ + +static const u8 input01[] __initconst = { }; +static const u8 output01[] __initconst = { }; +static const u8 key01[] __initconst = { + 0x09, 0xf4, 0xe8, 0x57, 0x10, 0xf2, 0x12, 0xc3, + 0xc6, 0x91, 0xc4, 0x09, 0x97, 0x46, 0xef, 0xfe, + 0x02, 0x00, 0xe4, 0x5c, 0x82, 0xed, 0x16, 0xf3, + 0x32, 0xbe, 0xec, 0x7a, 0xe6, 0x68, 0x12, 0x26 +}; +enum { nonce01 = 0x3834e2afca3c66d3ULL }; + +static const u8 input02[] __initconst = { + 0x9d +}; +static const u8 output02[] __initconst = { + 0x94 +}; +static const u8 key02[] __initconst = { + 0x8c, 0x01, 0xac, 0xaf, 0x62, 0x63, 0x56, 0x7a, + 0xad, 0x23, 0x4c, 0x58, 0x29, 0x29, 0xbe, 0xab, + 0xe9, 0xf8, 0xdf, 0x6c, 0x8c, 0x74, 0x4d, 0x7d, + 0x13, 0x94, 0x10, 0x02, 0x3d, 0x8e, 0x9f, 0x94 +}; +enum { nonce02 = 0x5d1b3bfdedd9f73aULL }; + +static const u8 input03[] __initconst = { + 0x04, 0x16 +}; +static const u8 output03[] __initconst = { + 0x92, 0x07 +}; +static const u8 key03[] __initconst = { + 0x22, 0x0c, 0x79, 0x2c, 0x38, 0x51, 0xbe, 0x99, + 0xa9, 0x59, 0x24, 0x50, 0xef, 0x87, 0x38, 0xa6, + 0xa0, 0x97, 0x20, 0xcb, 0xb4, 0x0c, 0x94, 0x67, + 0x1f, 0x98, 0xdc, 0xc4, 0x83, 0xbc, 0x35, 0x4d +}; +enum { nonce03 = 0x7a3353ad720a3e2eULL }; + +static const u8 input04[] __initconst = { + 0xc7, 0xcc, 0xd0 +}; +static const u8 output04[] __initconst = { + 0xd8, 0x41, 0x80 +}; +static const u8 key04[] __initconst = { + 0x81, 0x5e, 0x12, 0x01, 0xc4, 0x36, 0x15, 0x03, + 0x11, 0xa0, 0xe9, 0x86, 0xbb, 0x5a, 0xdc, 0x45, + 0x7d, 0x5e, 0x98, 0xf8, 0x06, 0x76, 0x1c, 0xec, + 0xc0, 0xf7, 0xca, 0x4e, 0x99, 0xd9, 0x42, 0x38 +}; +enum { nonce04 = 0x6816e2fc66176da2ULL }; + +static const u8 input05[] __initconst = { + 0x48, 0xf1, 0x31, 0x5f +}; +static const u8 output05[] __initconst = { + 0x48, 0xf7, 0x13, 0x67 +}; +static const u8 key05[] __initconst = { + 0x3f, 0xd6, 0xb6, 0x5e, 0x2f, 0xda, 0x82, 0x39, + 0x97, 0x06, 0xd3, 0x62, 0x4f, 0xbd, 0xcb, 0x9b, + 0x1d, 0xe6, 0x4a, 0x76, 0xab, 0xdd, 0x14, 0x50, + 0x59, 0x21, 0xe3, 0xb2, 0xc7, 0x95, 0xbc, 0x45 +}; +enum { nonce05 = 0xc41a7490e228cc42ULL }; + +static const u8 input06[] __initconst = { + 0xae, 0xa2, 0x85, 0x1d, 0xc8 +}; +static const u8 output06[] __initconst = { + 0xfa, 0xff, 0x45, 0x6b, 0x6f +}; +static const u8 key06[] __initconst = { + 0x04, 0x8d, 0xea, 0x67, 0x20, 0x78, 0xfb, 0x8f, + 0x49, 0x80, 0x35, 0xb5, 0x7b, 0xe4, 0x31, 0x74, + 0x57, 0x43, 0x3a, 0x64, 0x64, 0xb9, 0xe6, 0x23, + 0x4d, 0xfe, 0xb8, 0x7b, 0x71, 0x4d, 0x9d, 0x21 +}; +enum { nonce06 = 0x251366db50b10903ULL }; + +static const u8 input07[] __initconst = { + 0x1a, 0x32, 0x85, 0xb6, 0xe8, 0x52 +}; +static const u8 output07[] __initconst = { + 0xd3, 0x5f, 0xf0, 0x07, 0x69, 0xec +}; +static const u8 key07[] __initconst = { + 0xbf, 0x2d, 0x42, 0x99, 0x97, 0x76, 0x04, 0xad, + 0xd3, 0x8f, 0x6e, 0x6a, 0x34, 0x85, 0xaf, 0x81, + 0xef, 0x36, 0x33, 0xd5, 0x43, 0xa2, 0xaa, 0x08, + 0x0f, 0x77, 0x42, 0x83, 0x58, 0xc5, 0x42, 0x2a +}; +enum { nonce07 = 0xe0796da17dba9b58ULL }; + +static const u8 input08[] __initconst = { + 0x40, 0xae, 0xcd, 0xe4, 0x3d, 0x22, 0xe0 +}; +static const u8 output08[] __initconst = { + 0xfd, 0x8a, 0x9f, 0x3d, 0x05, 0xc9, 0xd3 +}; +static const u8 key08[] __initconst = { + 0xdc, 0x3f, 0x41, 0xe3, 0x23, 0x2a, 0x8d, 0xf6, + 0x41, 0x2a, 0xa7, 0x66, 0x05, 0x68, 0xe4, 0x7b, + 0xc4, 0x58, 0xd6, 0xcc, 0xdf, 0x0d, 0xc6, 0x25, + 0x1b, 0x61, 0x32, 0x12, 0x4e, 0xf1, 0xe6, 0x29 +}; +enum { nonce08 = 0xb1d2536d9e159832ULL }; + +static const u8 input09[] __initconst = { + 0xba, 0x1d, 0x14, 0x16, 0x9f, 0x83, 0x67, 0x24 +}; +static const u8 output09[] __initconst = { + 0x7c, 0xe3, 0x78, 0x1d, 0xa2, 0xe7, 0xe9, 0x39 +}; +static const u8 key09[] __initconst = { + 0x17, 0x55, 0x90, 0x52, 0xa4, 0xce, 0x12, 0xae, + 0xd4, 0xfd, 0xd4, 0xfb, 0xd5, 0x18, 0x59, 0x50, + 0x4e, 0x51, 0x99, 0x32, 0x09, 0x31, 0xfc, 0xf7, + 0x27, 0x10, 0x8e, 0xa2, 0x4b, 0xa5, 0xf5, 0x62 +}; +enum { nonce09 = 0x495fc269536d003ULL }; + +static const u8 input10[] __initconst = { + 0x09, 0xfd, 0x3c, 0x0b, 0x3d, 0x0e, 0xf3, 0x9d, + 0x27 +}; +static const u8 output10[] __initconst = { + 0xdc, 0xe4, 0x33, 0x60, 0x0c, 0x07, 0xcb, 0x51, + 0x6b +}; +static const u8 key10[] __initconst = { + 0x4e, 0x00, 0x72, 0x37, 0x0f, 0x52, 0x4d, 0x6f, + 0x37, 0x50, 0x3c, 0xb3, 0x51, 0x81, 0x49, 0x16, + 0x7e, 0xfd, 0xb1, 0x51, 0x72, 0x2e, 0xe4, 0x16, + 0x68, 0x5c, 0x5b, 0x8a, 0xc3, 0x90, 0x70, 0x04 +}; +enum { nonce10 = 0x1ad9d1114d88cbbdULL }; + +static const u8 input11[] __initconst = { + 0x70, 0x18, 0x52, 0x85, 0xba, 0x66, 0xff, 0x2c, + 0x9a, 0x46 +}; +static const u8 output11[] __initconst = { + 0xf5, 0x2a, 0x7a, 0xfd, 0x31, 0x7c, 0x91, 0x41, + 0xb1, 0xcf +}; +static const u8 key11[] __initconst = { + 0x48, 0xb4, 0xd0, 0x7c, 0x88, 0xd1, 0x96, 0x0d, + 0x80, 0x33, 0xb4, 0xd5, 0x31, 0x9a, 0x88, 0xca, + 0x14, 0xdc, 0xf0, 0xa8, 0xf3, 0xac, 0xb8, 0x47, + 0x75, 0x86, 0x7c, 0x88, 0x50, 0x11, 0x43, 0x40 +}; +enum { nonce11 = 0x47c35dd1f4f8aa4fULL }; + +static const u8 input12[] __initconst = { + 0x9e, 0x8e, 0x3d, 0x2a, 0x05, 0xfd, 0xe4, 0x90, + 0x24, 0x1c, 0xd3 +}; +static const u8 output12[] __initconst = { + 0x97, 0x72, 0x40, 0x9f, 0xc0, 0x6b, 0x05, 0x33, + 0x42, 0x7e, 0x28 +}; +static const u8 key12[] __initconst = { + 0xee, 0xff, 0x33, 0x33, 0xe0, 0x28, 0xdf, 0xa2, + 0xb6, 0x5e, 0x25, 0x09, 0x52, 0xde, 0xa5, 0x9c, + 0x8f, 0x95, 0xa9, 0x03, 0x77, 0x0f, 0xbe, 0xa1, + 0xd0, 0x7d, 0x73, 0x2f, 0xf8, 0x7e, 0x51, 0x44 +}; +enum { nonce12 = 0xc22d044dc6ea4af3ULL }; + +static const u8 input13[] __initconst = { + 0x9c, 0x16, 0xa2, 0x22, 0x4d, 0xbe, 0x04, 0x9a, + 0xb3, 0xb5, 0xc6, 0x58 +}; +static const u8 output13[] __initconst = { + 0xf0, 0x81, 0xdb, 0x6d, 0xa3, 0xe9, 0xb2, 0xc6, + 0x32, 0x50, 0x16, 0x9f +}; +static const u8 key13[] __initconst = { + 0x96, 0xb3, 0x01, 0xd2, 0x7a, 0x8c, 0x94, 0x09, + 0x4f, 0x58, 0xbe, 0x80, 0xcc, 0xa9, 0x7e, 0x2d, + 0xad, 0x58, 0x3b, 0x63, 0xb8, 0x5c, 0x17, 0xce, + 0xbf, 0x43, 0x33, 0x7a, 0x7b, 0x82, 0x28, 0x2f +}; +enum { nonce13 = 0x2a5d05d88cd7b0daULL }; + +static const u8 input14[] __initconst = { + 0x57, 0x4f, 0xaa, 0x30, 0xe6, 0x23, 0x50, 0x86, + 0x91, 0xa5, 0x60, 0x96, 0x2b +}; +static const u8 output14[] __initconst = { + 0x6c, 0x1f, 0x3b, 0x42, 0xb6, 0x2f, 0xf0, 0xbd, + 0x76, 0x60, 0xc7, 0x7e, 0x8d +}; +static const u8 key14[] __initconst = { + 0x22, 0x85, 0xaf, 0x8f, 0xa3, 0x53, 0xa0, 0xc4, + 0xb5, 0x75, 0xc0, 0xba, 0x30, 0x92, 0xc3, 0x32, + 0x20, 0x5a, 0x8f, 0x7e, 0x93, 0xda, 0x65, 0x18, + 0xd1, 0xf6, 0x9a, 0x9b, 0x8f, 0x85, 0x30, 0xe6 +}; +enum { nonce14 = 0xf9946c166aa4475fULL }; + +static const u8 input15[] __initconst = { + 0x89, 0x81, 0xc7, 0xe2, 0x00, 0xac, 0x52, 0x70, + 0xa4, 0x79, 0xab, 0xeb, 0x74, 0xf7 +}; +static const u8 output15[] __initconst = { + 0xb4, 0xd0, 0xa9, 0x9d, 0x15, 0x5f, 0x48, 0xd6, + 0x00, 0x7e, 0x4c, 0x77, 0x5a, 0x46 +}; +static const u8 key15[] __initconst = { + 0x0a, 0x66, 0x36, 0xca, 0x5d, 0x82, 0x23, 0xb6, + 0xe4, 0x9b, 0xad, 0x5e, 0xd0, 0x7f, 0xf6, 0x7a, + 0x7b, 0x03, 0xa7, 0x4c, 0xfd, 0xec, 0xd5, 0xa1, + 0xfc, 0x25, 0x54, 0xda, 0x5a, 0x5c, 0xf0, 0x2c +}; +enum { nonce15 = 0x9ab2b87a35e772c8ULL }; + +static const u8 input16[] __initconst = { + 0x5f, 0x09, 0xc0, 0x8b, 0x1e, 0xde, 0xca, 0xd9, + 0xb7, 0x5c, 0x23, 0xc9, 0x55, 0x1e, 0xcf +}; +static const u8 output16[] __initconst = { + 0x76, 0x9b, 0x53, 0xf3, 0x66, 0x88, 0x28, 0x60, + 0x98, 0x80, 0x2c, 0xa8, 0x80, 0xa6, 0x48 +}; +static const u8 key16[] __initconst = { + 0x80, 0xb5, 0x51, 0xdf, 0x17, 0x5b, 0xb0, 0xef, + 0x8b, 0x5b, 0x2e, 0x3e, 0xc5, 0xe3, 0xa5, 0x86, + 0xac, 0x0d, 0x8e, 0x32, 0x90, 0x9d, 0x82, 0x27, + 0xf1, 0x23, 0x26, 0xc3, 0xea, 0x55, 0xb6, 0x63 +}; +enum { nonce16 = 0xa82e9d39e4d02ef5ULL }; + +static const u8 input17[] __initconst = { + 0x87, 0x0b, 0x36, 0x71, 0x7c, 0xb9, 0x0b, 0x80, + 0x4d, 0x77, 0x5c, 0x4f, 0xf5, 0x51, 0x0e, 0x1a +}; +static const u8 output17[] __initconst = { + 0xf1, 0x12, 0x4a, 0x8a, 0xd9, 0xd0, 0x08, 0x67, + 0x66, 0xd7, 0x34, 0xea, 0x32, 0x3b, 0x54, 0x0e +}; +static const u8 key17[] __initconst = { + 0xfb, 0x71, 0x5f, 0x3f, 0x7a, 0xc0, 0x9a, 0xc8, + 0xc8, 0xcf, 0xe8, 0xbc, 0xfb, 0x09, 0xbf, 0x89, + 0x6a, 0xef, 0xd5, 0xe5, 0x36, 0x87, 0x14, 0x76, + 0x00, 0xb9, 0x32, 0x28, 0xb2, 0x00, 0x42, 0x53 +}; +enum { nonce17 = 0x229b87e73d557b96ULL }; + +static const u8 input18[] __initconst = { + 0x38, 0x42, 0xb5, 0x37, 0xb4, 0x3d, 0xfe, 0x59, + 0x38, 0x68, 0x88, 0xfa, 0x89, 0x8a, 0x5f, 0x90, + 0x3c +}; +static const u8 output18[] __initconst = { + 0xac, 0xad, 0x14, 0xe8, 0x7e, 0xd7, 0xce, 0x96, + 0x3d, 0xb3, 0x78, 0x85, 0x22, 0x5a, 0xcb, 0x39, + 0xd4 +}; +static const u8 key18[] __initconst = { + 0xe1, 0xc1, 0xa8, 0xe0, 0x91, 0xe7, 0x38, 0x66, + 0x80, 0x17, 0x12, 0x3c, 0x5e, 0x2d, 0xbb, 0xea, + 0xeb, 0x6c, 0x8b, 0xc8, 0x1b, 0x6f, 0x7c, 0xea, + 0x50, 0x57, 0x23, 0x1e, 0x65, 0x6f, 0x6d, 0x81 +}; +enum { nonce18 = 0xfaf5fcf8f30e57a9ULL }; + +static const u8 input19[] __initconst = { + 0x1c, 0x4a, 0x30, 0x26, 0xef, 0x9a, 0x32, 0xa7, + 0x8f, 0xe5, 0xc0, 0x0f, 0x30, 0x3a, 0xbf, 0x38, + 0x54, 0xba +}; +static const u8 output19[] __initconst = { + 0x57, 0x67, 0x54, 0x4f, 0x31, 0xd6, 0xef, 0x35, + 0x0b, 0xd9, 0x52, 0xa7, 0x46, 0x7d, 0x12, 0x17, + 0x1e, 0xe3 +}; +static const u8 key19[] __initconst = { + 0x5a, 0x79, 0xc1, 0xea, 0x33, 0xb3, 0xc7, 0x21, + 0xec, 0xf8, 0xcb, 0xd2, 0x58, 0x96, 0x23, 0xd6, + 0x4d, 0xed, 0x2f, 0xdf, 0x8a, 0x79, 0xe6, 0x8b, + 0x38, 0xa3, 0xc3, 0x7a, 0x33, 0xda, 0x02, 0xc7 +}; +enum { nonce19 = 0x2b23b61840429604ULL }; + +static const u8 input20[] __initconst = { + 0xab, 0xe9, 0x32, 0xbb, 0x35, 0x17, 0xe0, 0x60, + 0x80, 0xb1, 0x27, 0xdc, 0xe6, 0x62, 0x9e, 0x0c, + 0x77, 0xf4, 0x50 +}; +static const u8 output20[] __initconst = { + 0x54, 0x6d, 0xaa, 0xfc, 0x08, 0xfb, 0x71, 0xa8, + 0xd6, 0x1d, 0x7d, 0xf3, 0x45, 0x10, 0xb5, 0x4c, + 0xcc, 0x4b, 0x45 +}; +static const u8 key20[] __initconst = { + 0xa3, 0xfd, 0x3d, 0xa9, 0xeb, 0xea, 0x2c, 0x69, + 0xcf, 0x59, 0x38, 0x13, 0x5b, 0xa7, 0x53, 0x8f, + 0x5e, 0xa2, 0x33, 0x86, 0x4c, 0x75, 0x26, 0xaf, + 0x35, 0x12, 0x09, 0x71, 0x81, 0xea, 0x88, 0x66 +}; +enum { nonce20 = 0x7459667a8fadff58ULL }; + +static const u8 input21[] __initconst = { + 0xa6, 0x82, 0x21, 0x23, 0xad, 0x27, 0x3f, 0xc6, + 0xd7, 0x16, 0x0d, 0x6d, 0x24, 0x15, 0x54, 0xc5, + 0x96, 0x72, 0x59, 0x8a +}; +static const u8 output21[] __initconst = { + 0x5f, 0x34, 0x32, 0xea, 0x06, 0xd4, 0x9e, 0x01, + 0xdc, 0x32, 0x32, 0x40, 0x66, 0x73, 0x6d, 0x4a, + 0x6b, 0x12, 0x20, 0xe8 +}; +static const u8 key21[] __initconst = { + 0x96, 0xfd, 0x13, 0x23, 0xa9, 0x89, 0x04, 0xe6, + 0x31, 0xa5, 0x2c, 0xc1, 0x40, 0xd5, 0x69, 0x5c, + 0x32, 0x79, 0x56, 0xe0, 0x29, 0x93, 0x8f, 0xe8, + 0x5f, 0x65, 0x53, 0x7f, 0xc1, 0xe9, 0xaf, 0xaf +}; +enum { nonce21 = 0xba8defee9d8e13b5ULL }; + +static const u8 input22[] __initconst = { + 0xb8, 0x32, 0x1a, 0x81, 0xd8, 0x38, 0x89, 0x5a, + 0xb0, 0x05, 0xbe, 0xf4, 0xd2, 0x08, 0xc6, 0xee, + 0x79, 0x7b, 0x3a, 0x76, 0x59 +}; +static const u8 output22[] __initconst = { + 0xb7, 0xba, 0xae, 0x80, 0xe4, 0x9f, 0x79, 0x84, + 0x5a, 0x48, 0x50, 0x6d, 0xcb, 0xd0, 0x06, 0x0c, + 0x15, 0x63, 0xa7, 0x5e, 0xbd +}; +static const u8 key22[] __initconst = { + 0x0f, 0x35, 0x3d, 0xeb, 0x5f, 0x0a, 0x82, 0x0d, + 0x24, 0x59, 0x71, 0xd8, 0xe6, 0x2d, 0x5f, 0xe1, + 0x7e, 0x0c, 0xae, 0xf6, 0xdc, 0x2c, 0xc5, 0x4a, + 0x38, 0x88, 0xf2, 0xde, 0xd9, 0x5f, 0x76, 0x7c +}; +enum { nonce22 = 0xe77f1760e9f5e192ULL }; + +static const u8 input23[] __initconst = { + 0x4b, 0x1e, 0x79, 0x99, 0xcf, 0xef, 0x64, 0x4b, + 0xb0, 0x66, 0xae, 0x99, 0x2e, 0x68, 0x97, 0xf5, + 0x5d, 0x9b, 0x3f, 0x7a, 0xa9, 0xd9 +}; +static const u8 output23[] __initconst = { + 0x5f, 0xa4, 0x08, 0x39, 0xca, 0xfa, 0x2b, 0x83, + 0x5d, 0x95, 0x70, 0x7c, 0x2e, 0xd4, 0xae, 0xfa, + 0x45, 0x4a, 0x77, 0x7f, 0xa7, 0x65 +}; +static const u8 key23[] __initconst = { + 0x4a, 0x06, 0x83, 0x64, 0xaa, 0xe3, 0x38, 0x32, + 0x28, 0x5d, 0xa4, 0xb2, 0x5a, 0xee, 0xcf, 0x8e, + 0x19, 0x67, 0xf1, 0x09, 0xe8, 0xc9, 0xf6, 0x40, + 0x02, 0x6d, 0x0b, 0xde, 0xfa, 0x81, 0x03, 0xb1 +}; +enum { nonce23 = 0x9b3f349158709849ULL }; + +static const u8 input24[] __initconst = { + 0xc6, 0xfc, 0x47, 0x5e, 0xd8, 0xed, 0xa9, 0xe5, + 0x4f, 0x82, 0x79, 0x35, 0xee, 0x3e, 0x7e, 0x3e, + 0x35, 0x70, 0x6e, 0xfa, 0x6d, 0x08, 0xe8 +}; +static const u8 output24[] __initconst = { + 0x3b, 0xc5, 0xf8, 0xc2, 0xbf, 0x2b, 0x90, 0x33, + 0xa6, 0xae, 0xf5, 0x5a, 0x65, 0xb3, 0x3d, 0xe1, + 0xcd, 0x5f, 0x55, 0xfa, 0xe7, 0xa5, 0x4a +}; +static const u8 key24[] __initconst = { + 0x00, 0x24, 0xc3, 0x65, 0x5f, 0xe6, 0x31, 0xbb, + 0x6d, 0xfc, 0x20, 0x7b, 0x1b, 0xa8, 0x96, 0x26, + 0x55, 0x21, 0x62, 0x25, 0x7e, 0xba, 0x23, 0x97, + 0xc9, 0xb8, 0x53, 0xa8, 0xef, 0xab, 0xad, 0x61 +}; +enum { nonce24 = 0x13ee0b8f526177c3ULL }; + +static const u8 input25[] __initconst = { + 0x33, 0x07, 0x16, 0xb1, 0x34, 0x33, 0x67, 0x04, + 0x9b, 0x0a, 0xce, 0x1b, 0xe9, 0xde, 0x1a, 0xec, + 0xd0, 0x55, 0xfb, 0xc6, 0x33, 0xaf, 0x2d, 0xe3 +}; +static const u8 output25[] __initconst = { + 0x05, 0x93, 0x10, 0xd1, 0x58, 0x6f, 0x68, 0x62, + 0x45, 0xdb, 0x91, 0xae, 0x70, 0xcf, 0xd4, 0x5f, + 0xee, 0xdf, 0xd5, 0xba, 0x9e, 0xde, 0x68, 0xe6 +}; +static const u8 key25[] __initconst = { + 0x83, 0xa9, 0x4f, 0x5d, 0x74, 0xd5, 0x91, 0xb3, + 0xc9, 0x97, 0x19, 0x15, 0xdb, 0x0d, 0x0b, 0x4a, + 0x3d, 0x55, 0xcf, 0xab, 0xb2, 0x05, 0x21, 0x35, + 0x45, 0x50, 0xeb, 0xf8, 0xf5, 0xbf, 0x36, 0x35 +}; +enum { nonce25 = 0x7c6f459e49ebfebcULL }; + +static const u8 input26[] __initconst = { + 0xc2, 0xd4, 0x7a, 0xa3, 0x92, 0xe1, 0xac, 0x46, + 0x1a, 0x15, 0x38, 0xc9, 0xb5, 0xfd, 0xdf, 0x84, + 0x38, 0xbc, 0x6b, 0x1d, 0xb0, 0x83, 0x43, 0x04, + 0x39 +}; +static const u8 output26[] __initconst = { + 0x7f, 0xde, 0xd6, 0x87, 0xcc, 0x34, 0xf4, 0x12, + 0xae, 0x55, 0xa5, 0x89, 0x95, 0x29, 0xfc, 0x18, + 0xd8, 0xc7, 0x7c, 0xd3, 0xcb, 0x85, 0x95, 0x21, + 0xd2 +}; +static const u8 key26[] __initconst = { + 0xe4, 0xd0, 0x54, 0x1d, 0x7d, 0x47, 0xa8, 0xc1, + 0x08, 0xca, 0xe2, 0x42, 0x52, 0x95, 0x16, 0x43, + 0xa3, 0x01, 0x23, 0x03, 0xcc, 0x3b, 0x81, 0x78, + 0x23, 0xcc, 0xa7, 0x36, 0xd7, 0xa0, 0x97, 0x8d +}; +enum { nonce26 = 0x524401012231683ULL }; + +static const u8 input27[] __initconst = { + 0x0d, 0xb0, 0xcf, 0xec, 0xfc, 0x38, 0x9d, 0x9d, + 0x89, 0x00, 0x96, 0xf2, 0x79, 0x8a, 0xa1, 0x8d, + 0x32, 0x5e, 0xc6, 0x12, 0x22, 0xec, 0xf6, 0x52, + 0xc1, 0x0b +}; +static const u8 output27[] __initconst = { + 0xef, 0xe1, 0xf2, 0x67, 0x8e, 0x2c, 0x00, 0x9f, + 0x1d, 0x4c, 0x66, 0x1f, 0x94, 0x58, 0xdc, 0xbb, + 0xb9, 0x11, 0x8f, 0x74, 0xfd, 0x0e, 0x14, 0x01, + 0xa8, 0x21 +}; +static const u8 key27[] __initconst = { + 0x78, 0x71, 0xa4, 0xe6, 0xb2, 0x95, 0x44, 0x12, + 0x81, 0xaa, 0x7e, 0x94, 0xa7, 0x8d, 0x44, 0xea, + 0xc4, 0xbc, 0x01, 0xb7, 0x9e, 0xf7, 0x82, 0x9e, + 0x3b, 0x23, 0x9f, 0x31, 0xdd, 0xb8, 0x0d, 0x18 +}; +enum { nonce27 = 0xd58fe0e58fb254d6ULL }; + +static const u8 input28[] __initconst = { + 0xaa, 0xb7, 0xaa, 0xd9, 0xa8, 0x91, 0xd7, 0x8a, + 0x97, 0x9b, 0xdb, 0x7c, 0x47, 0x2b, 0xdb, 0xd2, + 0xda, 0x77, 0xb1, 0xfa, 0x2d, 0x12, 0xe3, 0xe9, + 0xc4, 0x7f, 0x54 +}; +static const u8 output28[] __initconst = { + 0x87, 0x84, 0xa9, 0xa6, 0xad, 0x8f, 0xe6, 0x0f, + 0x69, 0xf8, 0x21, 0xc3, 0x54, 0x95, 0x0f, 0xb0, + 0x4e, 0xc7, 0x02, 0xe4, 0x04, 0xb0, 0x6c, 0x42, + 0x8c, 0x63, 0xe3 +}; +static const u8 key28[] __initconst = { + 0x12, 0x23, 0x37, 0x95, 0x04, 0xb4, 0x21, 0xe8, + 0xbc, 0x65, 0x46, 0x7a, 0xf4, 0x01, 0x05, 0x3f, + 0xb1, 0x34, 0x73, 0xd2, 0x49, 0xbf, 0x6f, 0x20, + 0xbd, 0x23, 0x58, 0x5f, 0xd1, 0x73, 0x57, 0xa6 +}; +enum { nonce28 = 0x3a04d51491eb4e07ULL }; + +static const u8 input29[] __initconst = { + 0x55, 0xd0, 0xd4, 0x4b, 0x17, 0xc8, 0xc4, 0x2b, + 0xc0, 0x28, 0xbd, 0x9d, 0x65, 0x4d, 0xaf, 0x77, + 0x72, 0x7c, 0x36, 0x68, 0xa7, 0xb6, 0x87, 0x4d, + 0xb9, 0x27, 0x25, 0x6c +}; +static const u8 output29[] __initconst = { + 0x0e, 0xac, 0x4c, 0xf5, 0x12, 0xb5, 0x56, 0xa5, + 0x00, 0x9a, 0xd6, 0xe5, 0x1a, 0x59, 0x2c, 0xf6, + 0x42, 0x22, 0xcf, 0x23, 0x98, 0x34, 0x29, 0xac, + 0x6e, 0xe3, 0x37, 0x6d +}; +static const u8 key29[] __initconst = { + 0xda, 0x9d, 0x05, 0x0c, 0x0c, 0xba, 0x75, 0xb9, + 0x9e, 0xb1, 0x8d, 0xd9, 0x73, 0x26, 0x2c, 0xa9, + 0x3a, 0xb5, 0xcb, 0x19, 0x49, 0xa7, 0x4f, 0xf7, + 0x64, 0x35, 0x23, 0x20, 0x2a, 0x45, 0x78, 0xc7 +}; +enum { nonce29 = 0xc25ac9982431cbfULL }; + +static const u8 input30[] __initconst = { + 0x4e, 0xd6, 0x85, 0xbb, 0xe7, 0x99, 0xfa, 0x04, + 0x33, 0x24, 0xfd, 0x75, 0x18, 0xe3, 0xd3, 0x25, + 0xcd, 0xca, 0xae, 0x00, 0xbe, 0x52, 0x56, 0x4a, + 0x31, 0xe9, 0x4f, 0xae, 0x8a +}; +static const u8 output30[] __initconst = { + 0x30, 0x36, 0x32, 0xa2, 0x3c, 0xb6, 0xf9, 0xf9, + 0x76, 0x70, 0xad, 0xa6, 0x10, 0x41, 0x00, 0x4a, + 0xfa, 0xce, 0x1b, 0x86, 0x05, 0xdb, 0x77, 0x96, + 0xb3, 0xb7, 0x8f, 0x61, 0x24 +}; +static const u8 key30[] __initconst = { + 0x49, 0x35, 0x4c, 0x15, 0x98, 0xfb, 0xc6, 0x57, + 0x62, 0x6d, 0x06, 0xc3, 0xd4, 0x79, 0x20, 0x96, + 0x05, 0x2a, 0x31, 0x63, 0xc0, 0x44, 0x42, 0x09, + 0x13, 0x13, 0xff, 0x1b, 0xc8, 0x63, 0x1f, 0x0b +}; +enum { nonce30 = 0x4967f9c08e41568bULL }; + +static const u8 input31[] __initconst = { + 0x91, 0x04, 0x20, 0x47, 0x59, 0xee, 0xa6, 0x0f, + 0x04, 0x75, 0xc8, 0x18, 0x95, 0x44, 0x01, 0x28, + 0x20, 0x6f, 0x73, 0x68, 0x66, 0xb5, 0x03, 0xb3, + 0x58, 0x27, 0x6e, 0x7a, 0x76, 0xb8 +}; +static const u8 output31[] __initconst = { + 0xe8, 0x03, 0x78, 0x9d, 0x13, 0x15, 0x98, 0xef, + 0x64, 0x68, 0x12, 0x41, 0xb0, 0x29, 0x94, 0x0c, + 0x83, 0x35, 0x46, 0xa9, 0x74, 0xe1, 0x75, 0xf0, + 0xb6, 0x96, 0xc3, 0x6f, 0xd7, 0x70 +}; +static const u8 key31[] __initconst = { + 0xef, 0xcd, 0x5a, 0x4a, 0xf4, 0x7e, 0x6a, 0x3a, + 0x11, 0x88, 0x72, 0x94, 0xb8, 0xae, 0x84, 0xc3, + 0x66, 0xe0, 0xde, 0x4b, 0x00, 0xa5, 0xd6, 0x2d, + 0x50, 0xb7, 0x28, 0xff, 0x76, 0x57, 0x18, 0x1f +}; +enum { nonce31 = 0xcb6f428fa4192e19ULL }; + +static const u8 input32[] __initconst = { + 0x90, 0x06, 0x50, 0x4b, 0x98, 0x14, 0x30, 0xf1, + 0xb8, 0xd7, 0xf0, 0xa4, 0x3e, 0x4e, 0xd8, 0x00, + 0xea, 0xdb, 0x4f, 0x93, 0x05, 0xef, 0x02, 0x71, + 0x1a, 0xcd, 0xa3, 0xb1, 0xae, 0xd3, 0x18 +}; +static const u8 output32[] __initconst = { + 0xcb, 0x4a, 0x37, 0x3f, 0xea, 0x40, 0xab, 0x86, + 0xfe, 0xcc, 0x07, 0xd5, 0xdc, 0xb2, 0x25, 0xb6, + 0xfd, 0x2a, 0x72, 0xbc, 0x5e, 0xd4, 0x75, 0xff, + 0x71, 0xfc, 0xce, 0x1e, 0x6f, 0x22, 0xc1 +}; +static const u8 key32[] __initconst = { + 0xfc, 0x6d, 0xc3, 0x80, 0xce, 0xa4, 0x31, 0xa1, + 0xcc, 0xfa, 0x9d, 0x10, 0x0b, 0xc9, 0x11, 0x77, + 0x34, 0xdb, 0xad, 0x1b, 0xc4, 0xfc, 0xeb, 0x79, + 0x91, 0xda, 0x59, 0x3b, 0x0d, 0xb1, 0x19, 0x3b +}; +enum { nonce32 = 0x88551bf050059467ULL }; + +static const u8 input33[] __initconst = { + 0x88, 0x94, 0x71, 0x92, 0xe8, 0xd7, 0xf9, 0xbd, + 0x55, 0xe3, 0x22, 0xdb, 0x99, 0x51, 0xfb, 0x50, + 0xbf, 0x82, 0xb5, 0x70, 0x8b, 0x2b, 0x6a, 0x03, + 0x37, 0xa0, 0xc6, 0x19, 0x5d, 0xc9, 0xbc, 0xcc +}; +static const u8 output33[] __initconst = { + 0xb6, 0x17, 0x51, 0xc8, 0xea, 0x8a, 0x14, 0xdc, + 0x23, 0x1b, 0xd4, 0xed, 0xbf, 0x50, 0xb9, 0x38, + 0x00, 0xc2, 0x3f, 0x78, 0x3d, 0xbf, 0xa0, 0x84, + 0xef, 0x45, 0xb2, 0x7d, 0x48, 0x7b, 0x62, 0xa7 +}; +static const u8 key33[] __initconst = { + 0xb9, 0x8f, 0x6a, 0xad, 0xb4, 0x6f, 0xb5, 0xdc, + 0x48, 0xfa, 0x43, 0x57, 0x62, 0x97, 0xef, 0x89, + 0x4c, 0x5a, 0x7b, 0x67, 0xb8, 0x9d, 0xf0, 0x42, + 0x2b, 0x8f, 0xf3, 0x18, 0x05, 0x2e, 0x48, 0xd0 +}; +enum { nonce33 = 0x31f16488fe8447f5ULL }; + +static const u8 input34[] __initconst = { + 0xda, 0x2b, 0x3d, 0x63, 0x9e, 0x4f, 0xc2, 0xb8, + 0x7f, 0xc2, 0x1a, 0x8b, 0x0d, 0x95, 0x65, 0x55, + 0x52, 0xba, 0x51, 0x51, 0xc0, 0x61, 0x9f, 0x0a, + 0x5d, 0xb0, 0x59, 0x8c, 0x64, 0x6a, 0xab, 0xf5, + 0x57 +}; +static const u8 output34[] __initconst = { + 0x5c, 0xf6, 0x62, 0x24, 0x8c, 0x45, 0xa3, 0x26, + 0xd0, 0xe4, 0x88, 0x1c, 0xed, 0xc4, 0x26, 0x58, + 0xb5, 0x5d, 0x92, 0xc4, 0x17, 0x44, 0x1c, 0xb8, + 0x2c, 0xf3, 0x55, 0x7e, 0xd6, 0xe5, 0xb3, 0x65, + 0xa8 +}; +static const u8 key34[] __initconst = { + 0xde, 0xd1, 0x27, 0xb7, 0x7c, 0xfa, 0xa6, 0x78, + 0x39, 0x80, 0xdf, 0xb7, 0x46, 0xac, 0x71, 0x26, + 0xd0, 0x2a, 0x56, 0x79, 0x12, 0xeb, 0x26, 0x37, + 0x01, 0x0d, 0x30, 0xe0, 0xe3, 0x66, 0xb2, 0xf4 +}; +enum { nonce34 = 0x92d0d9b252c24149ULL }; + +static const u8 input35[] __initconst = { + 0x3a, 0x15, 0x5b, 0x75, 0x6e, 0xd0, 0x52, 0x20, + 0x6c, 0x82, 0xfa, 0xce, 0x5b, 0xea, 0xf5, 0x43, + 0xc1, 0x81, 0x7c, 0xb2, 0xac, 0x16, 0x3f, 0xd3, + 0x5a, 0xaf, 0x55, 0x98, 0xf4, 0xc6, 0xba, 0x71, + 0x25, 0x8b +}; +static const u8 output35[] __initconst = { + 0xb3, 0xaf, 0xac, 0x6d, 0x4d, 0xc7, 0x68, 0x56, + 0x50, 0x5b, 0x69, 0x2a, 0xe5, 0x90, 0xf9, 0x5f, + 0x99, 0x88, 0xff, 0x0c, 0xa6, 0xb1, 0x83, 0xd6, + 0x80, 0xa6, 0x1b, 0xde, 0x94, 0xa4, 0x2c, 0xc3, + 0x74, 0xfa +}; +static const u8 key35[] __initconst = { + 0xd8, 0x24, 0xe2, 0x06, 0xd7, 0x7a, 0xce, 0x81, + 0x52, 0x72, 0x02, 0x69, 0x89, 0xc4, 0xe9, 0x53, + 0x3b, 0x08, 0x5f, 0x98, 0x1e, 0x1b, 0x99, 0x6e, + 0x28, 0x17, 0x6d, 0xba, 0xc0, 0x96, 0xf9, 0x3c +}; +enum { nonce35 = 0x7baf968c4c8e3a37ULL }; + +static const u8 input36[] __initconst = { + 0x31, 0x5d, 0x4f, 0xe3, 0xac, 0xad, 0x17, 0xa6, + 0xb5, 0x01, 0xe2, 0xc6, 0xd4, 0x7e, 0xc4, 0x80, + 0xc0, 0x59, 0x72, 0xbb, 0x4b, 0x74, 0x6a, 0x41, + 0x0f, 0x9c, 0xf6, 0xca, 0x20, 0xb3, 0x73, 0x07, + 0x6b, 0x02, 0x2a +}; +static const u8 output36[] __initconst = { + 0xf9, 0x09, 0x92, 0x94, 0x7e, 0x31, 0xf7, 0x53, + 0xe8, 0x8a, 0x5b, 0x20, 0xef, 0x9b, 0x45, 0x81, + 0xba, 0x5e, 0x45, 0x63, 0xc1, 0xc7, 0x9e, 0x06, + 0x0e, 0xd9, 0x62, 0x8e, 0x96, 0xf9, 0xfa, 0x43, + 0x4d, 0xd4, 0x28 +}; +static const u8 key36[] __initconst = { + 0x13, 0x30, 0x4c, 0x06, 0xae, 0x18, 0xde, 0x03, + 0x1d, 0x02, 0x40, 0xf5, 0xbb, 0x19, 0xe3, 0x88, + 0x41, 0xb1, 0x29, 0x15, 0x97, 0xc2, 0x69, 0x3f, + 0x32, 0x2a, 0x0c, 0x8b, 0xcf, 0x83, 0x8b, 0x6c +}; +enum { nonce36 = 0x226d251d475075a0ULL }; + +static const u8 input37[] __initconst = { + 0x10, 0x18, 0xbe, 0xfd, 0x66, 0xc9, 0x77, 0xcc, + 0x43, 0xe5, 0x46, 0x0b, 0x08, 0x8b, 0xae, 0x11, + 0x86, 0x15, 0xc2, 0xf6, 0x45, 0xd4, 0x5f, 0xd6, + 0xb6, 0x5f, 0x9f, 0x3e, 0x97, 0xb7, 0xd4, 0xad, + 0x0b, 0xe8, 0x31, 0x94 +}; +static const u8 output37[] __initconst = { + 0x03, 0x2c, 0x1c, 0xee, 0xc6, 0xdd, 0xed, 0x38, + 0x80, 0x6d, 0x84, 0x16, 0xc3, 0xc2, 0x04, 0x63, + 0xcd, 0xa7, 0x6e, 0x36, 0x8b, 0xed, 0x78, 0x63, + 0x95, 0xfc, 0x69, 0x7a, 0x3f, 0x8d, 0x75, 0x6b, + 0x6c, 0x26, 0x56, 0x4d +}; +static const u8 key37[] __initconst = { + 0xac, 0x84, 0x4d, 0xa9, 0x29, 0x49, 0x3c, 0x39, + 0x7f, 0xd9, 0xa6, 0x01, 0xf3, 0x7e, 0xfa, 0x4a, + 0x14, 0x80, 0x22, 0x74, 0xf0, 0x29, 0x30, 0x2d, + 0x07, 0x21, 0xda, 0xc0, 0x4d, 0x70, 0x56, 0xa2 +}; +enum { nonce37 = 0x167823ce3b64925aULL }; + +static const u8 input38[] __initconst = { + 0x30, 0x8f, 0xfa, 0x24, 0x29, 0xb1, 0xfb, 0xce, + 0x31, 0x62, 0xdc, 0xd0, 0x46, 0xab, 0xe1, 0x31, + 0xd9, 0xae, 0x60, 0x0d, 0xca, 0x0a, 0x49, 0x12, + 0x3d, 0x92, 0xe9, 0x91, 0x67, 0x12, 0x62, 0x18, + 0x89, 0xe2, 0xf9, 0x1c, 0xcc +}; +static const u8 output38[] __initconst = { + 0x56, 0x9c, 0xc8, 0x7a, 0xc5, 0x98, 0xa3, 0x0f, + 0xba, 0xd5, 0x3e, 0xe1, 0xc9, 0x33, 0x64, 0x33, + 0xf0, 0xd5, 0xf7, 0x43, 0x66, 0x0e, 0x08, 0x9a, + 0x6e, 0x09, 0xe4, 0x01, 0x0d, 0x1e, 0x2f, 0x4b, + 0xed, 0x9c, 0x08, 0x8c, 0x03 +}; +static const u8 key38[] __initconst = { + 0x77, 0x52, 0x2a, 0x23, 0xf1, 0xc5, 0x96, 0x2b, + 0x89, 0x4f, 0x3e, 0xf3, 0xff, 0x0e, 0x94, 0xce, + 0xf1, 0xbd, 0x53, 0xf5, 0x77, 0xd6, 0x9e, 0x47, + 0x49, 0x3d, 0x16, 0x64, 0xff, 0x95, 0x42, 0x42 +}; +enum { nonce38 = 0xff629d7b82cef357ULL }; + +static const u8 input39[] __initconst = { + 0x38, 0x26, 0x27, 0xd0, 0xc2, 0xf5, 0x34, 0xba, + 0xda, 0x0f, 0x1c, 0x1c, 0x9a, 0x70, 0xe5, 0x8a, + 0x78, 0x2d, 0x8f, 0x9a, 0xbf, 0x89, 0x6a, 0xfd, + 0xd4, 0x9c, 0x33, 0xf1, 0xb6, 0x89, 0x16, 0xe3, + 0x6a, 0x00, 0xfa, 0x3a, 0x0f, 0x26 +}; +static const u8 output39[] __initconst = { + 0x0f, 0xaf, 0x91, 0x6d, 0x9c, 0x99, 0xa4, 0xf7, + 0x3b, 0x9d, 0x9a, 0x98, 0xca, 0xbb, 0x50, 0x48, + 0xee, 0xcb, 0x5d, 0xa1, 0x37, 0x2d, 0x36, 0x09, + 0x2a, 0xe2, 0x1c, 0x3d, 0x98, 0x40, 0x1c, 0x16, + 0x56, 0xa7, 0x98, 0xe9, 0x7d, 0x2b +}; +static const u8 key39[] __initconst = { + 0x6e, 0x83, 0x15, 0x4d, 0xf8, 0x78, 0xa8, 0x0e, + 0x71, 0x37, 0xd4, 0x6e, 0x28, 0x5c, 0x06, 0xa1, + 0x2d, 0x6c, 0x72, 0x7a, 0xfd, 0xf8, 0x65, 0x1a, + 0xb8, 0xe6, 0x29, 0x7b, 0xe5, 0xb3, 0x23, 0x79 +}; +enum { nonce39 = 0xa4d8c491cf093e9dULL }; + +static const u8 input40[] __initconst = { + 0x8f, 0x32, 0x7c, 0x40, 0x37, 0x95, 0x08, 0x00, + 0x00, 0xfe, 0x2f, 0x95, 0x20, 0x12, 0x40, 0x18, + 0x5e, 0x7e, 0x5e, 0x99, 0xee, 0x8d, 0x91, 0x7d, + 0x50, 0x7d, 0x21, 0x45, 0x27, 0xe1, 0x7f, 0xd4, + 0x73, 0x10, 0xe1, 0x33, 0xbc, 0xf8, 0xdd +}; +static const u8 output40[] __initconst = { + 0x78, 0x7c, 0xdc, 0x55, 0x2b, 0xd9, 0x2b, 0x3a, + 0xdd, 0x56, 0x11, 0x52, 0xd3, 0x2e, 0xe0, 0x0d, + 0x23, 0x20, 0x8a, 0xf1, 0x4f, 0xee, 0xf1, 0x68, + 0xf6, 0xdc, 0x53, 0xcf, 0x17, 0xd4, 0xf0, 0x6c, + 0xdc, 0x80, 0x5f, 0x1c, 0xa4, 0x91, 0x05 +}; +static const u8 key40[] __initconst = { + 0x0d, 0x86, 0xbf, 0x8a, 0xba, 0x9e, 0x39, 0x91, + 0xa8, 0xe7, 0x22, 0xf0, 0x0c, 0x43, 0x18, 0xe4, + 0x1f, 0xb0, 0xaf, 0x8a, 0x34, 0x31, 0xf4, 0x41, + 0xf0, 0x89, 0x85, 0xca, 0x5d, 0x05, 0x3b, 0x94 +}; +enum { nonce40 = 0xae7acc4f5986439eULL }; + +static const u8 input41[] __initconst = { + 0x20, 0x5f, 0xc1, 0x83, 0x36, 0x02, 0x76, 0x96, + 0xf0, 0xbf, 0x8e, 0x0e, 0x1a, 0xd1, 0xc7, 0x88, + 0x18, 0xc7, 0x09, 0xc4, 0x15, 0xd9, 0x4f, 0x5e, + 0x1f, 0xb3, 0xb4, 0x6d, 0xcb, 0xa0, 0xd6, 0x8a, + 0x3b, 0x40, 0x8e, 0x80, 0xf1, 0xe8, 0x8f, 0x5f +}; +static const u8 output41[] __initconst = { + 0x0b, 0xd1, 0x49, 0x9a, 0x9d, 0xe8, 0x97, 0xb8, + 0xd1, 0xeb, 0x90, 0x62, 0x37, 0xd2, 0x99, 0x15, + 0x67, 0x6d, 0x27, 0x93, 0xce, 0x37, 0x65, 0xa2, + 0x94, 0x88, 0xd6, 0x17, 0xbc, 0x1c, 0x6e, 0xa2, + 0xcc, 0xfb, 0x81, 0x0e, 0x30, 0x60, 0x5a, 0x6f +}; +static const u8 key41[] __initconst = { + 0x36, 0x27, 0x57, 0x01, 0x21, 0x68, 0x97, 0xc7, + 0x00, 0x67, 0x7b, 0xe9, 0x0f, 0x55, 0x49, 0xbb, + 0x92, 0x18, 0x98, 0xf5, 0x5e, 0xbc, 0xe7, 0x5a, + 0x9d, 0x3d, 0xc7, 0xbd, 0x59, 0xec, 0x82, 0x8e +}; +enum { nonce41 = 0x5da05e4c8dfab464ULL }; + +static const u8 input42[] __initconst = { + 0xca, 0x30, 0xcd, 0x63, 0xf0, 0x2d, 0xf1, 0x03, + 0x4d, 0x0d, 0xf2, 0xf7, 0x6f, 0xae, 0xd6, 0x34, + 0xea, 0xf6, 0x13, 0xcf, 0x1c, 0xa0, 0xd0, 0xe8, + 0xa4, 0x78, 0x80, 0x3b, 0x1e, 0xa5, 0x32, 0x4c, + 0x73, 0x12, 0xd4, 0x6a, 0x94, 0xbc, 0xba, 0x80, + 0x5e +}; +static const u8 output42[] __initconst = { + 0xec, 0x3f, 0x18, 0x31, 0xc0, 0x7b, 0xb5, 0xe2, + 0xad, 0xf3, 0xec, 0xa0, 0x16, 0x9d, 0xef, 0xce, + 0x05, 0x65, 0x59, 0x9d, 0x5a, 0xca, 0x3e, 0x13, + 0xb9, 0x5d, 0x5d, 0xb5, 0xeb, 0xae, 0xc0, 0x87, + 0xbb, 0xfd, 0xe7, 0xe4, 0x89, 0x5b, 0xd2, 0x6c, + 0x56 +}; +static const u8 key42[] __initconst = { + 0x7c, 0x6b, 0x7e, 0x77, 0xcc, 0x8c, 0x1b, 0x03, + 0x8b, 0x2a, 0xb3, 0x7c, 0x5a, 0x73, 0xcc, 0xac, + 0xdd, 0x53, 0x54, 0x0c, 0x85, 0xed, 0xcd, 0x47, + 0x24, 0xc1, 0xb8, 0x9b, 0x2e, 0x41, 0x92, 0x36 +}; +enum { nonce42 = 0xe4d7348b09682c9cULL }; + +static const u8 input43[] __initconst = { + 0x52, 0xf2, 0x4b, 0x7c, 0xe5, 0x58, 0xe8, 0xd2, + 0xb7, 0xf3, 0xa1, 0x29, 0x68, 0xa2, 0x50, 0x50, + 0xae, 0x9c, 0x1b, 0xe2, 0x67, 0x77, 0xe2, 0xdb, + 0x85, 0x55, 0x7e, 0x84, 0x8a, 0x12, 0x3c, 0xb6, + 0x2e, 0xed, 0xd3, 0xec, 0x47, 0x68, 0xfa, 0x52, + 0x46, 0x9d +}; +static const u8 output43[] __initconst = { + 0x1b, 0xf0, 0x05, 0xe4, 0x1c, 0xd8, 0x74, 0x9a, + 0xf0, 0xee, 0x00, 0x54, 0xce, 0x02, 0x83, 0x15, + 0xfb, 0x23, 0x35, 0x78, 0xc3, 0xda, 0x98, 0xd8, + 0x9d, 0x1b, 0xb2, 0x51, 0x82, 0xb0, 0xff, 0xbe, + 0x05, 0xa9, 0xa4, 0x04, 0xba, 0xea, 0x4b, 0x73, + 0x47, 0x6e +}; +static const u8 key43[] __initconst = { + 0xeb, 0xec, 0x0e, 0xa1, 0x65, 0xe2, 0x99, 0x46, + 0xd8, 0x54, 0x8c, 0x4a, 0x93, 0xdf, 0x6d, 0xbf, + 0x93, 0x34, 0x94, 0x57, 0xc9, 0x12, 0x9d, 0x68, + 0x05, 0xc5, 0x05, 0xad, 0x5a, 0xc9, 0x2a, 0x3b +}; +enum { nonce43 = 0xe14f6a902b7827fULL }; + +static const u8 input44[] __initconst = { + 0x3e, 0x22, 0x3e, 0x8e, 0xcd, 0x18, 0xe2, 0xa3, + 0x8d, 0x8b, 0x38, 0xc3, 0x02, 0xa3, 0x31, 0x48, + 0xc6, 0x0e, 0xec, 0x99, 0x51, 0x11, 0x6d, 0x8b, + 0x32, 0x35, 0x3b, 0x08, 0x58, 0x76, 0x25, 0x30, + 0xe2, 0xfc, 0xa2, 0x46, 0x7d, 0x6e, 0x34, 0x87, + 0xac, 0x42, 0xbf +}; +static const u8 output44[] __initconst = { + 0x08, 0x92, 0x58, 0x02, 0x1a, 0xf4, 0x1f, 0x3d, + 0x38, 0x7b, 0x6b, 0xf6, 0x84, 0x07, 0xa3, 0x19, + 0x17, 0x2a, 0xed, 0x57, 0x1c, 0xf9, 0x55, 0x37, + 0x4e, 0xf4, 0x68, 0x68, 0x82, 0x02, 0x4f, 0xca, + 0x21, 0x00, 0xc6, 0x66, 0x79, 0x53, 0x19, 0xef, + 0x7f, 0xdd, 0x74 +}; +static const u8 key44[] __initconst = { + 0x73, 0xb6, 0x3e, 0xf4, 0x57, 0x52, 0xa6, 0x43, + 0x51, 0xd8, 0x25, 0x00, 0xdb, 0xb4, 0x52, 0x69, + 0xd6, 0x27, 0x49, 0xeb, 0x9b, 0xf1, 0x7b, 0xa0, + 0xd6, 0x7c, 0x9c, 0xd8, 0x95, 0x03, 0x69, 0x26 +}; +enum { nonce44 = 0xf5e6dc4f35ce24e5ULL }; + +static const u8 input45[] __initconst = { + 0x55, 0x76, 0xc0, 0xf1, 0x74, 0x03, 0x7a, 0x6d, + 0x14, 0xd8, 0x36, 0x2c, 0x9f, 0x9a, 0x59, 0x7a, + 0x2a, 0xf5, 0x77, 0x84, 0x70, 0x7c, 0x1d, 0x04, + 0x90, 0x45, 0xa4, 0xc1, 0x5e, 0xdd, 0x2e, 0x07, + 0x18, 0x34, 0xa6, 0x85, 0x56, 0x4f, 0x09, 0xaf, + 0x2f, 0x83, 0xe1, 0xc6 +}; +static const u8 output45[] __initconst = { + 0x22, 0x46, 0xe4, 0x0b, 0x3a, 0x55, 0xcc, 0x9b, + 0xf0, 0xc0, 0x53, 0xcd, 0x95, 0xc7, 0x57, 0x6c, + 0x77, 0x46, 0x41, 0x72, 0x07, 0xbf, 0xa8, 0xe5, + 0x68, 0x69, 0xd8, 0x1e, 0x45, 0xc1, 0xa2, 0x50, + 0xa5, 0xd1, 0x62, 0xc9, 0x5a, 0x7d, 0x08, 0x14, + 0xae, 0x44, 0x16, 0xb9 +}; +static const u8 key45[] __initconst = { + 0x41, 0xf3, 0x88, 0xb2, 0x51, 0x25, 0x47, 0x02, + 0x39, 0xe8, 0x15, 0x3a, 0x22, 0x78, 0x86, 0x0b, + 0xf9, 0x1e, 0x8d, 0x98, 0xb2, 0x22, 0x82, 0xac, + 0x42, 0x94, 0xde, 0x64, 0xf0, 0xfd, 0xb3, 0x6c +}; +enum { nonce45 = 0xf51a582daf4aa01aULL }; + +static const u8 input46[] __initconst = { + 0xf6, 0xff, 0x20, 0xf9, 0x26, 0x7e, 0x0f, 0xa8, + 0x6a, 0x45, 0x5a, 0x91, 0x73, 0xc4, 0x4c, 0x63, + 0xe5, 0x61, 0x59, 0xca, 0xec, 0xc0, 0x20, 0x35, + 0xbc, 0x9f, 0x58, 0x9c, 0x5e, 0xa1, 0x17, 0x46, + 0xcc, 0xab, 0x6e, 0xd0, 0x4f, 0x24, 0xeb, 0x05, + 0x4d, 0x40, 0x41, 0xe0, 0x9d +}; +static const u8 output46[] __initconst = { + 0x31, 0x6e, 0x63, 0x3f, 0x9c, 0xe6, 0xb1, 0xb7, + 0xef, 0x47, 0x46, 0xd7, 0xb1, 0x53, 0x42, 0x2f, + 0x2c, 0xc8, 0x01, 0xae, 0x8b, 0xec, 0x42, 0x2c, + 0x6b, 0x2c, 0x9c, 0xb2, 0xf0, 0x29, 0x06, 0xa5, + 0xcd, 0x7e, 0xc7, 0x3a, 0x38, 0x98, 0x8a, 0xde, + 0x03, 0x29, 0x14, 0x8f, 0xf9 +}; +static const u8 key46[] __initconst = { + 0xac, 0xa6, 0x44, 0x4a, 0x0d, 0x42, 0x10, 0xbc, + 0xd3, 0xc9, 0x8e, 0x9e, 0x71, 0xa3, 0x1c, 0x14, + 0x9d, 0x65, 0x0d, 0x49, 0x4d, 0x8c, 0xec, 0x46, + 0xe1, 0x41, 0xcd, 0xf5, 0xfc, 0x82, 0x75, 0x34 +}; +enum { nonce46 = 0x25f85182df84dec5ULL }; + +static const u8 input47[] __initconst = { + 0xa1, 0xd2, 0xf2, 0x52, 0x2f, 0x79, 0x50, 0xb2, + 0x42, 0x29, 0x5b, 0x44, 0x20, 0xf9, 0xbd, 0x85, + 0xb7, 0x65, 0x77, 0x86, 0xce, 0x3e, 0x1c, 0xe4, + 0x70, 0x80, 0xdd, 0x72, 0x07, 0x48, 0x0f, 0x84, + 0x0d, 0xfd, 0x97, 0xc0, 0xb7, 0x48, 0x9b, 0xb4, + 0xec, 0xff, 0x73, 0x14, 0x99, 0xe4 +}; +static const u8 output47[] __initconst = { + 0xe5, 0x3c, 0x78, 0x66, 0x31, 0x1e, 0xd6, 0xc4, + 0x9e, 0x71, 0xb3, 0xd7, 0xd5, 0xad, 0x84, 0xf2, + 0x78, 0x61, 0x77, 0xf8, 0x31, 0xf0, 0x13, 0xad, + 0x66, 0xf5, 0x31, 0x7d, 0xeb, 0xdf, 0xaf, 0xcb, + 0xac, 0x28, 0x6c, 0xc2, 0x9e, 0xe7, 0x78, 0xa2, + 0xa2, 0x58, 0xce, 0x84, 0x76, 0x70 +}; +static const u8 key47[] __initconst = { + 0x05, 0x7f, 0xc0, 0x7f, 0x37, 0x20, 0x71, 0x02, + 0x3a, 0xe7, 0x20, 0x5a, 0x0a, 0x8f, 0x79, 0x5a, + 0xfe, 0xbb, 0x43, 0x4d, 0x2f, 0xcb, 0xf6, 0x9e, + 0xa2, 0x97, 0x00, 0xad, 0x0d, 0x51, 0x7e, 0x17 +}; +enum { nonce47 = 0xae707c60f54de32bULL }; + +static const u8 input48[] __initconst = { + 0x80, 0x93, 0x77, 0x2e, 0x8d, 0xe8, 0xe6, 0xc1, + 0x27, 0xe6, 0xf2, 0x89, 0x5b, 0x33, 0x62, 0x18, + 0x80, 0x6e, 0x17, 0x22, 0x8e, 0x83, 0x31, 0x40, + 0x8f, 0xc9, 0x5c, 0x52, 0x6c, 0x0e, 0xa5, 0xe9, + 0x6c, 0x7f, 0xd4, 0x6a, 0x27, 0x56, 0x99, 0xce, + 0x8d, 0x37, 0x59, 0xaf, 0xc0, 0x0e, 0xe1 +}; +static const u8 output48[] __initconst = { + 0x02, 0xa4, 0x2e, 0x33, 0xb7, 0x7c, 0x2b, 0x9a, + 0x18, 0x5a, 0xba, 0x53, 0x38, 0xaf, 0x00, 0xeb, + 0xd8, 0x3d, 0x02, 0x77, 0x43, 0x45, 0x03, 0x91, + 0xe2, 0x5e, 0x4e, 0xeb, 0x50, 0xd5, 0x5b, 0xe0, + 0xf3, 0x33, 0xa7, 0xa2, 0xac, 0x07, 0x6f, 0xeb, + 0x3f, 0x6c, 0xcd, 0xf2, 0x6c, 0x61, 0x64 +}; +static const u8 key48[] __initconst = { + 0xf3, 0x79, 0xe7, 0xf8, 0x0e, 0x02, 0x05, 0x6b, + 0x83, 0x1a, 0xe7, 0x86, 0x6b, 0xe6, 0x8f, 0x3f, + 0xd3, 0xa3, 0xe4, 0x6e, 0x29, 0x06, 0xad, 0xbc, + 0xe8, 0x33, 0x56, 0x39, 0xdf, 0xb0, 0xe2, 0xfe +}; +enum { nonce48 = 0xd849b938c6569da0ULL }; + +static const u8 input49[] __initconst = { + 0x89, 0x3b, 0x88, 0x9e, 0x7b, 0x38, 0x16, 0x9f, + 0xa1, 0x28, 0xf6, 0xf5, 0x23, 0x74, 0x28, 0xb0, + 0xdf, 0x6c, 0x9e, 0x8a, 0x71, 0xaf, 0xed, 0x7a, + 0x39, 0x21, 0x57, 0x7d, 0x31, 0x6c, 0xee, 0x0d, + 0x11, 0x8d, 0x41, 0x9a, 0x5f, 0xb7, 0x27, 0x40, + 0x08, 0xad, 0xc6, 0xe0, 0x00, 0x43, 0x9e, 0xae +}; +static const u8 output49[] __initconst = { + 0x4d, 0xfd, 0xdb, 0x4c, 0x77, 0xc1, 0x05, 0x07, + 0x4d, 0x6d, 0x32, 0xcb, 0x2e, 0x0e, 0xff, 0x65, + 0xc9, 0x27, 0xeb, 0xa9, 0x46, 0x5b, 0xab, 0x06, + 0xe6, 0xb6, 0x5a, 0x1e, 0x00, 0xfb, 0xcf, 0xe4, + 0xb9, 0x71, 0x40, 0x10, 0xef, 0x12, 0x39, 0xf0, + 0xea, 0x40, 0xb8, 0x9a, 0xa2, 0x85, 0x38, 0x48 +}; +static const u8 key49[] __initconst = { + 0xe7, 0x10, 0x40, 0xd9, 0x66, 0xc0, 0xa8, 0x6d, + 0xa3, 0xcc, 0x8b, 0xdd, 0x93, 0xf2, 0x6e, 0xe0, + 0x90, 0x7f, 0xd0, 0xf4, 0x37, 0x0c, 0x8b, 0x9b, + 0x4c, 0x4d, 0xe6, 0xf2, 0x1f, 0xe9, 0x95, 0x24 +}; +enum { nonce49 = 0xf269817bdae01bc0ULL }; + +static const u8 input50[] __initconst = { + 0xda, 0x5b, 0x60, 0xcd, 0xed, 0x58, 0x8e, 0x7f, + 0xae, 0xdd, 0xc8, 0x2e, 0x16, 0x90, 0xea, 0x4b, + 0x0c, 0x74, 0x14, 0x35, 0xeb, 0xee, 0x2c, 0xff, + 0x46, 0x99, 0x97, 0x6e, 0xae, 0xa7, 0x8e, 0x6e, + 0x38, 0xfe, 0x63, 0xe7, 0x51, 0xd9, 0xaa, 0xce, + 0x7b, 0x1e, 0x7e, 0x5d, 0xc0, 0xe8, 0x10, 0x06, + 0x14 +}; +static const u8 output50[] __initconst = { + 0xe4, 0xe5, 0x86, 0x1b, 0x66, 0x19, 0xac, 0x49, + 0x1c, 0xbd, 0xee, 0x03, 0xaf, 0x11, 0xfc, 0x1f, + 0x6a, 0xd2, 0x50, 0x5c, 0xea, 0x2c, 0xa5, 0x75, + 0xfd, 0xb7, 0x0e, 0x80, 0x8f, 0xed, 0x3f, 0x31, + 0x47, 0xac, 0x67, 0x43, 0xb8, 0x2e, 0xb4, 0x81, + 0x6d, 0xe4, 0x1e, 0xb7, 0x8b, 0x0c, 0x53, 0xa9, + 0x26 +}; +static const u8 key50[] __initconst = { + 0xd7, 0xb2, 0x04, 0x76, 0x30, 0xcc, 0x38, 0x45, + 0xef, 0xdb, 0xc5, 0x86, 0x08, 0x61, 0xf0, 0xee, + 0x6d, 0xd8, 0x22, 0x04, 0x8c, 0xfb, 0xcb, 0x37, + 0xa6, 0xfb, 0x95, 0x22, 0xe1, 0x87, 0xb7, 0x6f +}; +enum { nonce50 = 0x3b44d09c45607d38ULL }; + +static const u8 input51[] __initconst = { + 0xa9, 0x41, 0x02, 0x4b, 0xd7, 0xd5, 0xd1, 0xf1, + 0x21, 0x55, 0xb2, 0x75, 0x6d, 0x77, 0x1b, 0x86, + 0xa9, 0xc8, 0x90, 0xfd, 0xed, 0x4a, 0x7b, 0x6c, + 0xb2, 0x5f, 0x9b, 0x5f, 0x16, 0xa1, 0x54, 0xdb, + 0xd6, 0x3f, 0x6a, 0x7f, 0x2e, 0x51, 0x9d, 0x49, + 0x5b, 0xa5, 0x0e, 0xf9, 0xfb, 0x2a, 0x38, 0xff, + 0x20, 0x8c +}; +static const u8 output51[] __initconst = { + 0x18, 0xf7, 0x88, 0xc1, 0x72, 0xfd, 0x90, 0x4b, + 0xa9, 0x2d, 0xdb, 0x47, 0xb0, 0xa5, 0xc4, 0x37, + 0x01, 0x95, 0xc4, 0xb1, 0xab, 0xc5, 0x5b, 0xcd, + 0xe1, 0x97, 0x78, 0x13, 0xde, 0x6a, 0xff, 0x36, + 0xce, 0xa4, 0x67, 0xc5, 0x4a, 0x45, 0x2b, 0xd9, + 0xff, 0x8f, 0x06, 0x7c, 0x63, 0xbb, 0x83, 0x17, + 0xb4, 0x6b +}; +static const u8 key51[] __initconst = { + 0x82, 0x1a, 0x79, 0xab, 0x9a, 0xb5, 0x49, 0x6a, + 0x30, 0x6b, 0x99, 0x19, 0x11, 0xc7, 0xa2, 0xf4, + 0xca, 0x55, 0xb9, 0xdd, 0xe7, 0x2f, 0xe7, 0xc1, + 0xdd, 0x27, 0xad, 0x80, 0xf2, 0x56, 0xad, 0xf3 +}; +enum { nonce51 = 0xe93aff94ca71a4a6ULL }; + +static const u8 input52[] __initconst = { + 0x89, 0xdd, 0xf3, 0xfa, 0xb6, 0xc1, 0xaa, 0x9a, + 0xc8, 0xad, 0x6b, 0x00, 0xa1, 0x65, 0xea, 0x14, + 0x55, 0x54, 0x31, 0x8f, 0xf0, 0x03, 0x84, 0x51, + 0x17, 0x1e, 0x0a, 0x93, 0x6e, 0x79, 0x96, 0xa3, + 0x2a, 0x85, 0x9c, 0x89, 0xf8, 0xd1, 0xe2, 0x15, + 0x95, 0x05, 0xf4, 0x43, 0x4d, 0x6b, 0xf0, 0x71, + 0x3b, 0x3e, 0xba +}; +static const u8 output52[] __initconst = { + 0x0c, 0x42, 0x6a, 0xb3, 0x66, 0x63, 0x5d, 0x2c, + 0x9f, 0x3d, 0xa6, 0x6e, 0xc7, 0x5f, 0x79, 0x2f, + 0x50, 0xe3, 0xd6, 0x07, 0x56, 0xa4, 0x2b, 0x2d, + 0x8d, 0x10, 0xc0, 0x6c, 0xa2, 0xfc, 0x97, 0xec, + 0x3f, 0x5c, 0x8d, 0x59, 0xbe, 0x84, 0xf1, 0x3e, + 0x38, 0x47, 0x4f, 0x75, 0x25, 0x66, 0x88, 0x14, + 0x03, 0xdd, 0xde +}; +static const u8 key52[] __initconst = { + 0x4f, 0xb0, 0x27, 0xb6, 0xdd, 0x24, 0x0c, 0xdb, + 0x6b, 0x71, 0x2e, 0xac, 0xfc, 0x3f, 0xa6, 0x48, + 0x5d, 0xd5, 0xff, 0x53, 0xb5, 0x62, 0xf1, 0xe0, + 0x93, 0xfe, 0x39, 0x4c, 0x9f, 0x03, 0x11, 0xa7 +}; +enum { nonce52 = 0xed8becec3bdf6f25ULL }; + +static const u8 input53[] __initconst = { + 0x68, 0xd1, 0xc7, 0x74, 0x44, 0x1c, 0x84, 0xde, + 0x27, 0x27, 0x35, 0xf0, 0x18, 0x0b, 0x57, 0xaa, + 0xd0, 0x1a, 0xd3, 0x3b, 0x5e, 0x5c, 0x62, 0x93, + 0xd7, 0x6b, 0x84, 0x3b, 0x71, 0x83, 0x77, 0x01, + 0x3e, 0x59, 0x45, 0xf4, 0x77, 0x6c, 0x6b, 0xcb, + 0x88, 0x45, 0x09, 0x1d, 0xc6, 0x45, 0x6e, 0xdc, + 0x6e, 0x51, 0xb8, 0x28 +}; +static const u8 output53[] __initconst = { + 0xc5, 0x90, 0x96, 0x78, 0x02, 0xf5, 0xc4, 0x3c, + 0xde, 0xd4, 0xd4, 0xc6, 0xa7, 0xad, 0x12, 0x47, + 0x45, 0xce, 0xcd, 0x8c, 0x35, 0xcc, 0xa6, 0x9e, + 0x5a, 0xc6, 0x60, 0xbb, 0xe3, 0xed, 0xec, 0x68, + 0x3f, 0x64, 0xf7, 0x06, 0x63, 0x9c, 0x8c, 0xc8, + 0x05, 0x3a, 0xad, 0x32, 0x79, 0x8b, 0x45, 0x96, + 0x93, 0x73, 0x4c, 0xe0 +}; +static const u8 key53[] __initconst = { + 0x42, 0x4b, 0x20, 0x81, 0x49, 0x50, 0xe9, 0xc2, + 0x43, 0x69, 0x36, 0xe7, 0x68, 0xae, 0xd5, 0x7e, + 0x42, 0x1a, 0x1b, 0xb4, 0x06, 0x4d, 0xa7, 0x17, + 0xb5, 0x31, 0xd6, 0x0c, 0xb0, 0x5c, 0x41, 0x0b +}; +enum { nonce53 = 0xf44ce1931fbda3d7ULL }; + +static const u8 input54[] __initconst = { + 0x7b, 0xf6, 0x8b, 0xae, 0xc0, 0xcb, 0x10, 0x8e, + 0xe8, 0xd8, 0x2e, 0x3b, 0x14, 0xba, 0xb4, 0xd2, + 0x58, 0x6b, 0x2c, 0xec, 0xc1, 0x81, 0x71, 0xb4, + 0xc6, 0xea, 0x08, 0xc5, 0xc9, 0x78, 0xdb, 0xa2, + 0xfa, 0x44, 0x50, 0x9b, 0xc8, 0x53, 0x8d, 0x45, + 0x42, 0xe7, 0x09, 0xc4, 0x29, 0xd8, 0x75, 0x02, + 0xbb, 0xb2, 0x78, 0xcf, 0xe7 +}; +static const u8 output54[] __initconst = { + 0xaf, 0x2c, 0x83, 0x26, 0x6e, 0x7f, 0xa6, 0xe9, + 0x03, 0x75, 0xfe, 0xfe, 0x87, 0x58, 0xcf, 0xb5, + 0xbc, 0x3c, 0x9d, 0xa1, 0x6e, 0x13, 0xf1, 0x0f, + 0x9e, 0xbc, 0xe0, 0x54, 0x24, 0x32, 0xce, 0x95, + 0xe6, 0xa5, 0x59, 0x3d, 0x24, 0x1d, 0x8f, 0xb1, + 0x74, 0x6c, 0x56, 0xe7, 0x96, 0xc1, 0x91, 0xc8, + 0x2d, 0x0e, 0xb7, 0x51, 0x10 +}; +static const u8 key54[] __initconst = { + 0x00, 0x68, 0x74, 0xdc, 0x30, 0x9e, 0xe3, 0x52, + 0xa9, 0xae, 0xb6, 0x7c, 0xa1, 0xdc, 0x12, 0x2d, + 0x98, 0x32, 0x7a, 0x77, 0xe1, 0xdd, 0xa3, 0x76, + 0x72, 0x34, 0x83, 0xd8, 0xb7, 0x69, 0xba, 0x77 +}; +enum { nonce54 = 0xbea57d79b798b63aULL }; + +static const u8 input55[] __initconst = { + 0xb5, 0xf4, 0x2f, 0xc1, 0x5e, 0x10, 0xa7, 0x4e, + 0x74, 0x3d, 0xa3, 0x96, 0xc0, 0x4d, 0x7b, 0x92, + 0x8f, 0xdb, 0x2d, 0x15, 0x52, 0x6a, 0x95, 0x5e, + 0x40, 0x81, 0x4f, 0x70, 0x73, 0xea, 0x84, 0x65, + 0x3d, 0x9a, 0x4e, 0x03, 0x95, 0xf8, 0x5d, 0x2f, + 0x07, 0x02, 0x13, 0x13, 0xdd, 0x82, 0xe6, 0x3b, + 0xe1, 0x5f, 0xb3, 0x37, 0x9b, 0x88 +}; +static const u8 output55[] __initconst = { + 0xc1, 0x88, 0xbd, 0x92, 0x77, 0xad, 0x7c, 0x5f, + 0xaf, 0xa8, 0x57, 0x0e, 0x40, 0x0a, 0xdc, 0x70, + 0xfb, 0xc6, 0x71, 0xfd, 0xc4, 0x74, 0x60, 0xcc, + 0xa0, 0x89, 0x8e, 0x99, 0xf0, 0x06, 0xa6, 0x7c, + 0x97, 0x42, 0x21, 0x81, 0x6a, 0x07, 0xe7, 0xb3, + 0xf7, 0xa5, 0x03, 0x71, 0x50, 0x05, 0x63, 0x17, + 0xa9, 0x46, 0x0b, 0xff, 0x30, 0x78 +}; +static const u8 key55[] __initconst = { + 0x19, 0x8f, 0xe7, 0xd7, 0x6b, 0x7f, 0x6f, 0x69, + 0x86, 0x91, 0x0f, 0xa7, 0x4a, 0x69, 0x8e, 0x34, + 0xf3, 0xdb, 0xde, 0xaf, 0xf2, 0x66, 0x1d, 0x64, + 0x97, 0x0c, 0xcf, 0xfa, 0x33, 0x84, 0xfd, 0x0c +}; +enum { nonce55 = 0x80aa3d3e2c51ef06ULL }; + +static const u8 input56[] __initconst = { + 0x6b, 0xe9, 0x73, 0x42, 0x27, 0x5e, 0x12, 0xcd, + 0xaa, 0x45, 0x12, 0x8b, 0xb3, 0xe6, 0x54, 0x33, + 0x31, 0x7d, 0xe2, 0x25, 0xc6, 0x86, 0x47, 0x67, + 0x86, 0x83, 0xe4, 0x46, 0xb5, 0x8f, 0x2c, 0xbb, + 0xe4, 0xb8, 0x9f, 0xa2, 0xa4, 0xe8, 0x75, 0x96, + 0x92, 0x51, 0x51, 0xac, 0x8e, 0x2e, 0x6f, 0xfc, + 0xbd, 0x0d, 0xa3, 0x9f, 0x16, 0x55, 0x3e +}; +static const u8 output56[] __initconst = { + 0x42, 0x99, 0x73, 0x6c, 0xd9, 0x4b, 0x16, 0xe5, + 0x18, 0x63, 0x1a, 0xd9, 0x0e, 0xf1, 0x15, 0x2e, + 0x0f, 0x4b, 0xe4, 0x5f, 0xa0, 0x4d, 0xde, 0x9f, + 0xa7, 0x18, 0xc1, 0x0c, 0x0b, 0xae, 0x55, 0xe4, + 0x89, 0x18, 0xa4, 0x78, 0x9d, 0x25, 0x0d, 0xd5, + 0x94, 0x0f, 0xf9, 0x78, 0xa3, 0xa6, 0xe9, 0x9e, + 0x2c, 0x73, 0xf0, 0xf7, 0x35, 0xf3, 0x2b +}; +static const u8 key56[] __initconst = { + 0x7d, 0x12, 0xad, 0x51, 0xd5, 0x6f, 0x8f, 0x96, + 0xc0, 0x5d, 0x9a, 0xd1, 0x7e, 0x20, 0x98, 0x0e, + 0x3c, 0x0a, 0x67, 0x6b, 0x1b, 0x88, 0x69, 0xd4, + 0x07, 0x8c, 0xaf, 0x0f, 0x3a, 0x28, 0xe4, 0x5d +}; +enum { nonce56 = 0x70f4c372fb8b5984ULL }; + +static const u8 input57[] __initconst = { + 0x28, 0xa3, 0x06, 0xe8, 0xe7, 0x08, 0xb9, 0xef, + 0x0d, 0x63, 0x15, 0x99, 0xb2, 0x78, 0x7e, 0xaf, + 0x30, 0x50, 0xcf, 0xea, 0xc9, 0x91, 0x41, 0x2f, + 0x3b, 0x38, 0x70, 0xc4, 0x87, 0xb0, 0x3a, 0xee, + 0x4a, 0xea, 0xe3, 0x83, 0x68, 0x8b, 0xcf, 0xda, + 0x04, 0xa5, 0xbd, 0xb2, 0xde, 0x3c, 0x55, 0x13, + 0xfe, 0x96, 0xad, 0xc1, 0x61, 0x1b, 0x98, 0xde +}; +static const u8 output57[] __initconst = { + 0xf4, 0x44, 0xe9, 0xd2, 0x6d, 0xc2, 0x5a, 0xe9, + 0xfd, 0x7e, 0x41, 0x54, 0x3f, 0xf4, 0x12, 0xd8, + 0x55, 0x0d, 0x12, 0x9b, 0xd5, 0x2e, 0x95, 0xe5, + 0x77, 0x42, 0x3f, 0x2c, 0xfb, 0x28, 0x9d, 0x72, + 0x6d, 0x89, 0x82, 0x27, 0x64, 0x6f, 0x0d, 0x57, + 0xa1, 0x25, 0xa3, 0x6b, 0x88, 0x9a, 0xac, 0x0c, + 0x76, 0x19, 0x90, 0xe2, 0x50, 0x5a, 0xf8, 0x12 +}; +static const u8 key57[] __initconst = { + 0x08, 0x26, 0xb8, 0xac, 0xf3, 0xa5, 0xc6, 0xa3, + 0x7f, 0x09, 0x87, 0xf5, 0x6c, 0x5a, 0x85, 0x6c, + 0x3d, 0xbd, 0xde, 0xd5, 0x87, 0xa3, 0x98, 0x7a, + 0xaa, 0x40, 0x3e, 0xf7, 0xff, 0x44, 0x5d, 0xee +}; +enum { nonce57 = 0xc03a6130bf06b089ULL }; + +static const u8 input58[] __initconst = { + 0x82, 0xa5, 0x38, 0x6f, 0xaa, 0xb4, 0xaf, 0xb2, + 0x42, 0x01, 0xa8, 0x39, 0x3f, 0x15, 0x51, 0xa8, + 0x11, 0x1b, 0x93, 0xca, 0x9c, 0xa0, 0x57, 0x68, + 0x8f, 0xdb, 0x68, 0x53, 0x51, 0x6d, 0x13, 0x22, + 0x12, 0x9b, 0xbd, 0x33, 0xa8, 0x52, 0x40, 0x57, + 0x80, 0x9b, 0x98, 0xef, 0x56, 0x70, 0x11, 0xfa, + 0x36, 0x69, 0x7d, 0x15, 0x48, 0xf9, 0x3b, 0xeb, + 0x42 +}; +static const u8 output58[] __initconst = { + 0xff, 0x3a, 0x74, 0xc3, 0x3e, 0x44, 0x64, 0x4d, + 0x0e, 0x5f, 0x9d, 0xa8, 0xdb, 0xbe, 0x12, 0xef, + 0xba, 0x56, 0x65, 0x50, 0x76, 0xaf, 0xa4, 0x4e, + 0x01, 0xc1, 0xd3, 0x31, 0x14, 0xe2, 0xbe, 0x7b, + 0xa5, 0x67, 0xb4, 0xe3, 0x68, 0x40, 0x9c, 0xb0, + 0xb1, 0x78, 0xef, 0x49, 0x03, 0x0f, 0x2d, 0x56, + 0xb4, 0x37, 0xdb, 0xbc, 0x2d, 0x68, 0x1c, 0x3c, + 0xf1 +}; +static const u8 key58[] __initconst = { + 0x7e, 0xf1, 0x7c, 0x20, 0x65, 0xed, 0xcd, 0xd7, + 0x57, 0xe8, 0xdb, 0x90, 0x87, 0xdb, 0x5f, 0x63, + 0x3d, 0xdd, 0xb8, 0x2b, 0x75, 0x8e, 0x04, 0xb5, + 0xf4, 0x12, 0x79, 0xa9, 0x4d, 0x42, 0x16, 0x7f +}; +enum { nonce58 = 0x92838183f80d2f7fULL }; + +static const u8 input59[] __initconst = { + 0x37, 0xf1, 0x9d, 0xdd, 0xd7, 0x08, 0x9f, 0x13, + 0xc5, 0x21, 0x82, 0x75, 0x08, 0x9e, 0x25, 0x16, + 0xb1, 0xd1, 0x71, 0x42, 0x28, 0x63, 0xac, 0x47, + 0x71, 0x54, 0xb1, 0xfc, 0x39, 0xf0, 0x61, 0x4f, + 0x7c, 0x6d, 0x4f, 0xc8, 0x33, 0xef, 0x7e, 0xc8, + 0xc0, 0x97, 0xfc, 0x1a, 0x61, 0xb4, 0x87, 0x6f, + 0xdd, 0x5a, 0x15, 0x7b, 0x1b, 0x95, 0x50, 0x94, + 0x1d, 0xba +}; +static const u8 output59[] __initconst = { + 0x73, 0x67, 0xc5, 0x07, 0xbb, 0x57, 0x79, 0xd5, + 0xc9, 0x04, 0xdd, 0x88, 0xf3, 0x86, 0xe5, 0x70, + 0x49, 0x31, 0xe0, 0xcc, 0x3b, 0x1d, 0xdf, 0xb0, + 0xaf, 0xf4, 0x2d, 0xe0, 0x06, 0x10, 0x91, 0x8d, + 0x1c, 0xcf, 0x31, 0x0b, 0xf6, 0x73, 0xda, 0x1c, + 0xf0, 0x17, 0x52, 0x9e, 0x20, 0x2e, 0x9f, 0x8c, + 0xb3, 0x59, 0xce, 0xd4, 0xd3, 0xc1, 0x81, 0xe9, + 0x11, 0x36 +}; +static const u8 key59[] __initconst = { + 0xbd, 0x07, 0xd0, 0x53, 0x2c, 0xb3, 0xcc, 0x3f, + 0xc4, 0x95, 0xfd, 0xe7, 0x81, 0xb3, 0x29, 0x99, + 0x05, 0x45, 0xd6, 0x95, 0x25, 0x0b, 0x72, 0xd3, + 0xcd, 0xbb, 0x73, 0xf8, 0xfa, 0xc0, 0x9b, 0x7a +}; +enum { nonce59 = 0x4a0db819b0d519e2ULL }; + +static const u8 input60[] __initconst = { + 0x58, 0x4e, 0xdf, 0x94, 0x3c, 0x76, 0x0a, 0x79, + 0x47, 0xf1, 0xbe, 0x88, 0xd3, 0xba, 0x94, 0xd8, + 0xe2, 0x8f, 0xe3, 0x2f, 0x2f, 0x74, 0x82, 0x55, + 0xc3, 0xda, 0xe2, 0x4e, 0x2c, 0x8c, 0x45, 0x1d, + 0x72, 0x8f, 0x54, 0x41, 0xb5, 0xb7, 0x69, 0xe4, + 0xdc, 0xd2, 0x36, 0x21, 0x5c, 0x28, 0x52, 0xf7, + 0x98, 0x8e, 0x72, 0xa7, 0x6d, 0x57, 0xed, 0xdc, + 0x3c, 0xe6, 0x6a +}; +static const u8 output60[] __initconst = { + 0xda, 0xaf, 0xb5, 0xe3, 0x30, 0x65, 0x5c, 0xb1, + 0x48, 0x08, 0x43, 0x7b, 0x9e, 0xd2, 0x6a, 0x62, + 0x56, 0x7c, 0xad, 0xd9, 0xe5, 0xf6, 0x09, 0x71, + 0xcd, 0xe6, 0x05, 0x6b, 0x3f, 0x44, 0x3a, 0x5c, + 0xf6, 0xf8, 0xd7, 0xce, 0x7d, 0xd1, 0xe0, 0x4f, + 0x88, 0x15, 0x04, 0xd8, 0x20, 0xf0, 0x3e, 0xef, + 0xae, 0xa6, 0x27, 0xa3, 0x0e, 0xfc, 0x18, 0x90, + 0x33, 0xcd, 0xd3 +}; +static const u8 key60[] __initconst = { + 0xbf, 0xfd, 0x25, 0xb5, 0xb2, 0xfc, 0x78, 0x0c, + 0x8e, 0xb9, 0x57, 0x2f, 0x26, 0x4a, 0x7e, 0x71, + 0xcc, 0xf2, 0xe0, 0xfd, 0x24, 0x11, 0x20, 0x23, + 0x57, 0x00, 0xff, 0x80, 0x11, 0x0c, 0x1e, 0xff +}; +enum { nonce60 = 0xf18df56fdb7954adULL }; + +static const u8 input61[] __initconst = { + 0xb0, 0xf3, 0x06, 0xbc, 0x22, 0xae, 0x49, 0x40, + 0xae, 0xff, 0x1b, 0x31, 0xa7, 0x98, 0xab, 0x1d, + 0xe7, 0x40, 0x23, 0x18, 0x4f, 0xab, 0x8e, 0x93, + 0x82, 0xf4, 0x56, 0x61, 0xfd, 0x2b, 0xcf, 0xa7, + 0xc4, 0xb4, 0x0a, 0xf4, 0xcb, 0xc7, 0x8c, 0x40, + 0x57, 0xac, 0x0b, 0x3e, 0x2a, 0x0a, 0x67, 0x83, + 0x50, 0xbf, 0xec, 0xb0, 0xc7, 0xf1, 0x32, 0x26, + 0x98, 0x80, 0x33, 0xb4 +}; +static const u8 output61[] __initconst = { + 0x9d, 0x23, 0x0e, 0xff, 0xcc, 0x7c, 0xd5, 0xcf, + 0x1a, 0xb8, 0x59, 0x1e, 0x92, 0xfd, 0x7f, 0xca, + 0xca, 0x3c, 0x18, 0x81, 0xde, 0xfa, 0x59, 0xc8, + 0x6f, 0x9c, 0x24, 0x3f, 0x3a, 0xe6, 0x0b, 0xb4, + 0x34, 0x48, 0x69, 0xfc, 0xb6, 0xea, 0xb2, 0xde, + 0x9f, 0xfd, 0x92, 0x36, 0x18, 0x98, 0x99, 0xaa, + 0x65, 0xe2, 0xea, 0xf4, 0xb1, 0x47, 0x8e, 0xb0, + 0xe7, 0xd4, 0x7a, 0x2c +}; +static const u8 key61[] __initconst = { + 0xd7, 0xfd, 0x9b, 0xbd, 0x8f, 0x65, 0x0d, 0x00, + 0xca, 0xa1, 0x6c, 0x85, 0x85, 0xa4, 0x6d, 0xf1, + 0xb1, 0x68, 0x0c, 0x8b, 0x5d, 0x37, 0x72, 0xd0, + 0xd8, 0xd2, 0x25, 0xab, 0x9f, 0x7b, 0x7d, 0x95 +}; +enum { nonce61 = 0xd82caf72a9c4864fULL }; + +static const u8 input62[] __initconst = { + 0x10, 0x77, 0xf3, 0x2f, 0xc2, 0x50, 0xd6, 0x0c, + 0xba, 0xa8, 0x8d, 0xce, 0x0d, 0x58, 0x9e, 0x87, + 0xb1, 0x59, 0x66, 0x0a, 0x4a, 0xb3, 0xd8, 0xca, + 0x0a, 0x6b, 0xf8, 0xc6, 0x2b, 0x3f, 0x8e, 0x09, + 0xe0, 0x0a, 0x15, 0x85, 0xfe, 0xaa, 0xc6, 0xbd, + 0x30, 0xef, 0xe4, 0x10, 0x78, 0x03, 0xc1, 0xc7, + 0x8a, 0xd9, 0xde, 0x0b, 0x51, 0x07, 0xc4, 0x7b, + 0xe2, 0x2e, 0x36, 0x3a, 0xc2 +}; +static const u8 output62[] __initconst = { + 0xa0, 0x0c, 0xfc, 0xc1, 0xf6, 0xaf, 0xc2, 0xb8, + 0x5c, 0xef, 0x6e, 0xf3, 0xce, 0x15, 0x48, 0x05, + 0xb5, 0x78, 0x49, 0x51, 0x1f, 0x9d, 0xf4, 0xbf, + 0x2f, 0x53, 0xa2, 0xd1, 0x15, 0x20, 0x82, 0x6b, + 0xd2, 0x22, 0x6c, 0x4e, 0x14, 0x87, 0xe3, 0xd7, + 0x49, 0x45, 0x84, 0xdb, 0x5f, 0x68, 0x60, 0xc4, + 0xb3, 0xe6, 0x3f, 0xd1, 0xfc, 0xa5, 0x73, 0xf3, + 0xfc, 0xbb, 0xbe, 0xc8, 0x9d +}; +static const u8 key62[] __initconst = { + 0x6e, 0xc9, 0xaf, 0xce, 0x35, 0xb9, 0x86, 0xd1, + 0xce, 0x5f, 0xd9, 0xbb, 0xd5, 0x1f, 0x7c, 0xcd, + 0xfe, 0x19, 0xaa, 0x3d, 0xea, 0x64, 0xc1, 0x28, + 0x40, 0xba, 0xa1, 0x28, 0xcd, 0x40, 0xb6, 0xf2 +}; +enum { nonce62 = 0xa1c0c265f900cde8ULL }; + +static const u8 input63[] __initconst = { + 0x7a, 0x70, 0x21, 0x2c, 0xef, 0xa6, 0x36, 0xd4, + 0xe0, 0xab, 0x8c, 0x25, 0x73, 0x34, 0xc8, 0x94, + 0x6c, 0x81, 0xcb, 0x19, 0x8d, 0x5a, 0x49, 0xaa, + 0x6f, 0xba, 0x83, 0x72, 0x02, 0x5e, 0xf5, 0x89, + 0xce, 0x79, 0x7e, 0x13, 0x3d, 0x5b, 0x98, 0x60, + 0x5d, 0xd9, 0xfb, 0x15, 0x93, 0x4c, 0xf3, 0x51, + 0x49, 0x55, 0xd1, 0x58, 0xdd, 0x7e, 0x6d, 0xfe, + 0xdd, 0x84, 0x23, 0x05, 0xba, 0xe9 +}; +static const u8 output63[] __initconst = { + 0x20, 0xb3, 0x5c, 0x03, 0x03, 0x78, 0x17, 0xfc, + 0x3b, 0x35, 0x30, 0x9a, 0x00, 0x18, 0xf5, 0xc5, + 0x06, 0x53, 0xf5, 0x04, 0x24, 0x9d, 0xd1, 0xb2, + 0xac, 0x5a, 0xb6, 0x2a, 0xa5, 0xda, 0x50, 0x00, + 0xec, 0xff, 0xa0, 0x7a, 0x14, 0x7b, 0xe4, 0x6b, + 0x63, 0xe8, 0x66, 0x86, 0x34, 0xfd, 0x74, 0x44, + 0xa2, 0x50, 0x97, 0x0d, 0xdc, 0xc3, 0x84, 0xf8, + 0x71, 0x02, 0x31, 0x95, 0xed, 0x54 +}; +static const u8 key63[] __initconst = { + 0x7d, 0x64, 0xb4, 0x12, 0x81, 0xe4, 0xe6, 0x8f, + 0xcc, 0xe7, 0xd1, 0x1f, 0x70, 0x20, 0xfd, 0xb8, + 0x3a, 0x7d, 0xa6, 0x53, 0x65, 0x30, 0x5d, 0xe3, + 0x1a, 0x44, 0xbe, 0x62, 0xed, 0x90, 0xc4, 0xd1 +}; +enum { nonce63 = 0xe8e849596c942276ULL }; + +static const u8 input64[] __initconst = { + 0x84, 0xf8, 0xda, 0x87, 0x23, 0x39, 0x60, 0xcf, + 0xc5, 0x50, 0x7e, 0xc5, 0x47, 0x29, 0x7c, 0x05, + 0xc2, 0xb4, 0xf4, 0xb2, 0xec, 0x5d, 0x48, 0x36, + 0xbf, 0xfc, 0x06, 0x8c, 0xf2, 0x0e, 0x88, 0xe7, + 0xc9, 0xc5, 0xa4, 0xa2, 0x83, 0x20, 0xa1, 0x6f, + 0x37, 0xe5, 0x2d, 0xa1, 0x72, 0xa1, 0x19, 0xef, + 0x05, 0x42, 0x08, 0xf2, 0x57, 0x47, 0x31, 0x1e, + 0x17, 0x76, 0x13, 0xd3, 0xcc, 0x75, 0x2c +}; +static const u8 output64[] __initconst = { + 0xcb, 0xec, 0x90, 0x88, 0xeb, 0x31, 0x69, 0x20, + 0xa6, 0xdc, 0xff, 0x76, 0x98, 0xb0, 0x24, 0x49, + 0x7b, 0x20, 0xd9, 0xd1, 0x1b, 0xe3, 0x61, 0xdc, + 0xcf, 0x51, 0xf6, 0x70, 0x72, 0x33, 0x28, 0x94, + 0xac, 0x73, 0x18, 0xcf, 0x93, 0xfd, 0xca, 0x08, + 0x0d, 0xa2, 0xb9, 0x57, 0x1e, 0x51, 0xb6, 0x07, + 0x5c, 0xc1, 0x13, 0x64, 0x1d, 0x18, 0x6f, 0xe6, + 0x0b, 0xb7, 0x14, 0x03, 0x43, 0xb6, 0xaf +}; +static const u8 key64[] __initconst = { + 0xbf, 0x82, 0x65, 0xe4, 0x50, 0xf9, 0x5e, 0xea, + 0x28, 0x91, 0xd1, 0xd2, 0x17, 0x7c, 0x13, 0x7e, + 0xf5, 0xd5, 0x6b, 0x06, 0x1c, 0x20, 0xc2, 0x82, + 0xa1, 0x7a, 0xa2, 0x14, 0xa1, 0xb0, 0x54, 0x58 +}; +enum { nonce64 = 0xe57c5095aa5723c9ULL }; + +static const u8 input65[] __initconst = { + 0x1c, 0xfb, 0xd3, 0x3f, 0x85, 0xd7, 0xba, 0x7b, + 0xae, 0xb1, 0xa5, 0xd2, 0xe5, 0x40, 0xce, 0x4d, + 0x3e, 0xab, 0x17, 0x9d, 0x7d, 0x9f, 0x03, 0x98, + 0x3f, 0x9f, 0xc8, 0xdd, 0x36, 0x17, 0x43, 0x5c, + 0x34, 0xd1, 0x23, 0xe0, 0x77, 0xbf, 0x35, 0x5d, + 0x8f, 0xb1, 0xcb, 0x82, 0xbb, 0x39, 0x69, 0xd8, + 0x90, 0x45, 0x37, 0xfd, 0x98, 0x25, 0xf7, 0x5b, + 0xce, 0x06, 0x43, 0xba, 0x61, 0xa8, 0x47, 0xb9 +}; +static const u8 output65[] __initconst = { + 0x73, 0xa5, 0x68, 0xab, 0x8b, 0xa5, 0xc3, 0x7e, + 0x74, 0xf8, 0x9d, 0xf5, 0x93, 0x6e, 0xf2, 0x71, + 0x6d, 0xde, 0x82, 0xc5, 0x40, 0xa0, 0x46, 0xb3, + 0x9a, 0x78, 0xa8, 0xf7, 0xdf, 0xb1, 0xc3, 0xdd, + 0x8d, 0x90, 0x00, 0x68, 0x21, 0x48, 0xe8, 0xba, + 0x56, 0x9f, 0x8f, 0xe7, 0xa4, 0x4d, 0x36, 0x55, + 0xd0, 0x34, 0x99, 0xa6, 0x1c, 0x4c, 0xc1, 0xe2, + 0x65, 0x98, 0x14, 0x8e, 0x6a, 0x05, 0xb1, 0x2b +}; +static const u8 key65[] __initconst = { + 0xbd, 0x5c, 0x8a, 0xb0, 0x11, 0x29, 0xf3, 0x00, + 0x7a, 0x78, 0x32, 0x63, 0x34, 0x00, 0xe6, 0x7d, + 0x30, 0x54, 0xde, 0x37, 0xda, 0xc2, 0xc4, 0x3d, + 0x92, 0x6b, 0x4c, 0xc2, 0x92, 0xe9, 0x9e, 0x2a +}; +enum { nonce65 = 0xf654a3031de746f2ULL }; + +static const u8 input66[] __initconst = { + 0x4b, 0x27, 0x30, 0x8f, 0x28, 0xd8, 0x60, 0x46, + 0x39, 0x06, 0x49, 0xea, 0x1b, 0x71, 0x26, 0xe0, + 0x99, 0x2b, 0xd4, 0x8f, 0x64, 0x64, 0xcd, 0xac, + 0x1d, 0x78, 0x88, 0x90, 0xe1, 0x5c, 0x24, 0x4b, + 0xdc, 0x2d, 0xb7, 0xee, 0x3a, 0xe6, 0x86, 0x2c, + 0x21, 0xe4, 0x2b, 0xfc, 0xe8, 0x19, 0xca, 0x65, + 0xe7, 0xdd, 0x6f, 0x52, 0xb3, 0x11, 0xe1, 0xe2, + 0xbf, 0xe8, 0x70, 0xe3, 0x0d, 0x45, 0xb8, 0xa5, + 0x20, 0xb7, 0xb5, 0xaf, 0xff, 0x08, 0xcf, 0x23, + 0x65, 0xdf, 0x8d, 0xc3, 0x31, 0xf3, 0x1e, 0x6a, + 0x58, 0x8d, 0xcc, 0x45, 0x16, 0x86, 0x1f, 0x31, + 0x5c, 0x27, 0xcd, 0xc8, 0x6b, 0x19, 0x1e, 0xec, + 0x44, 0x75, 0x63, 0x97, 0xfd, 0x79, 0xf6, 0x62, + 0xc5, 0xba, 0x17, 0xc7, 0xab, 0x8f, 0xbb, 0xed, + 0x85, 0x2a, 0x98, 0x79, 0x21, 0xec, 0x6e, 0x4d, + 0xdc, 0xfa, 0x72, 0x52, 0xba, 0xc8, 0x4c +}; +static const u8 output66[] __initconst = { + 0x76, 0x5b, 0x2c, 0xa7, 0x62, 0xb9, 0x08, 0x4a, + 0xc6, 0x4a, 0x92, 0xc3, 0xbb, 0x10, 0xb3, 0xee, + 0xff, 0xb9, 0x07, 0xc7, 0x27, 0xcb, 0x1e, 0xcf, + 0x58, 0x6f, 0xa1, 0x64, 0xe8, 0xf1, 0x4e, 0xe1, + 0xef, 0x18, 0x96, 0xab, 0x97, 0x28, 0xd1, 0x7c, + 0x71, 0x6c, 0xd1, 0xe2, 0xfa, 0xd9, 0x75, 0xcb, + 0xeb, 0xea, 0x0c, 0x86, 0x82, 0xd8, 0xf4, 0xcc, + 0xea, 0xa3, 0x00, 0xfa, 0x82, 0xd2, 0xcd, 0xcb, + 0xdb, 0x63, 0x28, 0xe2, 0x82, 0xe9, 0x01, 0xed, + 0x31, 0xe6, 0x71, 0x45, 0x08, 0x89, 0x8a, 0x23, + 0xa8, 0xb5, 0xc2, 0xe2, 0x9f, 0xe9, 0xb8, 0x9a, + 0xc4, 0x79, 0x6d, 0x71, 0x52, 0x61, 0x74, 0x6c, + 0x1b, 0xd7, 0x65, 0x6d, 0x03, 0xc4, 0x1a, 0xc0, + 0x50, 0xba, 0xd6, 0xc9, 0x43, 0x50, 0xbe, 0x09, + 0x09, 0x8a, 0xdb, 0xaa, 0x76, 0x4e, 0x3b, 0x61, + 0x3c, 0x7c, 0x44, 0xe7, 0xdb, 0x10, 0xa7 +}; +static const u8 key66[] __initconst = { + 0x88, 0xdf, 0xca, 0x68, 0xaf, 0x4f, 0xb3, 0xfd, + 0x6e, 0xa7, 0x95, 0x35, 0x8a, 0xe8, 0x37, 0xe8, + 0xc8, 0x55, 0xa2, 0x2a, 0x6d, 0x77, 0xf8, 0x93, + 0x7a, 0x41, 0xf3, 0x7b, 0x95, 0xdf, 0x89, 0xf5 +}; +enum { nonce66 = 0x1024b4fdd415cf82ULL }; + +static const u8 input67[] __initconst = { + 0xd4, 0x2e, 0xfa, 0x92, 0xe9, 0x29, 0x68, 0xb7, + 0x54, 0x2c, 0xf7, 0xa4, 0x2d, 0xb7, 0x50, 0xb5, + 0xc5, 0xb2, 0x9d, 0x17, 0x5e, 0x0a, 0xca, 0x37, + 0xbf, 0x60, 0xae, 0xd2, 0x98, 0xe9, 0xfa, 0x59, + 0x67, 0x62, 0xe6, 0x43, 0x0c, 0x77, 0x80, 0x82, + 0x33, 0x61, 0xa3, 0xff, 0xc1, 0xa0, 0x8f, 0x56, + 0xbc, 0xec, 0x65, 0x43, 0x88, 0xa5, 0xff, 0x51, + 0x64, 0x30, 0xee, 0x34, 0xb7, 0x5c, 0x28, 0x68, + 0xc3, 0x52, 0xd2, 0xac, 0x78, 0x2a, 0xa6, 0x10, + 0xb8, 0xb2, 0x4c, 0x80, 0x4f, 0x99, 0xb2, 0x36, + 0x94, 0x8f, 0x66, 0xcb, 0xa1, 0x91, 0xed, 0x06, + 0x42, 0x6d, 0xc1, 0xae, 0x55, 0x93, 0xdd, 0x93, + 0x9e, 0x88, 0x34, 0x7f, 0x98, 0xeb, 0xbe, 0x61, + 0xf9, 0xa9, 0x0f, 0xd9, 0xc4, 0x87, 0xd5, 0xef, + 0xcc, 0x71, 0x8c, 0x0e, 0xce, 0xad, 0x02, 0xcf, + 0xa2, 0x61, 0xdf, 0xb1, 0xfe, 0x3b, 0xdc, 0xc0, + 0x58, 0xb5, 0x71, 0xa1, 0x83, 0xc9, 0xb4, 0xaf, + 0x9d, 0x54, 0x12, 0xcd, 0xea, 0x06, 0xd6, 0x4e, + 0xe5, 0x27, 0x0c, 0xc3, 0xbb, 0xa8, 0x0a, 0x81, + 0x75, 0xc3, 0xc9, 0xd4, 0x35, 0x3e, 0x53, 0x9f, + 0xaa, 0x20, 0xc0, 0x68, 0x39, 0x2c, 0x96, 0x39, + 0x53, 0x81, 0xda, 0x07, 0x0f, 0x44, 0xa5, 0x47, + 0x0e, 0xb3, 0x87, 0x0d, 0x1b, 0xc1, 0xe5, 0x41, + 0x35, 0x12, 0x58, 0x96, 0x69, 0x8a, 0x1a, 0xa3, + 0x9d, 0x3d, 0xd4, 0xb1, 0x8e, 0x1f, 0x96, 0x87, + 0xda, 0xd3, 0x19, 0xe2, 0xb1, 0x3a, 0x19, 0x74, + 0xa0, 0x00, 0x9f, 0x4d, 0xbc, 0xcb, 0x0c, 0xe9, + 0xec, 0x10, 0xdf, 0x2a, 0x88, 0xdc, 0x30, 0x51, + 0x46, 0x56, 0x53, 0x98, 0x6a, 0x26, 0x14, 0x05, + 0x54, 0x81, 0x55, 0x0b, 0x3c, 0x85, 0xdd, 0x33, + 0x81, 0x11, 0x29, 0x82, 0x46, 0x35, 0xe1, 0xdb, + 0x59, 0x7b +}; +static const u8 output67[] __initconst = { + 0x64, 0x6c, 0xda, 0x7f, 0xd4, 0xa9, 0x2a, 0x5e, + 0x22, 0xae, 0x8d, 0x67, 0xdb, 0xee, 0xfd, 0xd0, + 0x44, 0x80, 0x17, 0xb2, 0xe3, 0x87, 0xad, 0x57, + 0x15, 0xcb, 0x88, 0x64, 0xc0, 0xf1, 0x49, 0x3d, + 0xfa, 0xbe, 0xa8, 0x9f, 0x12, 0xc3, 0x57, 0x56, + 0x70, 0xa5, 0xc5, 0x6b, 0xf1, 0xab, 0xd5, 0xde, + 0x77, 0x92, 0x6a, 0x56, 0x03, 0xf5, 0x21, 0x0d, + 0xb6, 0xc4, 0xcc, 0x62, 0x44, 0x3f, 0xb1, 0xc1, + 0x61, 0x41, 0x90, 0xb2, 0xd5, 0xb8, 0xf3, 0x57, + 0xfb, 0xc2, 0x6b, 0x25, 0x58, 0xc8, 0x45, 0x20, + 0x72, 0x29, 0x6f, 0x9d, 0xb5, 0x81, 0x4d, 0x2b, + 0xb2, 0x89, 0x9e, 0x91, 0x53, 0x97, 0x1c, 0xd9, + 0x3d, 0x79, 0xdc, 0x14, 0xae, 0x01, 0x73, 0x75, + 0xf0, 0xca, 0xd5, 0xab, 0x62, 0x5c, 0x7a, 0x7d, + 0x3f, 0xfe, 0x22, 0x7d, 0xee, 0xe2, 0xcb, 0x76, + 0x55, 0xec, 0x06, 0xdd, 0x41, 0x47, 0x18, 0x62, + 0x1d, 0x57, 0xd0, 0xd6, 0xb6, 0x0f, 0x4b, 0xfc, + 0x79, 0x19, 0xf4, 0xd6, 0x37, 0x86, 0x18, 0x1f, + 0x98, 0x0d, 0x9e, 0x15, 0x2d, 0xb6, 0x9a, 0x8a, + 0x8c, 0x80, 0x22, 0x2f, 0x82, 0xc4, 0xc7, 0x36, + 0xfa, 0xfa, 0x07, 0xbd, 0xc2, 0x2a, 0xe2, 0xea, + 0x93, 0xc8, 0xb2, 0x90, 0x33, 0xf2, 0xee, 0x4b, + 0x1b, 0xf4, 0x37, 0x92, 0x13, 0xbb, 0xe2, 0xce, + 0xe3, 0x03, 0xcf, 0x07, 0x94, 0xab, 0x9a, 0xc9, + 0xff, 0x83, 0x69, 0x3a, 0xda, 0x2c, 0xd0, 0x47, + 0x3d, 0x6c, 0x1a, 0x60, 0x68, 0x47, 0xb9, 0x36, + 0x52, 0xdd, 0x16, 0xef, 0x6c, 0xbf, 0x54, 0x11, + 0x72, 0x62, 0xce, 0x8c, 0x9d, 0x90, 0xa0, 0x25, + 0x06, 0x92, 0x3e, 0x12, 0x7e, 0x1a, 0x1d, 0xe5, + 0xa2, 0x71, 0xce, 0x1c, 0x4c, 0x6a, 0x7c, 0xdc, + 0x3d, 0xe3, 0x6e, 0x48, 0x9d, 0xb3, 0x64, 0x7d, + 0x78, 0x40 +}; +static const u8 key67[] __initconst = { + 0xa9, 0x20, 0x75, 0x89, 0x7e, 0x37, 0x85, 0x48, + 0xa3, 0xfb, 0x7b, 0xe8, 0x30, 0xa7, 0xe3, 0x6e, + 0xa6, 0xc1, 0x71, 0x17, 0xc1, 0x6c, 0x9b, 0xc2, + 0xde, 0xf0, 0xa7, 0x19, 0xec, 0xce, 0xc6, 0x53 +}; +enum { nonce67 = 0x4adc4d1f968c8a10ULL }; + +static const u8 input68[] __initconst = { + 0x99, 0xae, 0x72, 0xfb, 0x16, 0xe1, 0xf1, 0x59, + 0x43, 0x15, 0x4e, 0x33, 0xa0, 0x95, 0xe7, 0x6c, + 0x74, 0x24, 0x31, 0xca, 0x3b, 0x2e, 0xeb, 0xd7, + 0x11, 0xd8, 0xe0, 0x56, 0x92, 0x91, 0x61, 0x57, + 0xe2, 0x82, 0x9f, 0x8f, 0x37, 0xf5, 0x3d, 0x24, + 0x92, 0x9d, 0x87, 0x00, 0x8d, 0x89, 0xe0, 0x25, + 0x8b, 0xe4, 0x20, 0x5b, 0x8a, 0x26, 0x2c, 0x61, + 0x78, 0xb0, 0xa6, 0x3e, 0x82, 0x18, 0xcf, 0xdc, + 0x2d, 0x24, 0xdd, 0x81, 0x42, 0xc4, 0x95, 0xf0, + 0x48, 0x60, 0x71, 0xe3, 0xe3, 0xac, 0xec, 0xbe, + 0x98, 0x6b, 0x0c, 0xb5, 0x6a, 0xa9, 0xc8, 0x79, + 0x23, 0x2e, 0x38, 0x0b, 0x72, 0x88, 0x8c, 0xe7, + 0x71, 0x8b, 0x36, 0xe3, 0x58, 0x3d, 0x9c, 0xa0, + 0xa2, 0xea, 0xcf, 0x0c, 0x6a, 0x6c, 0x64, 0xdf, + 0x97, 0x21, 0x8f, 0x93, 0xfb, 0xba, 0xf3, 0x5a, + 0xd7, 0x8f, 0xa6, 0x37, 0x15, 0x50, 0x43, 0x02, + 0x46, 0x7f, 0x93, 0x46, 0x86, 0x31, 0xe2, 0xaa, + 0x24, 0xa8, 0x26, 0xae, 0xe6, 0xc0, 0x05, 0x73, + 0x0b, 0x4f, 0x7e, 0xed, 0x65, 0xeb, 0x56, 0x1e, + 0xb6, 0xb3, 0x0b, 0xc3, 0x0e, 0x31, 0x95, 0xa9, + 0x18, 0x4d, 0xaf, 0x38, 0xd7, 0xec, 0xc6, 0x44, + 0x72, 0x77, 0x4e, 0x25, 0x4b, 0x25, 0xdd, 0x1e, + 0x8c, 0xa2, 0xdf, 0xf6, 0x2a, 0x97, 0x1a, 0x88, + 0x2c, 0x8a, 0x5d, 0xfe, 0xe8, 0xfb, 0x35, 0xe8, + 0x0f, 0x2b, 0x7a, 0x18, 0x69, 0x43, 0x31, 0x1d, + 0x38, 0x6a, 0x62, 0x95, 0x0f, 0x20, 0x4b, 0xbb, + 0x97, 0x3c, 0xe0, 0x64, 0x2f, 0x52, 0xc9, 0x2d, + 0x4d, 0x9d, 0x54, 0x04, 0x3d, 0xc9, 0xea, 0xeb, + 0xd0, 0x86, 0x52, 0xff, 0x42, 0xe1, 0x0d, 0x7a, + 0xad, 0x88, 0xf9, 0x9b, 0x1e, 0x5e, 0x12, 0x27, + 0x95, 0x3e, 0x0c, 0x2c, 0x13, 0x00, 0x6f, 0x8e, + 0x93, 0x69, 0x0e, 0x01, 0x8c, 0xc1, 0xfd, 0xb3 +}; +static const u8 output68[] __initconst = { + 0x26, 0x3e, 0xf2, 0xb1, 0xf5, 0xef, 0x81, 0xa4, + 0xb7, 0x42, 0xd4, 0x26, 0x18, 0x4b, 0xdd, 0x6a, + 0x47, 0x15, 0xcb, 0x0e, 0x57, 0xdb, 0xa7, 0x29, + 0x7e, 0x7b, 0x3f, 0x47, 0x89, 0x57, 0xab, 0xea, + 0x14, 0x7b, 0xcf, 0x37, 0xdb, 0x1c, 0xe1, 0x11, + 0x77, 0xae, 0x2e, 0x4c, 0xd2, 0x08, 0x3f, 0xa6, + 0x62, 0x86, 0xa6, 0xb2, 0x07, 0xd5, 0x3f, 0x9b, + 0xdc, 0xc8, 0x50, 0x4b, 0x7b, 0xb9, 0x06, 0xe6, + 0xeb, 0xac, 0x98, 0x8c, 0x36, 0x0c, 0x1e, 0xb2, + 0xc8, 0xfb, 0x24, 0x60, 0x2c, 0x08, 0x17, 0x26, + 0x5b, 0xc8, 0xc2, 0xdf, 0x9c, 0x73, 0x67, 0x4a, + 0xdb, 0xcf, 0xd5, 0x2c, 0x2b, 0xca, 0x24, 0xcc, + 0xdb, 0xc9, 0xa8, 0xf2, 0x5d, 0x67, 0xdf, 0x5c, + 0x62, 0x0b, 0x58, 0xc0, 0x83, 0xde, 0x8b, 0xf6, + 0x15, 0x0a, 0xd6, 0x32, 0xd8, 0xf5, 0xf2, 0x5f, + 0x33, 0xce, 0x7e, 0xab, 0x76, 0xcd, 0x14, 0x91, + 0xd8, 0x41, 0x90, 0x93, 0xa1, 0xaf, 0xf3, 0x45, + 0x6c, 0x1b, 0x25, 0xbd, 0x48, 0x51, 0x6d, 0x15, + 0x47, 0xe6, 0x23, 0x50, 0x32, 0x69, 0x1e, 0xb5, + 0x94, 0xd3, 0x97, 0xba, 0xd7, 0x37, 0x4a, 0xba, + 0xb9, 0xcd, 0xfb, 0x96, 0x9a, 0x90, 0xe0, 0x37, + 0xf8, 0xdf, 0x91, 0x6c, 0x62, 0x13, 0x19, 0x21, + 0x4b, 0xa9, 0xf1, 0x12, 0x66, 0xe2, 0x74, 0xd7, + 0x81, 0xa0, 0x74, 0x8d, 0x7e, 0x7e, 0xc9, 0xb1, + 0x69, 0x8f, 0xed, 0xb3, 0xf6, 0x97, 0xcd, 0x72, + 0x78, 0x93, 0xd3, 0x54, 0x6b, 0x43, 0xac, 0x29, + 0xb4, 0xbc, 0x7d, 0xa4, 0x26, 0x4b, 0x7b, 0xab, + 0xd6, 0x67, 0x22, 0xff, 0x03, 0x92, 0xb6, 0xd4, + 0x96, 0x94, 0x5a, 0xe5, 0x02, 0x35, 0x77, 0xfa, + 0x3f, 0x54, 0x1d, 0xdd, 0x35, 0x39, 0xfe, 0x03, + 0xdd, 0x8e, 0x3c, 0x8c, 0xc2, 0x69, 0x2a, 0xb1, + 0xb7, 0xb3, 0xa1, 0x89, 0x84, 0xea, 0x16, 0xe2 +}; +static const u8 key68[] __initconst = { + 0xd2, 0x49, 0x7f, 0xd7, 0x49, 0x66, 0x0d, 0xb3, + 0x5a, 0x7e, 0x3c, 0xfc, 0x37, 0x83, 0x0e, 0xf7, + 0x96, 0xd8, 0xd6, 0x33, 0x79, 0x2b, 0x84, 0x53, + 0x06, 0xbc, 0x6c, 0x0a, 0x55, 0x84, 0xfe, 0xab +}; +enum { nonce68 = 0x6a6df7ff0a20de06ULL }; + +static const u8 input69[] __initconst = { + 0xf9, 0x18, 0x4c, 0xd2, 0x3f, 0xf7, 0x22, 0xd9, + 0x58, 0xb6, 0x3b, 0x38, 0x69, 0x79, 0xf4, 0x71, + 0x5f, 0x38, 0x52, 0x1f, 0x17, 0x6f, 0x6f, 0xd9, + 0x09, 0x2b, 0xfb, 0x67, 0xdc, 0xc9, 0xe8, 0x4a, + 0x70, 0x9f, 0x2e, 0x3c, 0x06, 0xe5, 0x12, 0x20, + 0x25, 0x29, 0xd0, 0xdc, 0x81, 0xc5, 0xc6, 0x0f, + 0xd2, 0xa8, 0x81, 0x15, 0x98, 0xb2, 0x71, 0x5a, + 0x9a, 0xe9, 0xfb, 0xaf, 0x0e, 0x5f, 0x8a, 0xf3, + 0x16, 0x4a, 0x47, 0xf2, 0x5c, 0xbf, 0xda, 0x52, + 0x9a, 0xa6, 0x36, 0xfd, 0xc6, 0xf7, 0x66, 0x00, + 0xcc, 0x6c, 0xd4, 0xb3, 0x07, 0x6d, 0xeb, 0xfe, + 0x92, 0x71, 0x25, 0xd0, 0xcf, 0x9c, 0xe8, 0x65, + 0x45, 0x10, 0xcf, 0x62, 0x74, 0x7d, 0xf2, 0x1b, + 0x57, 0xa0, 0xf1, 0x6b, 0xa4, 0xd5, 0xfa, 0x12, + 0x27, 0x5a, 0xf7, 0x99, 0xfc, 0xca, 0xf3, 0xb8, + 0x2c, 0x8b, 0xba, 0x28, 0x74, 0xde, 0x8f, 0x78, + 0xa2, 0x8c, 0xaf, 0x89, 0x4b, 0x05, 0xe2, 0xf3, + 0xf8, 0xd2, 0xef, 0xac, 0xa4, 0xc4, 0xe2, 0xe2, + 0x36, 0xbb, 0x5e, 0xae, 0xe6, 0x87, 0x3d, 0x88, + 0x9f, 0xb8, 0x11, 0xbb, 0xcf, 0x57, 0xce, 0xd0, + 0xba, 0x62, 0xf4, 0xf8, 0x9b, 0x95, 0x04, 0xc9, + 0xcf, 0x01, 0xe9, 0xf1, 0xc8, 0xc6, 0x22, 0xa4, + 0xf2, 0x8b, 0x2f, 0x24, 0x0a, 0xf5, 0x6e, 0xb7, + 0xd4, 0x2c, 0xb6, 0xf7, 0x5c, 0x97, 0x61, 0x0b, + 0xd9, 0xb5, 0x06, 0xcd, 0xed, 0x3e, 0x1f, 0xc5, + 0xb2, 0x6c, 0xa3, 0xea, 0xb8, 0xad, 0xa6, 0x42, + 0x88, 0x7a, 0x52, 0xd5, 0x64, 0xba, 0xb5, 0x20, + 0x10, 0xa0, 0x0f, 0x0d, 0xea, 0xef, 0x5a, 0x9b, + 0x27, 0xb8, 0xca, 0x20, 0x19, 0x6d, 0xa8, 0xc4, + 0x46, 0x04, 0xb3, 0xe8, 0xf8, 0x66, 0x1b, 0x0a, + 0xce, 0x76, 0x5d, 0x59, 0x58, 0x05, 0xee, 0x3e, + 0x3c, 0x86, 0x5b, 0x49, 0x1c, 0x72, 0x18, 0x01, + 0x62, 0x92, 0x0f, 0x3e, 0xd1, 0x57, 0x5e, 0x20, + 0x7b, 0xfb, 0x4d, 0x3c, 0xc5, 0x35, 0x43, 0x2f, + 0xb0, 0xc5, 0x7c, 0xe4, 0xa2, 0x84, 0x13, 0x77 +}; +static const u8 output69[] __initconst = { + 0xbb, 0x4a, 0x7f, 0x7c, 0xd5, 0x2f, 0x89, 0x06, + 0xec, 0x20, 0xf1, 0x9a, 0x11, 0x09, 0x14, 0x2e, + 0x17, 0x50, 0xf9, 0xd5, 0xf5, 0x48, 0x7c, 0x7a, + 0x55, 0xc0, 0x57, 0x03, 0xe3, 0xc4, 0xb2, 0xb7, + 0x18, 0x47, 0x95, 0xde, 0xaf, 0x80, 0x06, 0x3c, + 0x5a, 0xf2, 0xc3, 0x53, 0xe3, 0x29, 0x92, 0xf8, + 0xff, 0x64, 0x85, 0xb9, 0xf7, 0xd3, 0x80, 0xd2, + 0x0c, 0x5d, 0x7b, 0x57, 0x0c, 0x51, 0x79, 0x86, + 0xf3, 0x20, 0xd2, 0xb8, 0x6e, 0x0c, 0x5a, 0xce, + 0xeb, 0x88, 0x02, 0x8b, 0x82, 0x1b, 0x7f, 0xf5, + 0xde, 0x7f, 0x48, 0x48, 0xdf, 0xa0, 0x55, 0xc6, + 0x0c, 0x22, 0xa1, 0x80, 0x8d, 0x3b, 0xcb, 0x40, + 0x2d, 0x3d, 0x0b, 0xf2, 0xe0, 0x22, 0x13, 0x99, + 0xe1, 0xa7, 0x27, 0x68, 0x31, 0xe1, 0x24, 0x5d, + 0xd2, 0xee, 0x16, 0xc1, 0xd7, 0xa8, 0x14, 0x19, + 0x23, 0x72, 0x67, 0x27, 0xdc, 0x5e, 0xb9, 0xc7, + 0xd8, 0xe3, 0x55, 0x50, 0x40, 0x98, 0x7b, 0xe7, + 0x34, 0x1c, 0x3b, 0x18, 0x14, 0xd8, 0x62, 0xc1, + 0x93, 0x84, 0xf3, 0x5b, 0xdd, 0x9e, 0x1f, 0x3b, + 0x0b, 0xbc, 0x4e, 0x5b, 0x79, 0xa3, 0xca, 0x74, + 0x2a, 0x98, 0xe8, 0x04, 0x39, 0xef, 0xc6, 0x76, + 0x6d, 0xee, 0x9f, 0x67, 0x5b, 0x59, 0x3a, 0xe5, + 0xf2, 0x3b, 0xca, 0x89, 0xe8, 0x9b, 0x03, 0x3d, + 0x11, 0xd2, 0x4a, 0x70, 0xaf, 0x88, 0xb0, 0x94, + 0x96, 0x26, 0xab, 0x3c, 0xc1, 0xb8, 0xe4, 0xe7, + 0x14, 0x61, 0x64, 0x3a, 0x61, 0x08, 0x0f, 0xa9, + 0xce, 0x64, 0xb2, 0x40, 0xf8, 0x20, 0x3a, 0xa9, + 0x31, 0xbd, 0x7e, 0x16, 0xca, 0xf5, 0x62, 0x0f, + 0x91, 0x9f, 0x8e, 0x1d, 0xa4, 0x77, 0xf3, 0x87, + 0x61, 0xe8, 0x14, 0xde, 0x18, 0x68, 0x4e, 0x9d, + 0x73, 0xcd, 0x8a, 0xe4, 0x80, 0x84, 0x23, 0xaa, + 0x9d, 0x64, 0x1c, 0x80, 0x41, 0xca, 0x82, 0x40, + 0x94, 0x55, 0xe3, 0x28, 0xa1, 0x97, 0x71, 0xba, + 0xf2, 0x2c, 0x39, 0x62, 0x29, 0x56, 0xd0, 0xff, + 0xb2, 0x82, 0x20, 0x59, 0x1f, 0xc3, 0x64, 0x57 +}; +static const u8 key69[] __initconst = { + 0x19, 0x09, 0xe9, 0x7c, 0xd9, 0x02, 0x4a, 0x0c, + 0x52, 0x25, 0xad, 0x5c, 0x2e, 0x8d, 0x86, 0x10, + 0x85, 0x2b, 0xba, 0xa4, 0x44, 0x5b, 0x39, 0x3e, + 0x18, 0xaa, 0xce, 0x0e, 0xe2, 0x69, 0x3c, 0xcf +}; +enum { nonce69 = 0xdb925a1948f0f060ULL }; + +static const u8 input70[] __initconst = { + 0x10, 0xe7, 0x83, 0xcf, 0x42, 0x9f, 0xf2, 0x41, + 0xc7, 0xe4, 0xdb, 0xf9, 0xa3, 0x02, 0x1d, 0x8d, + 0x50, 0x81, 0x2c, 0x6b, 0x92, 0xe0, 0x4e, 0xea, + 0x26, 0x83, 0x2a, 0xd0, 0x31, 0xf1, 0x23, 0xf3, + 0x0e, 0x88, 0x14, 0x31, 0xf9, 0x01, 0x63, 0x59, + 0x21, 0xd1, 0x8b, 0xdd, 0x06, 0xd0, 0xc6, 0xab, + 0x91, 0x71, 0x82, 0x4d, 0xd4, 0x62, 0x37, 0x17, + 0xf9, 0x50, 0xf9, 0xb5, 0x74, 0xce, 0x39, 0x80, + 0x80, 0x78, 0xf8, 0xdc, 0x1c, 0xdb, 0x7c, 0x3d, + 0xd4, 0x86, 0x31, 0x00, 0x75, 0x7b, 0xd1, 0x42, + 0x9f, 0x1b, 0x97, 0x88, 0x0e, 0x14, 0x0e, 0x1e, + 0x7d, 0x7b, 0xc4, 0xd2, 0xf3, 0xc1, 0x6d, 0x17, + 0x5d, 0xc4, 0x75, 0x54, 0x0f, 0x38, 0x65, 0x89, + 0xd8, 0x7d, 0xab, 0xc9, 0xa7, 0x0a, 0x21, 0x0b, + 0x37, 0x12, 0x05, 0x07, 0xb5, 0x68, 0x32, 0x32, + 0xb9, 0xf8, 0x97, 0x17, 0x03, 0xed, 0x51, 0x8f, + 0x3d, 0x5a, 0xd0, 0x12, 0x01, 0x6e, 0x2e, 0x91, + 0x1c, 0xbe, 0x6b, 0xa3, 0xcc, 0x75, 0x62, 0x06, + 0x8e, 0x65, 0xbb, 0xe2, 0x29, 0x71, 0x4b, 0x89, + 0x6a, 0x9d, 0x85, 0x8c, 0x8c, 0xdf, 0x94, 0x95, + 0x23, 0x66, 0xf8, 0x92, 0xee, 0x56, 0xeb, 0xb3, + 0xeb, 0xd2, 0x4a, 0x3b, 0x77, 0x8a, 0x6e, 0xf6, + 0xca, 0xd2, 0x34, 0x00, 0xde, 0xbe, 0x1d, 0x7a, + 0x73, 0xef, 0x2b, 0x80, 0x56, 0x16, 0x29, 0xbf, + 0x6e, 0x33, 0xed, 0x0d, 0xe2, 0x02, 0x60, 0x74, + 0xe9, 0x0a, 0xbc, 0xd1, 0xc5, 0xe8, 0x53, 0x02, + 0x79, 0x0f, 0x25, 0x0c, 0xef, 0xab, 0xd3, 0xbc, + 0xb7, 0xfc, 0xf3, 0xb0, 0x34, 0xd1, 0x07, 0xd2, + 0x5a, 0x31, 0x1f, 0xec, 0x1f, 0x87, 0xed, 0xdd, + 0x6a, 0xc1, 0xe8, 0xb3, 0x25, 0x4c, 0xc6, 0x9b, + 0x91, 0x73, 0xec, 0x06, 0x73, 0x9e, 0x57, 0x65, + 0x32, 0x75, 0x11, 0x74, 0x6e, 0xa4, 0x7d, 0x0d, + 0x74, 0x9f, 0x51, 0x10, 0x10, 0x47, 0xc9, 0x71, + 0x6e, 0x97, 0xae, 0x44, 0x41, 0xef, 0x98, 0x78, + 0xf4, 0xc5, 0xbd, 0x5e, 0x00, 0xe5, 0xfd, 0xe2, + 0xbe, 0x8c, 0xc2, 0xae, 0xc2, 0xee, 0x59, 0xf6, + 0xcb, 0x20, 0x54, 0x84, 0xc3, 0x31, 0x7e, 0x67, + 0x71, 0xb6, 0x76, 0xbe, 0x81, 0x8f, 0x82, 0xad, + 0x01, 0x8f, 0xc4, 0x00, 0x04, 0x3d, 0x8d, 0x34, + 0xaa, 0xea, 0xc0, 0xea, 0x91, 0x42, 0xb6, 0xb8, + 0x43, 0xf3, 0x17, 0xb2, 0x73, 0x64, 0x82, 0x97, + 0xd5, 0xc9, 0x07, 0x77, 0xb1, 0x26, 0xe2, 0x00, + 0x6a, 0xae, 0x70, 0x0b, 0xbe, 0xe6, 0xb8, 0x42, + 0x81, 0x55, 0xf7, 0xb8, 0x96, 0x41, 0x9d, 0xd4, + 0x2c, 0x27, 0x00, 0xcc, 0x91, 0x28, 0x22, 0xa4, + 0x7b, 0x42, 0x51, 0x9e, 0xd6, 0xec, 0xf3, 0x6b, + 0x00, 0xff, 0x5c, 0xa2, 0xac, 0x47, 0x33, 0x2d, + 0xf8, 0x11, 0x65, 0x5f, 0x4d, 0x79, 0x8b, 0x4f, + 0xad, 0xf0, 0x9d, 0xcd, 0xb9, 0x7b, 0x08, 0xf7, + 0x32, 0x51, 0xfa, 0x39, 0xaa, 0x78, 0x05, 0xb1, + 0xf3, 0x5d, 0xe8, 0x7c, 0x8e, 0x4f, 0xa2, 0xe0, + 0x98, 0x0c, 0xb2, 0xa7, 0xf0, 0x35, 0x8e, 0x70, + 0x7c, 0x82, 0xf3, 0x1b, 0x26, 0x28, 0x12, 0xe5, + 0x23, 0x57, 0xe4, 0xb4, 0x9b, 0x00, 0x39, 0x97, + 0xef, 0x7c, 0x46, 0x9b, 0x34, 0x6b, 0xe7, 0x0e, + 0xa3, 0x2a, 0x18, 0x11, 0x64, 0xc6, 0x7c, 0x8b, + 0x06, 0x02, 0xf5, 0x69, 0x76, 0xf9, 0xaa, 0x09, + 0x5f, 0x68, 0xf8, 0x4a, 0x79, 0x58, 0xec, 0x37, + 0xcf, 0x3a, 0xcc, 0x97, 0x70, 0x1d, 0x3e, 0x52, + 0x18, 0x0a, 0xad, 0x28, 0x5b, 0x3b, 0xe9, 0x03, + 0x84, 0xe9, 0x68, 0x50, 0xce, 0xc4, 0xbc, 0x3e, + 0x21, 0xad, 0x63, 0xfe, 0xc6, 0xfd, 0x6e, 0x69, + 0x84, 0xa9, 0x30, 0xb1, 0x7a, 0xc4, 0x31, 0x10, + 0xc1, 0x1f, 0x6e, 0xeb, 0xa5, 0xa6, 0x01 +}; +static const u8 output70[] __initconst = { + 0x0f, 0x93, 0x2a, 0x20, 0xb3, 0x87, 0x2d, 0xce, + 0xd1, 0x3b, 0x30, 0xfd, 0x06, 0x6d, 0x0a, 0xaa, + 0x3e, 0xc4, 0x29, 0x02, 0x8a, 0xde, 0xa6, 0x4b, + 0x45, 0x1b, 0x4f, 0x25, 0x59, 0xd5, 0x56, 0x6a, + 0x3b, 0x37, 0xbd, 0x3e, 0x47, 0x12, 0x2c, 0x4e, + 0x60, 0x5f, 0x05, 0x75, 0x61, 0x23, 0x05, 0x74, + 0xcb, 0xfc, 0x5a, 0xb3, 0xac, 0x5c, 0x3d, 0xab, + 0x52, 0x5f, 0x05, 0xbc, 0x57, 0xc0, 0x7e, 0xcf, + 0x34, 0x5d, 0x7f, 0x41, 0xa3, 0x17, 0x78, 0xd5, + 0x9f, 0xec, 0x0f, 0x1e, 0xf9, 0xfe, 0xa3, 0xbd, + 0x28, 0xb0, 0xba, 0x4d, 0x84, 0xdb, 0xae, 0x8f, + 0x1d, 0x98, 0xb7, 0xdc, 0xf9, 0xad, 0x55, 0x9c, + 0x89, 0xfe, 0x9b, 0x9c, 0xa9, 0x89, 0xf6, 0x97, + 0x9c, 0x3f, 0x09, 0x3e, 0xc6, 0x02, 0xc2, 0x55, + 0x58, 0x09, 0x54, 0x66, 0xe4, 0x36, 0x81, 0x35, + 0xca, 0x88, 0x17, 0x89, 0x80, 0x24, 0x2b, 0x21, + 0x89, 0xee, 0x45, 0x5a, 0xe7, 0x1f, 0xd5, 0xa5, + 0x16, 0xa4, 0xda, 0x70, 0x7e, 0xe9, 0x4f, 0x24, + 0x61, 0x97, 0xab, 0xa0, 0xe0, 0xe7, 0xb8, 0x5c, + 0x0f, 0x25, 0x17, 0x37, 0x75, 0x12, 0xb5, 0x40, + 0xde, 0x1c, 0x0d, 0x8a, 0x77, 0x62, 0x3c, 0x86, + 0xd9, 0x70, 0x2e, 0x96, 0x30, 0xd2, 0x55, 0xb3, + 0x6b, 0xc3, 0xf2, 0x9c, 0x47, 0xf3, 0x3a, 0x24, + 0x52, 0xc6, 0x38, 0xd8, 0x22, 0xb3, 0x0c, 0xfd, + 0x2f, 0xa3, 0x3c, 0xb5, 0xe8, 0x26, 0xe1, 0xa3, + 0xad, 0xb0, 0x82, 0x17, 0xc1, 0x53, 0xb8, 0x34, + 0x48, 0xee, 0x39, 0xae, 0x51, 0x43, 0xec, 0x82, + 0xce, 0x87, 0xc6, 0x76, 0xb9, 0x76, 0xd3, 0x53, + 0xfe, 0x49, 0x24, 0x7d, 0x02, 0x42, 0x2b, 0x72, + 0xfb, 0xcb, 0xd8, 0x96, 0x02, 0xc6, 0x9a, 0x20, + 0xf3, 0x5a, 0x67, 0xe8, 0x13, 0xf8, 0xb2, 0xcb, + 0xa2, 0xec, 0x18, 0x20, 0x4a, 0xb0, 0x73, 0x53, + 0x21, 0xb0, 0x77, 0x53, 0xd8, 0x76, 0xa1, 0x30, + 0x17, 0x72, 0x2e, 0x33, 0x5f, 0x33, 0x6b, 0x28, + 0xfb, 0xb0, 0xf4, 0xec, 0x8e, 0xed, 0x20, 0x7d, + 0x57, 0x8c, 0x74, 0x28, 0x64, 0x8b, 0xeb, 0x59, + 0x38, 0x3f, 0xe7, 0x83, 0x2e, 0xe5, 0x64, 0x4d, + 0x5c, 0x1f, 0xe1, 0x3b, 0xd9, 0x84, 0xdb, 0xc9, + 0xec, 0xd8, 0xc1, 0x7c, 0x1f, 0x1b, 0x68, 0x35, + 0xc6, 0x34, 0x10, 0xef, 0x19, 0xc9, 0x0a, 0xd6, + 0x43, 0x7f, 0xa6, 0xcb, 0x9d, 0xf4, 0xf0, 0x16, + 0xb1, 0xb1, 0x96, 0x64, 0xec, 0x8d, 0x22, 0x4c, + 0x4b, 0xe8, 0x1a, 0xba, 0x6f, 0xb7, 0xfc, 0xa5, + 0x69, 0x3e, 0xad, 0x78, 0x79, 0x19, 0xb5, 0x04, + 0x69, 0xe5, 0x3f, 0xff, 0x60, 0x8c, 0xda, 0x0b, + 0x7b, 0xf7, 0xe7, 0xe6, 0x29, 0x3a, 0x85, 0xba, + 0xb5, 0xb0, 0x35, 0xbd, 0x38, 0xce, 0x34, 0x5e, + 0xf2, 0xdc, 0xd1, 0x8f, 0xc3, 0x03, 0x24, 0xa2, + 0x03, 0xf7, 0x4e, 0x49, 0x5b, 0xcf, 0x6d, 0xb0, + 0xeb, 0xe3, 0x30, 0x28, 0xd5, 0x5b, 0x82, 0x5f, + 0xe4, 0x7c, 0x1e, 0xec, 0xd2, 0x39, 0xf9, 0x6f, + 0x2e, 0xb3, 0xcd, 0x01, 0xb1, 0x67, 0xaa, 0xea, + 0xaa, 0xb3, 0x63, 0xaf, 0xd9, 0xb2, 0x1f, 0xba, + 0x05, 0x20, 0xeb, 0x19, 0x32, 0xf0, 0x6c, 0x3f, + 0x40, 0xcc, 0x93, 0xb3, 0xd8, 0x25, 0xa6, 0xe4, + 0xce, 0xd7, 0x7e, 0x48, 0x99, 0x65, 0x7f, 0x86, + 0xc5, 0xd4, 0x79, 0x6b, 0xab, 0x43, 0xb8, 0x6b, + 0xf1, 0x2f, 0xea, 0x4c, 0x5e, 0xf0, 0x3b, 0xb4, + 0xb8, 0xb0, 0x94, 0x0c, 0x6b, 0xe7, 0x22, 0x93, + 0xaa, 0x01, 0xcb, 0xf1, 0x11, 0x60, 0xf6, 0x69, + 0xcf, 0x14, 0xde, 0xfb, 0x90, 0x05, 0x27, 0x0c, + 0x1a, 0x9e, 0xf0, 0xb4, 0xc6, 0xa1, 0xe8, 0xdd, + 0xd0, 0x4c, 0x25, 0x4f, 0x9c, 0xb7, 0xb1, 0xb0, + 0x21, 0xdb, 0x87, 0x09, 0x03, 0xf2, 0xb3 +}; +static const u8 key70[] __initconst = { + 0x3b, 0x5b, 0x59, 0x36, 0x44, 0xd1, 0xba, 0x71, + 0x55, 0x87, 0x4d, 0x62, 0x3d, 0xc2, 0xfc, 0xaa, + 0x3f, 0x4e, 0x1a, 0xe4, 0xca, 0x09, 0xfc, 0x6a, + 0xb2, 0xd6, 0x5d, 0x79, 0xf9, 0x1a, 0x91, 0xa7 +}; +enum { nonce70 = 0x3fd6786dd147a85ULL }; + +static const u8 input71[] __initconst = { + 0x18, 0x78, 0xd6, 0x79, 0xe4, 0x9a, 0x6c, 0x73, + 0x17, 0xd4, 0x05, 0x0f, 0x1e, 0x9f, 0xd9, 0x2b, + 0x86, 0x48, 0x7d, 0xf4, 0xd9, 0x1c, 0x76, 0xfc, + 0x8e, 0x22, 0x34, 0xe1, 0x48, 0x4a, 0x8d, 0x79, + 0xb7, 0xbb, 0x88, 0xab, 0x90, 0xde, 0xc5, 0xb4, + 0xb4, 0xe7, 0x85, 0x49, 0xda, 0x57, 0xeb, 0xc9, + 0xcd, 0x21, 0xfc, 0x45, 0x6e, 0x32, 0x67, 0xf2, + 0x4f, 0xa6, 0x54, 0xe5, 0x20, 0xed, 0xcf, 0xc6, + 0x62, 0x25, 0x8e, 0x00, 0xf8, 0x6b, 0xa2, 0x80, + 0xac, 0x88, 0xa6, 0x59, 0x27, 0x83, 0x95, 0x11, + 0x3f, 0x70, 0x5e, 0x3f, 0x11, 0xfb, 0x26, 0xbf, + 0xe1, 0x48, 0x75, 0xf9, 0x86, 0xbf, 0xa6, 0x5d, + 0x15, 0x61, 0x66, 0xbf, 0x78, 0x8f, 0x6b, 0x9b, + 0xda, 0x98, 0xb7, 0x19, 0xe2, 0xf2, 0xa3, 0x9c, + 0x7c, 0x6a, 0x9a, 0xd8, 0x3d, 0x4c, 0x2c, 0xe1, + 0x09, 0xb4, 0x28, 0x82, 0x4e, 0xab, 0x0c, 0x75, + 0x63, 0xeb, 0xbc, 0xd0, 0x71, 0xa2, 0x73, 0x85, + 0xed, 0x53, 0x7a, 0x3f, 0x68, 0x9f, 0xd0, 0xa9, + 0x00, 0x5a, 0x9e, 0x80, 0x55, 0x00, 0xe6, 0xae, + 0x0c, 0x03, 0x40, 0xed, 0xfc, 0x68, 0x4a, 0xb7, + 0x1e, 0x09, 0x65, 0x30, 0x5a, 0x3d, 0x97, 0x4d, + 0x5e, 0x51, 0x8e, 0xda, 0xc3, 0x55, 0x8c, 0xfb, + 0xcf, 0x83, 0x05, 0x35, 0x0d, 0x08, 0x1b, 0xf3, + 0x3a, 0x57, 0x96, 0xac, 0x58, 0x8b, 0xfa, 0x00, + 0x49, 0x15, 0x78, 0xd2, 0x4b, 0xed, 0xb8, 0x59, + 0x78, 0x9b, 0x7f, 0xaa, 0xfc, 0xe7, 0x46, 0xdc, + 0x7b, 0x34, 0xd0, 0x34, 0xe5, 0x10, 0xff, 0x4d, + 0x5a, 0x4d, 0x60, 0xa7, 0x16, 0x54, 0xc4, 0xfd, + 0xca, 0x5d, 0x68, 0xc7, 0x4a, 0x01, 0x8d, 0x7f, + 0x74, 0x5d, 0xff, 0xb8, 0x37, 0x15, 0x62, 0xfa, + 0x44, 0x45, 0xcf, 0x77, 0x3b, 0x1d, 0xb2, 0xd2, + 0x0d, 0x42, 0x00, 0x39, 0x68, 0x1f, 0xcc, 0x89, + 0x73, 0x5d, 0xa9, 0x2e, 0xfd, 0x58, 0x62, 0xca, + 0x35, 0x8e, 0x70, 0x70, 0xaa, 0x6e, 0x14, 0xe9, + 0xa4, 0xe2, 0x10, 0x66, 0x71, 0xdc, 0x4c, 0xfc, + 0xa9, 0xdc, 0x8f, 0x57, 0x4d, 0xc5, 0xac, 0xd7, + 0xa9, 0xf3, 0xf3, 0xa1, 0xff, 0x62, 0xa0, 0x8f, + 0xe4, 0x96, 0x3e, 0xcb, 0x9f, 0x76, 0x42, 0x39, + 0x1f, 0x24, 0xfd, 0xfd, 0x79, 0xe8, 0x27, 0xdf, + 0xa8, 0xf6, 0x33, 0x8b, 0x31, 0x59, 0x69, 0xcf, + 0x6a, 0xef, 0x89, 0x4d, 0xa7, 0xf6, 0x7e, 0x97, + 0x14, 0xbd, 0xda, 0xdd, 0xb4, 0x84, 0x04, 0x24, + 0xe0, 0x17, 0xe1, 0x0f, 0x1f, 0x8a, 0x6a, 0x71, + 0x74, 0x41, 0xdc, 0x59, 0x5c, 0x8f, 0x01, 0x25, + 0x92, 0xf0, 0x2e, 0x15, 0x62, 0x71, 0x9a, 0x9f, + 0x87, 0xdf, 0x62, 0x49, 0x7f, 0x86, 0x62, 0xfc, + 0x20, 0x84, 0xd7, 0xe3, 0x3a, 0xd9, 0x37, 0x85, + 0xb7, 0x84, 0x5a, 0xf9, 0xed, 0x21, 0x32, 0x94, + 0x3e, 0x04, 0xe7, 0x8c, 0x46, 0x76, 0x21, 0x67, + 0xf6, 0x95, 0x64, 0x92, 0xb7, 0x15, 0xf6, 0xe3, + 0x41, 0x27, 0x9d, 0xd7, 0xe3, 0x79, 0x75, 0x92, + 0xd0, 0xc1, 0xf3, 0x40, 0x92, 0x08, 0xde, 0x90, + 0x22, 0x82, 0xb2, 0x69, 0xae, 0x1a, 0x35, 0x11, + 0x89, 0xc8, 0x06, 0x82, 0x95, 0x23, 0x44, 0x08, + 0x22, 0xf2, 0x71, 0x73, 0x1b, 0x88, 0x11, 0xcf, + 0x1c, 0x7e, 0x8a, 0x2e, 0xdc, 0x79, 0x57, 0xce, + 0x1f, 0xe7, 0x6c, 0x07, 0xd8, 0x06, 0xbe, 0xec, + 0xa3, 0xcf, 0xf9, 0x68, 0xa5, 0xb8, 0xf0, 0xe3, + 0x3f, 0x01, 0x92, 0xda, 0xf1, 0xa0, 0x2d, 0x7b, + 0xab, 0x57, 0x58, 0x2a, 0xaf, 0xab, 0xbd, 0xf2, + 0xe5, 0xaf, 0x7e, 0x1f, 0x46, 0x24, 0x9e, 0x20, + 0x22, 0x0f, 0x84, 0x4c, 0xb7, 0xd8, 0x03, 0xe8, + 0x09, 0x73, 0x6c, 0xc6, 0x9b, 0x90, 0xe0, 0xdb, + 0xf2, 0x71, 0xba, 0xad, 0xb3, 0xec, 0xda, 0x7a +}; +static const u8 output71[] __initconst = { + 0x28, 0xc5, 0x9b, 0x92, 0xf9, 0x21, 0x4f, 0xbb, + 0xef, 0x3b, 0xf0, 0xf5, 0x3a, 0x6d, 0x7f, 0xd6, + 0x6a, 0x8d, 0xa1, 0x01, 0x5c, 0x62, 0x20, 0x8b, + 0x5b, 0x39, 0xd5, 0xd3, 0xc2, 0xf6, 0x9d, 0x5e, + 0xcc, 0xe1, 0xa2, 0x61, 0x16, 0xe2, 0xce, 0xe9, + 0x86, 0xd0, 0xfc, 0xce, 0x9a, 0x28, 0x27, 0xc4, + 0x0c, 0xb9, 0xaa, 0x8d, 0x48, 0xdb, 0xbf, 0x82, + 0x7d, 0xd0, 0x35, 0xc4, 0x06, 0x34, 0xb4, 0x19, + 0x51, 0x73, 0xf4, 0x7a, 0xf4, 0xfd, 0xe9, 0x1d, + 0xdc, 0x0f, 0x7e, 0xf7, 0x96, 0x03, 0xe3, 0xb1, + 0x2e, 0x22, 0x59, 0xb7, 0x6d, 0x1c, 0x97, 0x8c, + 0xd7, 0x31, 0x08, 0x26, 0x4c, 0x6d, 0xc6, 0x14, + 0xa5, 0xeb, 0x45, 0x6a, 0x88, 0xa3, 0xa2, 0x36, + 0xc4, 0x35, 0xb1, 0x5a, 0xa0, 0xad, 0xf7, 0x06, + 0x9b, 0x5d, 0xc1, 0x15, 0xc1, 0xce, 0x0a, 0xb0, + 0x57, 0x2e, 0x3f, 0x6f, 0x0d, 0x10, 0xd9, 0x11, + 0x2c, 0x9c, 0xad, 0x2d, 0xa5, 0x81, 0xfb, 0x4e, + 0x8f, 0xd5, 0x32, 0x4e, 0xaf, 0x5c, 0xc1, 0x86, + 0xde, 0x56, 0x5a, 0x33, 0x29, 0xf7, 0x67, 0xc6, + 0x37, 0x6f, 0xb2, 0x37, 0x4e, 0xd4, 0x69, 0x79, + 0xaf, 0xd5, 0x17, 0x79, 0xe0, 0xba, 0x62, 0xa3, + 0x68, 0xa4, 0x87, 0x93, 0x8d, 0x7e, 0x8f, 0xa3, + 0x9c, 0xef, 0xda, 0xe3, 0xa5, 0x1f, 0xcd, 0x30, + 0xa6, 0x55, 0xac, 0x4c, 0x69, 0x74, 0x02, 0xc7, + 0x5d, 0x95, 0x81, 0x4a, 0x68, 0x11, 0xd3, 0xa9, + 0x98, 0xb1, 0x0b, 0x0d, 0xae, 0x40, 0x86, 0x65, + 0xbf, 0xcc, 0x2d, 0xef, 0x57, 0xca, 0x1f, 0xe4, + 0x34, 0x4e, 0xa6, 0x5e, 0x82, 0x6e, 0x61, 0xad, + 0x0b, 0x3c, 0xf8, 0xeb, 0x01, 0x43, 0x7f, 0x87, + 0xa2, 0xa7, 0x6a, 0xe9, 0x62, 0x23, 0x24, 0x61, + 0xf1, 0xf7, 0x36, 0xdb, 0x10, 0xe5, 0x57, 0x72, + 0x3a, 0xc2, 0xae, 0xcc, 0x75, 0xc7, 0x80, 0x05, + 0x0a, 0x5c, 0x4c, 0x95, 0xda, 0x02, 0x01, 0x14, + 0x06, 0x6b, 0x5c, 0x65, 0xc2, 0xb8, 0x4a, 0xd6, + 0xd3, 0xb4, 0xd8, 0x12, 0x52, 0xb5, 0x60, 0xd3, + 0x8e, 0x5f, 0x5c, 0x76, 0x33, 0x7a, 0x05, 0xe5, + 0xcb, 0xef, 0x4f, 0x89, 0xf1, 0xba, 0x32, 0x6f, + 0x33, 0xcd, 0x15, 0x8d, 0xa3, 0x0c, 0x3f, 0x63, + 0x11, 0xe7, 0x0e, 0xe0, 0x00, 0x01, 0xe9, 0xe8, + 0x8e, 0x36, 0x34, 0x8d, 0x96, 0xb5, 0x03, 0xcf, + 0x55, 0x62, 0x49, 0x7a, 0x34, 0x44, 0xa5, 0xee, + 0x8c, 0x46, 0x06, 0x22, 0xab, 0x1d, 0x53, 0x9c, + 0xa1, 0xf9, 0x67, 0x18, 0x57, 0x89, 0xf9, 0xc2, + 0xd1, 0x7e, 0xbe, 0x36, 0x40, 0xcb, 0xe9, 0x04, + 0xde, 0xb1, 0x3b, 0x29, 0x52, 0xc5, 0x9a, 0xb5, + 0xa2, 0x7c, 0x7b, 0xfe, 0xe5, 0x92, 0x73, 0xea, + 0xea, 0x7b, 0xba, 0x0a, 0x8c, 0x88, 0x15, 0xe6, + 0x53, 0xbf, 0x1c, 0x33, 0xf4, 0x9b, 0x9a, 0x5e, + 0x8d, 0xae, 0x60, 0xdc, 0xcb, 0x5d, 0xfa, 0xbe, + 0x06, 0xc3, 0x3f, 0x06, 0xe7, 0x00, 0x40, 0x7b, + 0xaa, 0x94, 0xfa, 0x6d, 0x1f, 0xe4, 0xc5, 0xa9, + 0x1b, 0x5f, 0x36, 0xea, 0x5a, 0xdd, 0xa5, 0x48, + 0x6a, 0x55, 0xd2, 0x47, 0x28, 0xbf, 0x96, 0xf1, + 0x9f, 0xb6, 0x11, 0x4b, 0xd3, 0x44, 0x7d, 0x48, + 0x41, 0x61, 0xdb, 0x12, 0xd4, 0xc2, 0x59, 0x82, + 0x4c, 0x47, 0x5c, 0x04, 0xf6, 0x7b, 0xd3, 0x92, + 0x2e, 0xe8, 0x40, 0xef, 0x15, 0x32, 0x97, 0xdc, + 0x35, 0x4c, 0x6e, 0xa4, 0x97, 0xe9, 0x24, 0xde, + 0x63, 0x8b, 0xb1, 0x6b, 0x48, 0xbb, 0x46, 0x1f, + 0x84, 0xd6, 0x17, 0xb0, 0x5a, 0x4a, 0x4e, 0xd5, + 0x31, 0xd7, 0xcf, 0xa0, 0x39, 0xc6, 0x2e, 0xfc, + 0xa6, 0xa3, 0xd3, 0x0f, 0xa4, 0x28, 0xac, 0xb2, + 0xf4, 0x48, 0x8d, 0x50, 0xa5, 0x1c, 0x44, 0x5d, + 0x6e, 0x38, 0xb7, 0x2b, 0x8a, 0x45, 0xa7, 0x3d +}; +static const u8 key71[] __initconst = { + 0x8b, 0x68, 0xc4, 0xb7, 0x0d, 0x81, 0xef, 0x52, + 0x1e, 0x05, 0x96, 0x72, 0x62, 0x89, 0x27, 0x83, + 0xd0, 0xc7, 0x33, 0x6d, 0xf2, 0xcc, 0x69, 0xf9, + 0x23, 0xae, 0x99, 0xb1, 0xd1, 0x05, 0x4e, 0x54 +}; +enum { nonce71 = 0x983f03656d64b5f6ULL }; + +static const u8 input72[] __initconst = { + 0x6b, 0x09, 0xc9, 0x57, 0x3d, 0x79, 0x04, 0x8c, + 0x65, 0xad, 0x4a, 0x0f, 0xa1, 0x31, 0x3a, 0xdd, + 0x14, 0x8e, 0xe8, 0xfe, 0xbf, 0x42, 0x87, 0x98, + 0x2e, 0x8d, 0x83, 0xa3, 0xf8, 0x55, 0x3d, 0x84, + 0x1e, 0x0e, 0x05, 0x4a, 0x38, 0x9e, 0xe7, 0xfe, + 0xd0, 0x4d, 0x79, 0x74, 0x3a, 0x0b, 0x9b, 0xe1, + 0xfd, 0x51, 0x84, 0x4e, 0xb2, 0x25, 0xe4, 0x64, + 0x4c, 0xda, 0xcf, 0x46, 0xec, 0xba, 0x12, 0xeb, + 0x5a, 0x33, 0x09, 0x6e, 0x78, 0x77, 0x8f, 0x30, + 0xb1, 0x7d, 0x3f, 0x60, 0x8c, 0xf2, 0x1d, 0x8e, + 0xb4, 0x70, 0xa2, 0x90, 0x7c, 0x79, 0x1a, 0x2c, + 0xf6, 0x28, 0x79, 0x7c, 0x53, 0xc5, 0xfa, 0xcc, + 0x65, 0x9b, 0xe1, 0x51, 0xd1, 0x7f, 0x1d, 0xc4, + 0xdb, 0xd4, 0xd9, 0x04, 0x61, 0x7d, 0xbe, 0x12, + 0xfc, 0xcd, 0xaf, 0xe4, 0x0f, 0x9c, 0x20, 0xb5, + 0x22, 0x40, 0x18, 0xda, 0xe4, 0xda, 0x8c, 0x2d, + 0x84, 0xe3, 0x5f, 0x53, 0x17, 0xed, 0x78, 0xdc, + 0x2f, 0xe8, 0x31, 0xc7, 0xe6, 0x39, 0x71, 0x40, + 0xb4, 0x0f, 0xc9, 0xa9, 0x7e, 0x78, 0x87, 0xc1, + 0x05, 0x78, 0xbb, 0x01, 0xf2, 0x8f, 0x33, 0xb0, + 0x6e, 0x84, 0xcd, 0x36, 0x33, 0x5c, 0x5b, 0x8e, + 0xf1, 0xac, 0x30, 0xfe, 0x33, 0xec, 0x08, 0xf3, + 0x7e, 0xf2, 0xf0, 0x4c, 0xf2, 0xad, 0xd8, 0xc1, + 0xd4, 0x4e, 0x87, 0x06, 0xd4, 0x75, 0xe7, 0xe3, + 0x09, 0xd3, 0x4d, 0xe3, 0x21, 0x32, 0xba, 0xb4, + 0x68, 0x68, 0xcb, 0x4c, 0xa3, 0x1e, 0xb3, 0x87, + 0x7b, 0xd3, 0x0c, 0x63, 0x37, 0x71, 0x79, 0xfb, + 0x58, 0x36, 0x57, 0x0f, 0x34, 0x1d, 0xc1, 0x42, + 0x02, 0x17, 0xe7, 0xed, 0xe8, 0xe7, 0x76, 0xcb, + 0x42, 0xc4, 0x4b, 0xe2, 0xb2, 0x5e, 0x42, 0xd5, + 0xec, 0x9d, 0xc1, 0x32, 0x71, 0xe4, 0xeb, 0x10, + 0x68, 0x1a, 0x6e, 0x99, 0x8e, 0x73, 0x12, 0x1f, + 0x97, 0x0c, 0x9e, 0xcd, 0x02, 0x3e, 0x4c, 0xa0, + 0xf2, 0x8d, 0xe5, 0x44, 0xca, 0x6d, 0xfe, 0x07, + 0xe3, 0xe8, 0x9b, 0x76, 0xc1, 0x6d, 0xb7, 0x6e, + 0x0d, 0x14, 0x00, 0x6f, 0x8a, 0xfd, 0x43, 0xc6, + 0x43, 0xa5, 0x9c, 0x02, 0x47, 0x10, 0xd4, 0xb4, + 0x9b, 0x55, 0x67, 0xc8, 0x7f, 0xc1, 0x8a, 0x1f, + 0x1e, 0xd1, 0xbc, 0x99, 0x5d, 0x50, 0x4f, 0x89, + 0xf1, 0xe6, 0x5d, 0x91, 0x40, 0xdc, 0x20, 0x67, + 0x56, 0xc2, 0xef, 0xbd, 0x2c, 0xa2, 0x99, 0x38, + 0xe0, 0x45, 0xec, 0x44, 0x05, 0x52, 0x65, 0x11, + 0xfc, 0x3b, 0x19, 0xcb, 0x71, 0xc2, 0x8e, 0x0e, + 0x03, 0x2a, 0x03, 0x3b, 0x63, 0x06, 0x31, 0x9a, + 0xac, 0x53, 0x04, 0x14, 0xd4, 0x80, 0x9d, 0x6b, + 0x42, 0x7e, 0x7e, 0x4e, 0xdc, 0xc7, 0x01, 0x49, + 0x9f, 0xf5, 0x19, 0x86, 0x13, 0x28, 0x2b, 0xa6, + 0xa6, 0xbe, 0xa1, 0x7e, 0x71, 0x05, 0x00, 0xff, + 0x59, 0x2d, 0xb6, 0x63, 0xf0, 0x1e, 0x2e, 0x69, + 0x9b, 0x85, 0xf1, 0x1e, 0x8a, 0x64, 0x39, 0xab, + 0x00, 0x12, 0xe4, 0x33, 0x4b, 0xb5, 0xd8, 0xb3, + 0x6b, 0x5b, 0x8b, 0x5c, 0xd7, 0x6f, 0x23, 0xcf, + 0x3f, 0x2e, 0x5e, 0x47, 0xb9, 0xb8, 0x1f, 0xf0, + 0x1d, 0xda, 0xe7, 0x4f, 0x6e, 0xab, 0xc3, 0x36, + 0xb4, 0x74, 0x6b, 0xeb, 0xc7, 0x5d, 0x91, 0xe5, + 0xda, 0xf2, 0xc2, 0x11, 0x17, 0x48, 0xf8, 0x9c, + 0xc9, 0x8b, 0xc1, 0xa2, 0xf4, 0xcd, 0x16, 0xf8, + 0x27, 0xd9, 0x6c, 0x6f, 0xb5, 0x8f, 0x77, 0xca, + 0x1b, 0xd8, 0xef, 0x84, 0x68, 0x71, 0x53, 0xc1, + 0x43, 0x0f, 0x9f, 0x98, 0xae, 0x7e, 0x31, 0xd2, + 0x98, 0xfb, 0x20, 0xa2, 0xad, 0x00, 0x10, 0x83, + 0x00, 0x8b, 0xeb, 0x56, 0xd2, 0xc4, 0xcc, 0x7f, + 0x2f, 0x4e, 0xfa, 0x88, 0x13, 0xa4, 0x2c, 0xde, + 0x6b, 0x77, 0x86, 0x10, 0x6a, 0xab, 0x43, 0x0a, + 0x02 +}; +static const u8 output72[] __initconst = { + 0x42, 0x89, 0xa4, 0x80, 0xd2, 0xcb, 0x5f, 0x7f, + 0x2a, 0x1a, 0x23, 0x00, 0xa5, 0x6a, 0x95, 0xa3, + 0x9a, 0x41, 0xa1, 0xd0, 0x2d, 0x1e, 0xd6, 0x13, + 0x34, 0x40, 0x4e, 0x7f, 0x1a, 0xbe, 0xa0, 0x3d, + 0x33, 0x9c, 0x56, 0x2e, 0x89, 0x25, 0x45, 0xf9, + 0xf0, 0xba, 0x9c, 0x6d, 0xd1, 0xd1, 0xde, 0x51, + 0x47, 0x63, 0xc9, 0xbd, 0xfa, 0xa2, 0x9e, 0xad, + 0x6a, 0x7b, 0x21, 0x1a, 0x6c, 0x3e, 0xff, 0x46, + 0xbe, 0xf3, 0x35, 0x7a, 0x6e, 0xb3, 0xb9, 0xf7, + 0xda, 0x5e, 0xf0, 0x14, 0xb5, 0x70, 0xa4, 0x2b, + 0xdb, 0xbb, 0xc7, 0x31, 0x4b, 0x69, 0x5a, 0x83, + 0x70, 0xd9, 0x58, 0xd4, 0x33, 0x84, 0x23, 0xf0, + 0xae, 0xbb, 0x6d, 0x26, 0x7c, 0xc8, 0x30, 0xf7, + 0x24, 0xad, 0xbd, 0xe4, 0x2c, 0x38, 0x38, 0xac, + 0xe1, 0x4a, 0x9b, 0xac, 0x33, 0x0e, 0x4a, 0xf4, + 0x93, 0xed, 0x07, 0x82, 0x81, 0x4f, 0x8f, 0xb1, + 0xdd, 0x73, 0xd5, 0x50, 0x6d, 0x44, 0x1e, 0xbe, + 0xa7, 0xcd, 0x17, 0x57, 0xd5, 0x3b, 0x62, 0x36, + 0xcf, 0x7d, 0xc8, 0xd8, 0xd1, 0x78, 0xd7, 0x85, + 0x46, 0x76, 0x5d, 0xcc, 0xfe, 0xe8, 0x94, 0xc5, + 0xad, 0xbc, 0x5e, 0xbc, 0x8d, 0x1d, 0xdf, 0x03, + 0xc9, 0x6b, 0x1b, 0x81, 0xd1, 0xb6, 0x5a, 0x24, + 0xe3, 0xdc, 0x3f, 0x20, 0xc9, 0x07, 0x73, 0x4c, + 0x43, 0x13, 0x87, 0x58, 0x34, 0x0d, 0x14, 0x63, + 0x0f, 0x6f, 0xad, 0x8d, 0xac, 0x7c, 0x67, 0x68, + 0xa3, 0x9d, 0x7f, 0x00, 0xdf, 0x28, 0xee, 0x67, + 0xf4, 0x5c, 0x26, 0xcb, 0xef, 0x56, 0x71, 0xc8, + 0xc6, 0x67, 0x5f, 0x38, 0xbb, 0xa0, 0xb1, 0x5c, + 0x1f, 0xb3, 0x08, 0xd9, 0x38, 0xcf, 0x74, 0x54, + 0xc6, 0xa4, 0xc4, 0xc0, 0x9f, 0xb3, 0xd0, 0xda, + 0x62, 0x67, 0x8b, 0x81, 0x33, 0xf0, 0xa9, 0x73, + 0xa4, 0xd1, 0x46, 0x88, 0x8d, 0x85, 0x12, 0x40, + 0xba, 0x1a, 0xcd, 0x82, 0xd8, 0x8d, 0xc4, 0x52, + 0xe7, 0x01, 0x94, 0x2e, 0x0e, 0xd0, 0xaf, 0xe7, + 0x2d, 0x3f, 0x3c, 0xaa, 0xf4, 0xf5, 0xa7, 0x01, + 0x4c, 0x14, 0xe2, 0xc2, 0x96, 0x76, 0xbe, 0x05, + 0xaa, 0x19, 0xb1, 0xbd, 0x95, 0xbb, 0x5a, 0xf9, + 0xa5, 0xa7, 0xe6, 0x16, 0x38, 0x34, 0xf7, 0x9d, + 0x19, 0x66, 0x16, 0x8e, 0x7f, 0x2b, 0x5a, 0xfb, + 0xb5, 0x29, 0x79, 0xbf, 0x52, 0xae, 0x30, 0x95, + 0x3f, 0x31, 0x33, 0x28, 0xde, 0xc5, 0x0d, 0x55, + 0x89, 0xec, 0x21, 0x11, 0x0f, 0x8b, 0xfe, 0x63, + 0x3a, 0xf1, 0x95, 0x5c, 0xcd, 0x50, 0xe4, 0x5d, + 0x8f, 0xa7, 0xc8, 0xca, 0x93, 0xa0, 0x67, 0x82, + 0x63, 0x5c, 0xd0, 0xed, 0xe7, 0x08, 0xc5, 0x60, + 0xf8, 0xb4, 0x47, 0xf0, 0x1a, 0x65, 0x4e, 0xa3, + 0x51, 0x68, 0xc7, 0x14, 0xa1, 0xd9, 0x39, 0x72, + 0xa8, 0x6f, 0x7c, 0x7e, 0xf6, 0x03, 0x0b, 0x25, + 0x9b, 0xf2, 0xca, 0x49, 0xae, 0x5b, 0xf8, 0x0f, + 0x71, 0x51, 0x01, 0xa6, 0x23, 0xa9, 0xdf, 0xd0, + 0x7a, 0x39, 0x19, 0xf5, 0xc5, 0x26, 0x44, 0x7b, + 0x0a, 0x4a, 0x41, 0xbf, 0xf2, 0x8e, 0x83, 0x50, + 0x91, 0x96, 0x72, 0x02, 0xf6, 0x80, 0xbf, 0x95, + 0x41, 0xac, 0xda, 0xb0, 0xba, 0xe3, 0x76, 0xb1, + 0x9d, 0xff, 0x1f, 0x33, 0x02, 0x85, 0xfc, 0x2a, + 0x29, 0xe6, 0xe3, 0x9d, 0xd0, 0xef, 0xc2, 0xd6, + 0x9c, 0x4a, 0x62, 0xac, 0xcb, 0xea, 0x8b, 0xc3, + 0x08, 0x6e, 0x49, 0x09, 0x26, 0x19, 0xc1, 0x30, + 0xcc, 0x27, 0xaa, 0xc6, 0x45, 0x88, 0xbd, 0xae, + 0xd6, 0x79, 0xff, 0x4e, 0xfc, 0x66, 0x4d, 0x02, + 0xa5, 0xee, 0x8e, 0xa5, 0xb6, 0x15, 0x72, 0x24, + 0xb1, 0xbf, 0xbf, 0x64, 0xcf, 0xcc, 0x93, 0xe9, + 0xb6, 0xfd, 0xb4, 0xb6, 0x21, 0xb5, 0x48, 0x08, + 0x0f, 0x11, 0x65, 0xe1, 0x47, 0xee, 0x93, 0x29, + 0xad +}; +static const u8 key72[] __initconst = { + 0xb9, 0xa2, 0xfc, 0x59, 0x06, 0x3f, 0x77, 0xa5, + 0x66, 0xd0, 0x2b, 0x22, 0x74, 0x22, 0x4c, 0x1e, + 0x6a, 0x39, 0xdf, 0xe1, 0x0d, 0x4c, 0x64, 0x99, + 0x54, 0x8a, 0xba, 0x1d, 0x2c, 0x21, 0x5f, 0xc3 +}; +enum { nonce72 = 0x3d069308fa3db04bULL }; + +static const u8 input73[] __initconst = { + 0xe4, 0xdd, 0x36, 0xd4, 0xf5, 0x70, 0x51, 0x73, + 0x97, 0x1d, 0x45, 0x05, 0x92, 0xe7, 0xeb, 0xb7, + 0x09, 0x82, 0x6e, 0x25, 0x6c, 0x50, 0xf5, 0x40, + 0x19, 0xba, 0xbc, 0xf4, 0x39, 0x14, 0xc5, 0x15, + 0x83, 0x40, 0xbd, 0x26, 0xe0, 0xff, 0x3b, 0x22, + 0x7c, 0x7c, 0xd7, 0x0b, 0xe9, 0x25, 0x0c, 0x3d, + 0x92, 0x38, 0xbe, 0xe4, 0x22, 0x75, 0x65, 0xf1, + 0x03, 0x85, 0x34, 0x09, 0xb8, 0x77, 0xfb, 0x48, + 0xb1, 0x2e, 0x21, 0x67, 0x9b, 0x9d, 0xad, 0x18, + 0x82, 0x0d, 0x6b, 0xc3, 0xcf, 0x00, 0x61, 0x6e, + 0xda, 0xdc, 0xa7, 0x0b, 0x5c, 0x02, 0x1d, 0xa6, + 0x4e, 0x0d, 0x7f, 0x37, 0x01, 0x5a, 0x37, 0xf3, + 0x2b, 0xbf, 0xba, 0xe2, 0x1c, 0xb3, 0xa3, 0xbc, + 0x1c, 0x93, 0x1a, 0xb1, 0x71, 0xaf, 0xe2, 0xdd, + 0x17, 0xee, 0x53, 0xfa, 0xfb, 0x02, 0x40, 0x3e, + 0x03, 0xca, 0xe7, 0xc3, 0x51, 0x81, 0xcc, 0x8c, + 0xca, 0xcf, 0x4e, 0xc5, 0x78, 0x99, 0xfd, 0xbf, + 0xea, 0xab, 0x38, 0x81, 0xfc, 0xd1, 0x9e, 0x41, + 0x0b, 0x84, 0x25, 0xf1, 0x6b, 0x3c, 0xf5, 0x40, + 0x0d, 0xc4, 0x3e, 0xb3, 0x6a, 0xec, 0x6e, 0x75, + 0xdc, 0x9b, 0xdf, 0x08, 0x21, 0x16, 0xfb, 0x7a, + 0x8e, 0x19, 0x13, 0x02, 0xa7, 0xfc, 0x58, 0x21, + 0xc3, 0xb3, 0x59, 0x5a, 0x9c, 0xef, 0x38, 0xbd, + 0x87, 0x55, 0xd7, 0x0d, 0x1f, 0x84, 0xdc, 0x98, + 0x22, 0xca, 0x87, 0x96, 0x71, 0x6d, 0x68, 0x00, + 0xcb, 0x4f, 0x2f, 0xc4, 0x64, 0x0c, 0xc1, 0x53, + 0x0c, 0x90, 0xe7, 0x3c, 0x88, 0xca, 0xc5, 0x85, + 0xa3, 0x2a, 0x96, 0x7c, 0x82, 0x6d, 0x45, 0xf5, + 0xb7, 0x8d, 0x17, 0x69, 0xd6, 0xcd, 0x3c, 0xd3, + 0xe7, 0x1c, 0xce, 0x93, 0x50, 0xd4, 0x59, 0xa2, + 0xd8, 0x8b, 0x72, 0x60, 0x5b, 0x25, 0x14, 0xcd, + 0x5a, 0xe8, 0x8c, 0xdb, 0x23, 0x8d, 0x2b, 0x59, + 0x12, 0x13, 0x10, 0x47, 0xa4, 0xc8, 0x3c, 0xc1, + 0x81, 0x89, 0x6c, 0x98, 0xec, 0x8f, 0x7b, 0x32, + 0xf2, 0x87, 0xd9, 0xa2, 0x0d, 0xc2, 0x08, 0xf9, + 0xd5, 0xf3, 0x91, 0xe7, 0xb3, 0x87, 0xa7, 0x0b, + 0x64, 0x8f, 0xb9, 0x55, 0x1c, 0x81, 0x96, 0x6c, + 0xa1, 0xc9, 0x6e, 0x3b, 0xcd, 0x17, 0x1b, 0xfc, + 0xa6, 0x05, 0xba, 0x4a, 0x7d, 0x03, 0x3c, 0x59, + 0xc8, 0xee, 0x50, 0xb2, 0x5b, 0xe1, 0x4d, 0x6a, + 0x1f, 0x09, 0xdc, 0xa2, 0x51, 0xd1, 0x93, 0x3a, + 0x5f, 0x72, 0x1d, 0x26, 0x14, 0x62, 0xa2, 0x41, + 0x3d, 0x08, 0x70, 0x7b, 0x27, 0x3d, 0xbc, 0xdf, + 0x15, 0xfa, 0xb9, 0x5f, 0xb5, 0x38, 0x84, 0x0b, + 0x58, 0x3d, 0xee, 0x3f, 0x32, 0x65, 0x6d, 0xd7, + 0xce, 0x97, 0x3c, 0x8d, 0xfb, 0x63, 0xb9, 0xb0, + 0xa8, 0x4a, 0x72, 0x99, 0x97, 0x58, 0xc8, 0xa7, + 0xf9, 0x4c, 0xae, 0xc1, 0x63, 0xb9, 0x57, 0x18, + 0x8a, 0xfa, 0xab, 0xe9, 0xf3, 0x67, 0xe6, 0xfd, + 0xd2, 0x9d, 0x5c, 0xa9, 0x8e, 0x11, 0x0a, 0xf4, + 0x4b, 0xf1, 0xec, 0x1a, 0xaf, 0x50, 0x5d, 0x16, + 0x13, 0x69, 0x2e, 0xbd, 0x0d, 0xe6, 0xf0, 0xb2, + 0xed, 0xb4, 0x4c, 0x59, 0x77, 0x37, 0x00, 0x0b, + 0xc7, 0xa7, 0x9e, 0x37, 0xf3, 0x60, 0x70, 0xef, + 0xf3, 0xc1, 0x74, 0x52, 0x87, 0xc6, 0xa1, 0x81, + 0xbd, 0x0a, 0x2c, 0x5d, 0x2c, 0x0c, 0x6a, 0x81, + 0xa1, 0xfe, 0x26, 0x78, 0x6c, 0x03, 0x06, 0x07, + 0x34, 0xaa, 0xd1, 0x1b, 0x40, 0x03, 0x39, 0x56, + 0xcf, 0x2a, 0x92, 0xc1, 0x4e, 0xdf, 0x29, 0x24, + 0x83, 0x22, 0x7a, 0xea, 0x67, 0x1e, 0xe7, 0x54, + 0x64, 0xd3, 0xbd, 0x3a, 0x5d, 0xae, 0xca, 0xf0, + 0x9c, 0xd6, 0x5a, 0x9a, 0x62, 0xc8, 0xc7, 0x83, + 0xf9, 0x89, 0xde, 0x2d, 0x53, 0x64, 0x61, 0xf7, + 0xa3, 0xa7, 0x31, 0x38, 0xc6, 0x22, 0x9c, 0xb4, + 0x87, 0xe0 +}; +static const u8 output73[] __initconst = { + 0x34, 0xed, 0x05, 0xb0, 0x14, 0xbc, 0x8c, 0xcc, + 0x95, 0xbd, 0x99, 0x0f, 0xb1, 0x98, 0x17, 0x10, + 0xae, 0xe0, 0x08, 0x53, 0xa3, 0x69, 0xd2, 0xed, + 0x66, 0xdb, 0x2a, 0x34, 0x8d, 0x0c, 0x6e, 0xce, + 0x63, 0x69, 0xc9, 0xe4, 0x57, 0xc3, 0x0c, 0x8b, + 0xa6, 0x2c, 0xa7, 0xd2, 0x08, 0xff, 0x4f, 0xec, + 0x61, 0x8c, 0xee, 0x0d, 0xfa, 0x6b, 0xe0, 0xe8, + 0x71, 0xbc, 0x41, 0x46, 0xd7, 0x33, 0x1d, 0xc0, + 0xfd, 0xad, 0xca, 0x8b, 0x34, 0x56, 0xa4, 0x86, + 0x71, 0x62, 0xae, 0x5e, 0x3d, 0x2b, 0x66, 0x3e, + 0xae, 0xd8, 0xc0, 0xe1, 0x21, 0x3b, 0xca, 0xd2, + 0x6b, 0xa2, 0xb8, 0xc7, 0x98, 0x4a, 0xf3, 0xcf, + 0xb8, 0x62, 0xd8, 0x33, 0xe6, 0x80, 0xdb, 0x2f, + 0x0a, 0xaf, 0x90, 0x3c, 0xe1, 0xec, 0xe9, 0x21, + 0x29, 0x42, 0x9e, 0xa5, 0x50, 0xe9, 0x93, 0xd3, + 0x53, 0x1f, 0xac, 0x2a, 0x24, 0x07, 0xb8, 0xed, + 0xed, 0x38, 0x2c, 0xc4, 0xa1, 0x2b, 0x31, 0x5d, + 0x9c, 0x24, 0x7b, 0xbf, 0xd9, 0xbb, 0x4e, 0x87, + 0x8f, 0x32, 0x30, 0xf1, 0x11, 0x29, 0x54, 0x94, + 0x00, 0x95, 0x1d, 0x1d, 0x24, 0xc0, 0xd4, 0x34, + 0x49, 0x1d, 0xd5, 0xe3, 0xa6, 0xde, 0x8b, 0xbf, + 0x5a, 0x9f, 0x58, 0x5a, 0x9b, 0x70, 0xe5, 0x9b, + 0xb3, 0xdb, 0xe8, 0xb8, 0xca, 0x1b, 0x43, 0xe3, + 0xc6, 0x6f, 0x0a, 0xd6, 0x32, 0x11, 0xd4, 0x04, + 0xef, 0xa3, 0xe4, 0x3f, 0x12, 0xd8, 0xc1, 0x73, + 0x51, 0x87, 0x03, 0xbd, 0xba, 0x60, 0x79, 0xee, + 0x08, 0xcc, 0xf7, 0xc0, 0xaa, 0x4c, 0x33, 0xc4, + 0xc7, 0x09, 0xf5, 0x91, 0xcb, 0x74, 0x57, 0x08, + 0x1b, 0x90, 0xa9, 0x1b, 0x60, 0x02, 0xd2, 0x3f, + 0x7a, 0xbb, 0xfd, 0x78, 0xf0, 0x15, 0xf9, 0x29, + 0x82, 0x8f, 0xc4, 0xb2, 0x88, 0x1f, 0xbc, 0xcc, + 0x53, 0x27, 0x8b, 0x07, 0x5f, 0xfc, 0x91, 0x29, + 0x82, 0x80, 0x59, 0x0a, 0x3c, 0xea, 0xc4, 0x7e, + 0xad, 0xd2, 0x70, 0x46, 0xbd, 0x9e, 0x3b, 0x1c, + 0x8a, 0x62, 0xea, 0x69, 0xbd, 0xf6, 0x96, 0x15, + 0xb5, 0x57, 0xe8, 0x63, 0x5f, 0x65, 0x46, 0x84, + 0x58, 0x50, 0x87, 0x4b, 0x0e, 0x5b, 0x52, 0x90, + 0xb0, 0xae, 0x37, 0x0f, 0xdd, 0x7e, 0xa2, 0xa0, + 0x8b, 0x78, 0xc8, 0x5a, 0x1f, 0x53, 0xdb, 0xc5, + 0xbf, 0x73, 0x20, 0xa9, 0x44, 0xfb, 0x1e, 0xc7, + 0x97, 0xb2, 0x3a, 0x5a, 0x17, 0xe6, 0x8b, 0x9b, + 0xe8, 0xf8, 0x2a, 0x01, 0x27, 0xa3, 0x71, 0x28, + 0xe3, 0x19, 0xc6, 0xaf, 0xf5, 0x3a, 0x26, 0xc0, + 0x5c, 0x69, 0x30, 0x78, 0x75, 0x27, 0xf2, 0x0c, + 0x22, 0x71, 0x65, 0xc6, 0x8e, 0x7b, 0x47, 0xe3, + 0x31, 0xaf, 0x7b, 0xc6, 0xc2, 0x55, 0x68, 0x81, + 0xaa, 0x1b, 0x21, 0x65, 0xfb, 0x18, 0x35, 0x45, + 0x36, 0x9a, 0x44, 0xba, 0x5c, 0xff, 0x06, 0xde, + 0x3a, 0xc8, 0x44, 0x0b, 0xaa, 0x8e, 0x34, 0xe2, + 0x84, 0xac, 0x18, 0xfe, 0x9b, 0xe1, 0x4f, 0xaa, + 0xb6, 0x90, 0x0b, 0x1c, 0x2c, 0xd9, 0x9a, 0x10, + 0x18, 0xf9, 0x49, 0x41, 0x42, 0x1b, 0xb5, 0xe1, + 0x26, 0xac, 0x2d, 0x38, 0x00, 0x00, 0xe4, 0xb4, + 0x50, 0x6f, 0x14, 0x18, 0xd6, 0x3d, 0x00, 0x59, + 0x3c, 0x45, 0xf3, 0x42, 0x13, 0x44, 0xb8, 0x57, + 0xd4, 0x43, 0x5c, 0x8a, 0x2a, 0xb4, 0xfc, 0x0a, + 0x25, 0x5a, 0xdc, 0x8f, 0x11, 0x0b, 0x11, 0x44, + 0xc7, 0x0e, 0x54, 0x8b, 0x22, 0x01, 0x7e, 0x67, + 0x2e, 0x15, 0x3a, 0xb9, 0xee, 0x84, 0x10, 0xd4, + 0x80, 0x57, 0xd7, 0x75, 0xcf, 0x8b, 0xcb, 0x03, + 0xc9, 0x92, 0x2b, 0x69, 0xd8, 0x5a, 0x9b, 0x06, + 0x85, 0x47, 0xaa, 0x4c, 0x28, 0xde, 0x49, 0x58, + 0xe6, 0x11, 0x1e, 0x5e, 0x64, 0x8e, 0x3b, 0xe0, + 0x40, 0x2e, 0xac, 0x96, 0x97, 0x15, 0x37, 0x1e, + 0x30, 0xdd +}; +static const u8 key73[] __initconst = { + 0x96, 0x06, 0x1e, 0xc1, 0x6d, 0xba, 0x49, 0x5b, + 0x65, 0x80, 0x79, 0xdd, 0xf3, 0x67, 0xa8, 0x6e, + 0x2d, 0x9c, 0x54, 0x46, 0xd8, 0x4a, 0xeb, 0x7e, + 0x23, 0x86, 0x51, 0xd8, 0x49, 0x49, 0x56, 0xe0 +}; +enum { nonce73 = 0xbefb83cb67e11ffdULL }; + +static const u8 input74[] __initconst = { + 0x47, 0x22, 0x70, 0xe5, 0x2f, 0x41, 0x18, 0x45, + 0x07, 0xd3, 0x6d, 0x32, 0x0d, 0x43, 0x92, 0x2b, + 0x9b, 0x65, 0x73, 0x13, 0x1a, 0x4f, 0x49, 0x8f, + 0xff, 0xf8, 0xcc, 0xae, 0x15, 0xab, 0x9d, 0x7d, + 0xee, 0x22, 0x5d, 0x8b, 0xde, 0x81, 0x5b, 0x81, + 0x83, 0x49, 0x35, 0x9b, 0xb4, 0xbc, 0x4e, 0x01, + 0xc2, 0x29, 0xa7, 0xf1, 0xca, 0x3a, 0xce, 0x3f, + 0xf5, 0x31, 0x93, 0xa8, 0xe2, 0xc9, 0x7d, 0x03, + 0x26, 0xa4, 0xbc, 0xa8, 0x9c, 0xb9, 0x68, 0xf3, + 0xb3, 0x91, 0xe8, 0xe6, 0xc7, 0x2b, 0x1a, 0xce, + 0xd2, 0x41, 0x53, 0xbd, 0xa3, 0x2c, 0x54, 0x94, + 0x21, 0xa1, 0x40, 0xae, 0xc9, 0x0c, 0x11, 0x92, + 0xfd, 0x91, 0xa9, 0x40, 0xca, 0xde, 0x21, 0x4e, + 0x1e, 0x3d, 0xcc, 0x2c, 0x87, 0x11, 0xef, 0x46, + 0xed, 0x52, 0x03, 0x11, 0x19, 0x43, 0x25, 0xc7, + 0x0d, 0xc3, 0x37, 0x5f, 0xd3, 0x6f, 0x0c, 0x6a, + 0x45, 0x30, 0x88, 0xec, 0xf0, 0x21, 0xef, 0x1d, + 0x7b, 0x38, 0x63, 0x4b, 0x49, 0x0c, 0x72, 0xf6, + 0x4c, 0x40, 0xc3, 0xcc, 0x03, 0xa7, 0xae, 0xa8, + 0x8c, 0x37, 0x03, 0x1c, 0x11, 0xae, 0x0d, 0x1b, + 0x62, 0x97, 0x27, 0xfc, 0x56, 0x4b, 0xb7, 0xfd, + 0xbc, 0xfb, 0x0e, 0xfc, 0x61, 0xad, 0xc6, 0xb5, + 0x9c, 0x8c, 0xc6, 0x38, 0x27, 0x91, 0x29, 0x3d, + 0x29, 0xc8, 0x37, 0xc9, 0x96, 0x69, 0xe3, 0xdc, + 0x3e, 0x61, 0x35, 0x9b, 0x99, 0x4f, 0xb9, 0x4e, + 0x5a, 0x29, 0x1c, 0x2e, 0xcf, 0x16, 0xcb, 0x69, + 0x87, 0xe4, 0x1a, 0xc4, 0x6e, 0x78, 0x43, 0x00, + 0x03, 0xb2, 0x8b, 0x03, 0xd0, 0xb4, 0xf1, 0xd2, + 0x7d, 0x2d, 0x7e, 0xfc, 0x19, 0x66, 0x5b, 0xa3, + 0x60, 0x3f, 0x9d, 0xbd, 0xfa, 0x3e, 0xca, 0x7b, + 0x26, 0x08, 0x19, 0x16, 0x93, 0x5d, 0x83, 0xfd, + 0xf9, 0x21, 0xc6, 0x31, 0x34, 0x6f, 0x0c, 0xaa, + 0x28, 0xf9, 0x18, 0xa2, 0xc4, 0x78, 0x3b, 0x56, + 0xc0, 0x88, 0x16, 0xba, 0x22, 0x2c, 0x07, 0x2f, + 0x70, 0xd0, 0xb0, 0x46, 0x35, 0xc7, 0x14, 0xdc, + 0xbb, 0x56, 0x23, 0x1e, 0x36, 0x36, 0x2d, 0x73, + 0x78, 0xc7, 0xce, 0xf3, 0x58, 0xf7, 0x58, 0xb5, + 0x51, 0xff, 0x33, 0x86, 0x0e, 0x3b, 0x39, 0xfb, + 0x1a, 0xfd, 0xf8, 0x8b, 0x09, 0x33, 0x1b, 0x83, + 0xf2, 0xe6, 0x38, 0x37, 0xef, 0x47, 0x84, 0xd9, + 0x82, 0x77, 0x2b, 0x82, 0xcc, 0xf9, 0xee, 0x94, + 0x71, 0x78, 0x81, 0xc8, 0x4d, 0x91, 0xd7, 0x35, + 0x29, 0x31, 0x30, 0x5c, 0x4a, 0x23, 0x23, 0xb1, + 0x38, 0x6b, 0xac, 0x22, 0x3f, 0x80, 0xc7, 0xe0, + 0x7d, 0xfa, 0x76, 0x47, 0xd4, 0x6f, 0x93, 0xa0, + 0xa0, 0x93, 0x5d, 0x68, 0xf7, 0x43, 0x25, 0x8f, + 0x1b, 0xc7, 0x87, 0xea, 0x59, 0x0c, 0xa2, 0xfa, + 0xdb, 0x2f, 0x72, 0x43, 0xcf, 0x90, 0xf1, 0xd6, + 0x58, 0xf3, 0x17, 0x6a, 0xdf, 0xb3, 0x4e, 0x0e, + 0x38, 0x24, 0x48, 0x1f, 0xb7, 0x01, 0xec, 0x81, + 0xb1, 0x87, 0x5b, 0xec, 0x9c, 0x11, 0x1a, 0xff, + 0xa5, 0xca, 0x5a, 0x63, 0x31, 0xb2, 0xe4, 0xc6, + 0x3c, 0x1d, 0xaf, 0x27, 0xb2, 0xd4, 0x19, 0xa2, + 0xcc, 0x04, 0x92, 0x42, 0xd2, 0xc1, 0x8c, 0x3b, + 0xce, 0xf5, 0x74, 0xc1, 0x81, 0xf8, 0x20, 0x23, + 0x6f, 0x20, 0x6d, 0x78, 0x36, 0x72, 0x2c, 0x52, + 0xdf, 0x5e, 0xe8, 0x75, 0xce, 0x1c, 0x49, 0x9d, + 0x93, 0x6f, 0x65, 0xeb, 0xb1, 0xbd, 0x8e, 0x5e, + 0xe5, 0x89, 0xc4, 0x8a, 0x81, 0x3d, 0x9a, 0xa7, + 0x11, 0x82, 0x8e, 0x38, 0x5b, 0x5b, 0xca, 0x7d, + 0x4b, 0x72, 0xc2, 0x9c, 0x30, 0x5e, 0x7f, 0xc0, + 0x6f, 0x91, 0xd5, 0x67, 0x8c, 0x3e, 0xae, 0xda, + 0x2b, 0x3c, 0x53, 0xcc, 0x50, 0x97, 0x36, 0x0b, + 0x79, 0xd6, 0x73, 0x6e, 0x7d, 0x42, 0x56, 0xe1, + 0xaa, 0xfc, 0xb3, 0xa7, 0xc8, 0x01, 0xaa, 0xc1, + 0xfc, 0x5c, 0x72, 0x8e, 0x63, 0xa8, 0x46, 0x18, + 0xee, 0x11, 0xe7, 0x30, 0x09, 0x83, 0x6c, 0xd9, + 0xf4, 0x7a, 0x7b, 0xb5, 0x1f, 0x6d, 0xc7, 0xbc, + 0xcb, 0x55, 0xea, 0x40, 0x58, 0x7a, 0x00, 0x00, + 0x90, 0x60, 0xc5, 0x64, 0x69, 0x05, 0x99, 0xd2, + 0x49, 0x62, 0x4f, 0xcb, 0x97, 0xdf, 0xdd, 0x6b, + 0x60, 0x75, 0xe2, 0xe0, 0x6f, 0x76, 0xd0, 0x37, + 0x67, 0x0a, 0xcf, 0xff, 0xc8, 0x61, 0x84, 0x14, + 0x80, 0x7c, 0x1d, 0x31, 0x8d, 0x90, 0xde, 0x0b, + 0x1c, 0x74, 0x9f, 0x82, 0x96, 0x80, 0xda, 0xaf, + 0x8d, 0x99, 0x86, 0x9f, 0x24, 0x99, 0x28, 0x3e, + 0xe0, 0xa3, 0xc3, 0x90, 0x2d, 0x14, 0x65, 0x1e, + 0x3b, 0xb9, 0xba, 0x13, 0xa5, 0x77, 0x73, 0x63, + 0x9a, 0x06, 0x3d, 0xa9, 0x28, 0x9b, 0xba, 0x25, + 0x61, 0xc9, 0xcd, 0xcf, 0x7a, 0x4d, 0x96, 0x09, + 0xcb, 0xca, 0x03, 0x9c, 0x54, 0x34, 0x31, 0x85, + 0xa0, 0x3d, 0xe5, 0xbc, 0xa5, 0x5f, 0x1b, 0xd3, + 0x10, 0x63, 0x74, 0x9d, 0x01, 0x92, 0x88, 0xf0, + 0x27, 0x9c, 0x28, 0xd9, 0xfd, 0xe2, 0x4e, 0x01, + 0x8d, 0x61, 0x79, 0x60, 0x61, 0x5b, 0x76, 0xab, + 0x06, 0xd3, 0x44, 0x87, 0x43, 0x52, 0xcd, 0x06, + 0x68, 0x1e, 0x2d, 0xc5, 0xb0, 0x07, 0x25, 0xdf, + 0x0a, 0x50, 0xd7, 0xd9, 0x08, 0x53, 0x65, 0xf1, + 0x0c, 0x2c, 0xde, 0x3f, 0x9d, 0x03, 0x1f, 0xe1, + 0x49, 0x43, 0x3c, 0x83, 0x81, 0x37, 0xf8, 0xa2, + 0x0b, 0xf9, 0x61, 0x1c, 0xc1, 0xdb, 0x79, 0xbc, + 0x64, 0xce, 0x06, 0x4e, 0x87, 0x89, 0x62, 0x73, + 0x51, 0xbc, 0xa4, 0x32, 0xd4, 0x18, 0x62, 0xab, + 0x65, 0x7e, 0xad, 0x1e, 0x91, 0xa3, 0xfa, 0x2d, + 0x58, 0x9e, 0x2a, 0xe9, 0x74, 0x44, 0x64, 0x11, + 0xe6, 0xb6, 0xb3, 0x00, 0x7e, 0xa3, 0x16, 0xef, + 0x72 +}; +static const u8 output74[] __initconst = { + 0xf5, 0xca, 0x45, 0x65, 0x50, 0x35, 0x47, 0x67, + 0x6f, 0x4f, 0x67, 0xff, 0x34, 0xd9, 0xc3, 0x37, + 0x2a, 0x26, 0xb0, 0x4f, 0x08, 0x1e, 0x45, 0x13, + 0xc7, 0x2c, 0x14, 0x75, 0x33, 0xd8, 0x8e, 0x1e, + 0x1b, 0x11, 0x0d, 0x97, 0x04, 0x33, 0x8a, 0xe4, + 0xd8, 0x8d, 0x0e, 0x12, 0x8d, 0xdb, 0x6e, 0x02, + 0xfa, 0xe5, 0xbd, 0x3a, 0xb5, 0x28, 0x07, 0x7d, + 0x20, 0xf0, 0x12, 0x64, 0x83, 0x2f, 0x59, 0x79, + 0x17, 0x88, 0x3c, 0x2d, 0x08, 0x2f, 0x55, 0xda, + 0xcc, 0x02, 0x3a, 0x82, 0xcd, 0x03, 0x94, 0xdf, + 0xdf, 0xab, 0x8a, 0x13, 0xf5, 0xe6, 0x74, 0xdf, + 0x7b, 0xe2, 0xab, 0x34, 0xbc, 0x00, 0x85, 0xbf, + 0x5a, 0x48, 0xc8, 0xff, 0x8d, 0x6c, 0x27, 0x48, + 0x19, 0x2d, 0x08, 0xfa, 0x82, 0x62, 0x39, 0x55, + 0x32, 0x11, 0xa8, 0xd7, 0xb9, 0x08, 0x2c, 0xd6, + 0x7a, 0xd9, 0x83, 0x9f, 0x9b, 0xfb, 0xec, 0x3a, + 0xd1, 0x08, 0xc7, 0xad, 0xdc, 0x98, 0x4c, 0xbc, + 0x98, 0xeb, 0x36, 0xb0, 0x39, 0xf4, 0x3a, 0xd6, + 0x53, 0x02, 0xa0, 0xa9, 0x73, 0xa1, 0xca, 0xef, + 0xd8, 0xd2, 0xec, 0x0e, 0xf8, 0xf5, 0xac, 0x8d, + 0x34, 0x41, 0x06, 0xa8, 0xc6, 0xc3, 0x31, 0xbc, + 0xe5, 0xcc, 0x7e, 0x72, 0x63, 0x59, 0x3e, 0x63, + 0xc2, 0x8d, 0x2b, 0xd5, 0xb9, 0xfd, 0x1e, 0x31, + 0x69, 0x32, 0x05, 0xd6, 0xde, 0xc9, 0xe6, 0x4c, + 0xac, 0x68, 0xf7, 0x1f, 0x9d, 0xcd, 0x0e, 0xa2, + 0x15, 0x3d, 0xd6, 0x47, 0x99, 0xab, 0x08, 0x5f, + 0x28, 0xc3, 0x4c, 0xc2, 0xd5, 0xdd, 0x10, 0xb7, + 0xbd, 0xdb, 0x9b, 0xcf, 0x85, 0x27, 0x29, 0x76, + 0x98, 0xeb, 0xad, 0x31, 0x64, 0xe7, 0xfb, 0x61, + 0xe0, 0xd8, 0x1a, 0xa6, 0xe2, 0xe7, 0x43, 0x42, + 0x77, 0xc9, 0x82, 0x00, 0xac, 0x85, 0xe0, 0xa2, + 0xd4, 0x62, 0xe3, 0xb7, 0x17, 0x6e, 0xb2, 0x9e, + 0x21, 0x58, 0x73, 0xa9, 0x53, 0x2d, 0x3c, 0xe1, + 0xdd, 0xd6, 0x6e, 0x92, 0xf2, 0x1d, 0xc2, 0x22, + 0x5f, 0x9a, 0x7e, 0xd0, 0x52, 0xbf, 0x54, 0x19, + 0xd7, 0x80, 0x63, 0x3e, 0xd0, 0x08, 0x2d, 0x37, + 0x0c, 0x15, 0xf7, 0xde, 0xab, 0x2b, 0xe3, 0x16, + 0x21, 0x3a, 0xee, 0xa5, 0xdc, 0xdf, 0xde, 0xa3, + 0x69, 0xcb, 0xfd, 0x92, 0x89, 0x75, 0xcf, 0xc9, + 0x8a, 0xa4, 0xc8, 0xdd, 0xcc, 0x21, 0xe6, 0xfe, + 0x9e, 0x43, 0x76, 0xb2, 0x45, 0x22, 0xb9, 0xb5, + 0xac, 0x7e, 0x3d, 0x26, 0xb0, 0x53, 0xc8, 0xab, + 0xfd, 0xea, 0x2c, 0xd1, 0x44, 0xc5, 0x60, 0x1b, + 0x8a, 0x99, 0x0d, 0xa5, 0x0e, 0x67, 0x6e, 0x3a, + 0x96, 0x55, 0xec, 0xe8, 0xcc, 0xbe, 0x49, 0xd9, + 0xf2, 0x72, 0x9f, 0x30, 0x21, 0x97, 0x57, 0x19, + 0xbe, 0x5e, 0x33, 0x0c, 0xee, 0xc0, 0x72, 0x0d, + 0x2e, 0xd1, 0xe1, 0x52, 0xc2, 0xea, 0x41, 0xbb, + 0xe1, 0x6d, 0xd4, 0x17, 0xa9, 0x8d, 0x89, 0xa9, + 0xd6, 0x4b, 0xc6, 0x4c, 0xf2, 0x88, 0x97, 0x54, + 0x3f, 0x4f, 0x57, 0xb7, 0x37, 0xf0, 0x2c, 0x11, + 0x15, 0x56, 0xdb, 0x28, 0xb5, 0x16, 0x84, 0x66, + 0xce, 0x45, 0x3f, 0x61, 0x75, 0xb6, 0xbe, 0x00, + 0xd1, 0xe4, 0xf5, 0x27, 0x54, 0x7f, 0xc2, 0xf1, + 0xb3, 0x32, 0x9a, 0xe8, 0x07, 0x02, 0xf3, 0xdb, + 0xa9, 0xd1, 0xc2, 0xdf, 0xee, 0xad, 0xe5, 0x8a, + 0x3c, 0xfa, 0x67, 0xec, 0x6b, 0xa4, 0x08, 0xfe, + 0xba, 0x5a, 0x58, 0x0b, 0x78, 0x11, 0x91, 0x76, + 0xe3, 0x1a, 0x28, 0x54, 0x5e, 0xbd, 0x71, 0x1b, + 0x8b, 0xdc, 0x6c, 0xf4, 0x6f, 0xd7, 0xf4, 0xf3, + 0xe1, 0x03, 0xa4, 0x3c, 0x8d, 0x91, 0x2e, 0xba, + 0x5f, 0x7f, 0x8c, 0xaf, 0x69, 0x89, 0x29, 0x0a, + 0x5b, 0x25, 0x13, 0xc4, 0x2e, 0x16, 0xc2, 0x15, + 0x07, 0x5d, 0x58, 0x33, 0x7c, 0xe0, 0xf0, 0x55, + 0x5f, 0xbf, 0x5e, 0xf0, 0x71, 0x48, 0x8f, 0xf7, + 0x48, 0xb3, 0xf7, 0x0d, 0xa1, 0xd0, 0x63, 0xb1, + 0xad, 0xae, 0xb5, 0xb0, 0x5f, 0x71, 0xaf, 0x24, + 0x8b, 0xb9, 0x1c, 0x44, 0xd2, 0x1a, 0x53, 0xd1, + 0xd5, 0xb4, 0xa9, 0xff, 0x88, 0x73, 0xb5, 0xaa, + 0x15, 0x32, 0x5f, 0x59, 0x9d, 0x2e, 0xb5, 0xcb, + 0xde, 0x21, 0x2e, 0xe9, 0x35, 0xed, 0xfd, 0x0f, + 0xb6, 0xbb, 0xe6, 0x4b, 0x16, 0xf1, 0x45, 0x1e, + 0xb4, 0x84, 0xe9, 0x58, 0x1c, 0x0c, 0x95, 0xc0, + 0xcf, 0x49, 0x8b, 0x59, 0xa1, 0x78, 0xe6, 0x80, + 0x12, 0x49, 0x7a, 0xd4, 0x66, 0x62, 0xdf, 0x9c, + 0x18, 0xc8, 0x8c, 0xda, 0xc1, 0xa6, 0xbc, 0x65, + 0x28, 0xd2, 0xa4, 0xe8, 0xf1, 0x35, 0xdb, 0x5a, + 0x75, 0x1f, 0x73, 0x60, 0xec, 0xa8, 0xda, 0x5a, + 0x43, 0x15, 0x83, 0x9b, 0xe7, 0xb1, 0xa6, 0x81, + 0xbb, 0xef, 0xf3, 0x8f, 0x0f, 0xd3, 0x79, 0xa2, + 0xe5, 0xaa, 0x42, 0xef, 0xa0, 0x13, 0x4e, 0x91, + 0x2d, 0xcb, 0x61, 0x7a, 0x9a, 0x33, 0x14, 0x50, + 0x77, 0x4a, 0xd0, 0x91, 0x48, 0xe0, 0x0c, 0xe0, + 0x11, 0xcb, 0xdf, 0xb0, 0xce, 0x06, 0xd2, 0x79, + 0x4d, 0x69, 0xb9, 0xc9, 0x36, 0x74, 0x8f, 0x81, + 0x72, 0x73, 0xf3, 0x17, 0xb7, 0x13, 0xcb, 0x5b, + 0xd2, 0x5c, 0x33, 0x61, 0xb7, 0x61, 0x79, 0xb0, + 0xc0, 0x4d, 0xa1, 0xc7, 0x5d, 0x98, 0xc9, 0xe1, + 0x98, 0xbd, 0x78, 0x5a, 0x2c, 0x64, 0x53, 0xaf, + 0xaf, 0x66, 0x51, 0x47, 0xe4, 0x48, 0x66, 0x8b, + 0x07, 0x52, 0xa3, 0x03, 0x93, 0x28, 0xad, 0xcc, + 0xa3, 0x86, 0xad, 0x63, 0x04, 0x35, 0x6c, 0x49, + 0xd5, 0x28, 0x0e, 0x00, 0x47, 0xf4, 0xd4, 0x32, + 0x27, 0x19, 0xb3, 0x29, 0xe7, 0xbc, 0xbb, 0xce, + 0x3e, 0x3e, 0xd5, 0x67, 0x20, 0xe4, 0x0b, 0x75, + 0x95, 0x24, 0xe0, 0x6c, 0xb6, 0x29, 0x0c, 0x14, + 0xfd +}; +static const u8 key74[] __initconst = { + 0xf0, 0x41, 0x5b, 0x00, 0x56, 0xc4, 0xac, 0xf6, + 0xa2, 0x4c, 0x33, 0x41, 0x16, 0x09, 0x1b, 0x8e, + 0x4d, 0xe8, 0x8c, 0xd9, 0x48, 0xab, 0x3e, 0x60, + 0xcb, 0x49, 0x3e, 0xaf, 0x2b, 0x8b, 0xc8, 0xf0 +}; +enum { nonce74 = 0xcbdb0ffd0e923384ULL }; + +static const struct chacha20_testvec chacha20_testvecs[] __initconst = { + { input01, output01, key01, nonce01, sizeof(input01) }, + { input02, output02, key02, nonce02, sizeof(input02) }, + { input03, output03, key03, nonce03, sizeof(input03) }, + { input04, output04, key04, nonce04, sizeof(input04) }, + { input05, output05, key05, nonce05, sizeof(input05) }, + { input06, output06, key06, nonce06, sizeof(input06) }, + { input07, output07, key07, nonce07, sizeof(input07) }, + { input08, output08, key08, nonce08, sizeof(input08) }, + { input09, output09, key09, nonce09, sizeof(input09) }, + { input10, output10, key10, nonce10, sizeof(input10) }, + { input11, output11, key11, nonce11, sizeof(input11) }, + { input12, output12, key12, nonce12, sizeof(input12) }, + { input13, output13, key13, nonce13, sizeof(input13) }, + { input14, output14, key14, nonce14, sizeof(input14) }, + { input15, output15, key15, nonce15, sizeof(input15) }, + { input16, output16, key16, nonce16, sizeof(input16) }, + { input17, output17, key17, nonce17, sizeof(input17) }, + { input18, output18, key18, nonce18, sizeof(input18) }, + { input19, output19, key19, nonce19, sizeof(input19) }, + { input20, output20, key20, nonce20, sizeof(input20) }, + { input21, output21, key21, nonce21, sizeof(input21) }, + { input22, output22, key22, nonce22, sizeof(input22) }, + { input23, output23, key23, nonce23, sizeof(input23) }, + { input24, output24, key24, nonce24, sizeof(input24) }, + { input25, output25, key25, nonce25, sizeof(input25) }, + { input26, output26, key26, nonce26, sizeof(input26) }, + { input27, output27, key27, nonce27, sizeof(input27) }, + { input28, output28, key28, nonce28, sizeof(input28) }, + { input29, output29, key29, nonce29, sizeof(input29) }, + { input30, output30, key30, nonce30, sizeof(input30) }, + { input31, output31, key31, nonce31, sizeof(input31) }, + { input32, output32, key32, nonce32, sizeof(input32) }, + { input33, output33, key33, nonce33, sizeof(input33) }, + { input34, output34, key34, nonce34, sizeof(input34) }, + { input35, output35, key35, nonce35, sizeof(input35) }, + { input36, output36, key36, nonce36, sizeof(input36) }, + { input37, output37, key37, nonce37, sizeof(input37) }, + { input38, output38, key38, nonce38, sizeof(input38) }, + { input39, output39, key39, nonce39, sizeof(input39) }, + { input40, output40, key40, nonce40, sizeof(input40) }, + { input41, output41, key41, nonce41, sizeof(input41) }, + { input42, output42, key42, nonce42, sizeof(input42) }, + { input43, output43, key43, nonce43, sizeof(input43) }, + { input44, output44, key44, nonce44, sizeof(input44) }, + { input45, output45, key45, nonce45, sizeof(input45) }, + { input46, output46, key46, nonce46, sizeof(input46) }, + { input47, output47, key47, nonce47, sizeof(input47) }, + { input48, output48, key48, nonce48, sizeof(input48) }, + { input49, output49, key49, nonce49, sizeof(input49) }, + { input50, output50, key50, nonce50, sizeof(input50) }, + { input51, output51, key51, nonce51, sizeof(input51) }, + { input52, output52, key52, nonce52, sizeof(input52) }, + { input53, output53, key53, nonce53, sizeof(input53) }, + { input54, output54, key54, nonce54, sizeof(input54) }, + { input55, output55, key55, nonce55, sizeof(input55) }, + { input56, output56, key56, nonce56, sizeof(input56) }, + { input57, output57, key57, nonce57, sizeof(input57) }, + { input58, output58, key58, nonce58, sizeof(input58) }, + { input59, output59, key59, nonce59, sizeof(input59) }, + { input60, output60, key60, nonce60, sizeof(input60) }, + { input61, output61, key61, nonce61, sizeof(input61) }, + { input62, output62, key62, nonce62, sizeof(input62) }, + { input63, output63, key63, nonce63, sizeof(input63) }, + { input64, output64, key64, nonce64, sizeof(input64) }, + { input65, output65, key65, nonce65, sizeof(input65) }, + { input66, output66, key66, nonce66, sizeof(input66) }, + { input67, output67, key67, nonce67, sizeof(input67) }, + { input68, output68, key68, nonce68, sizeof(input68) }, + { input69, output69, key69, nonce69, sizeof(input69) }, + { input70, output70, key70, nonce70, sizeof(input70) }, + { input71, output71, key71, nonce71, sizeof(input71) }, + { input72, output72, key72, nonce72, sizeof(input72) }, + { input73, output73, key73, nonce73, sizeof(input73) }, + { input74, output74, key74, nonce74, sizeof(input74) } +}; + +static const struct hchacha20_testvec hchacha20_testvecs[] __initconst = {{ + .key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + .nonce = { 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00, 0x31, 0x41, 0x59, 0x27 }, + .output = { 0x82, 0x41, 0x3b, 0x42, 0x27, 0xb2, 0x7b, 0xfe, + 0xd3, 0x0e, 0x42, 0x50, 0x8a, 0x87, 0x7d, 0x73, + 0xa0, 0xf9, 0xe4, 0xd5, 0x8a, 0x74, 0xa8, 0x53, + 0xc1, 0x2e, 0xc4, 0x13, 0x26, 0xd3, 0xec, 0xdc } +}}; + +static bool __init chacha20_selftest(void) +{ + enum { + MAXIMUM_TEST_BUFFER_LEN = 1UL << 10, + OUTRAGEOUSLY_HUGE_BUFFER_LEN = PAGE_SIZE * 35 + 17 /* 143k */ + }; + size_t i, j, k; + u32 derived_key[CHACHA20_KEY_WORDS]; + u8 *offset_input = NULL, *computed_output = NULL, *massive_input = NULL; + u8 offset_key[CHACHA20_KEY_SIZE + 1] + __aligned(__alignof__(unsigned long)); + struct chacha20_ctx state; + bool success = true; + simd_context_t simd_context; + + offset_input = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + massive_input = vzalloc(OUTRAGEOUSLY_HUGE_BUFFER_LEN); + if (!computed_output || !offset_input || !massive_input) { + pr_err("chacha20 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20_testvecs); ++i) { + /* Boring case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Unaligned case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + 1, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + memcpy(offset_key + 1, chacha20_testvecs[i].key, + CHACHA20_KEY_SIZE); + chacha20_init(&state, offset_key + 1, chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + 1, offset_input + 1, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + 1, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned): FAIL\n", + i + 1); + success = false; + } + if (computed_output[0]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen + 1; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Chunked case */ + if (chacha20_testvecs[i].ilen <= CHACHA20_BLOCK_SIZE) + goto next_test; + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + CHACHA20_BLOCK_SIZE, &simd_context); + chacha20(&state, computed_output + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].input + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].ilen - CHACHA20_BLOCK_SIZE, + &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (chunked): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (chunked, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + +next_test: + /* Sliding unaligned case */ + if (chacha20_testvecs[i].ilen > CHACHA20_BLOCK_SIZE + 1 || + !chacha20_testvecs[i].ilen) + continue; + for (j = 1; j < CHACHA20_BLOCK_SIZE; ++j) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + j, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + j, offset_input + j, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + j, + chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu): FAIL\n", + i + 1, j); + success = false; + } + for (k = j; k < j; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + for (k = chacha20_testvecs[i].ilen + j; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + } + } + for (i = 0; i < ARRAY_SIZE(hchacha20_testvecs); ++i) { + memset(&derived_key, 0, sizeof(derived_key)); + hchacha20(derived_key, hchacha20_testvecs[i].nonce, + hchacha20_testvecs[i].key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + if (memcmp(derived_key, hchacha20_testvecs[i].output, + CHACHA20_KEY_SIZE)) { + pr_err("hchacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, &simd_context); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, DONT_USE_SIMD); + for (k = 0; k < OUTRAGEOUSLY_HUGE_BUFFER_LEN; ++k) { + if (massive_input[k]) { + pr_err("chacha20 self-test massive: FAIL\n"); + success = false; + break; + } + } + + simd_put(&simd_context); + +out: + kfree(offset_input); + kfree(computed_output); + vfree(massive_input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c new file mode 100644 index 0000000000000..c58ac6e69d541 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c @@ -0,0 +1,9076 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20poly1305_testvec { + const u8 *input, *output, *assoc, *nonce, *key; + size_t ilen, alen, nlen; + bool failure; +}; + +/* The first of these are the ChaCha20-Poly1305 AEAD test vectors from RFC7539 + * 2.8.2. After they are generated by reference implementations. And the final + * marked ones are taken from wycheproof, but we only do these for the encrypt + * side, because mostly we're stressing the primitives rather than the actual + * chapoly construction. This also requires adding a 96-bit nonce construction, + * just for the purpose of the tests. + */ + +static const u8 enc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 enc_output001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 enc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 enc_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 enc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 enc_input002[] __initconst = { }; +static const u8 enc_output002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 enc_assoc002[] __initconst = { }; +static const u8 enc_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 enc_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 enc_input003[] __initconst = { }; +static const u8 enc_output003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 enc_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 enc_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 enc_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 enc_input004[] __initconst = { + 0xa4 +}; +static const u8 enc_output004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 enc_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 enc_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 enc_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 enc_input005[] __initconst = { + 0x2d +}; +static const u8 enc_output005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 enc_assoc005[] __initconst = { }; +static const u8 enc_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 enc_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 enc_input006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 enc_output006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 enc_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 enc_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 enc_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 enc_input007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 enc_output007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 enc_assoc007[] __initconst = { }; +static const u8 enc_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 enc_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 enc_input008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 enc_output008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 enc_assoc008[] __initconst = { }; +static const u8 enc_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 enc_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 enc_input009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 enc_output009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 enc_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 enc_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 enc_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 enc_input010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 enc_output010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 enc_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 enc_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 enc_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 enc_input011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 enc_output011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 enc_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 enc_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 enc_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 enc_input012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 enc_output012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 enc_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 enc_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 enc_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + +/* wycheproof - misc */ +static const u8 enc_input053[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, + 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 +}; +static const u8 enc_assoc053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key053[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input054[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, + 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce +}; +static const u8 enc_assoc054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key054[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input055[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd, + 0x7a, 0xda, 0x44, 0x42, 0x42, 0x69, 0xbf, 0xfa, + 0x55, 0x27, 0xf2, 0x70, 0xac, 0xf6, 0x85, 0x02, + 0xb7, 0x4c, 0x5a, 0xe2, 0xe6, 0x0c, 0x05, 0x80, + 0x98, 0x1a, 0x49, 0x38, 0x45, 0x93, 0x92, 0xc4, + 0x9b, 0xb2, 0xf2, 0x84, 0xb6, 0x46, 0xef, 0xc7, + 0xf3, 0xf0, 0xb1, 0x36, 0x1d, 0xc3, 0x48, 0xed, + 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, + 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, + 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e +}; +static const u8 enc_assoc055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key055[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input056[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, + 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 +}; +static const u8 enc_assoc056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce056[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key056[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input057[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, + 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 +}; +static const u8 enc_assoc057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce057[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key057[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input058[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42, + 0x85, 0x25, 0xbb, 0xbd, 0xbd, 0x96, 0x40, 0x05, + 0xaa, 0xd8, 0x0d, 0x8f, 0x53, 0x09, 0x7a, 0xfd, + 0x48, 0xb3, 0xa5, 0x1d, 0x19, 0xf3, 0xfa, 0x7f, + 0x67, 0xe5, 0xb6, 0xc7, 0xba, 0x6c, 0x6d, 0x3b, + 0x64, 0x4d, 0x0d, 0x7b, 0x49, 0xb9, 0x10, 0x38, + 0x0c, 0x0f, 0x4e, 0xc9, 0xe2, 0x3c, 0xb7, 0x12, + 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, + 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, + 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 +}; +static const u8 enc_assoc058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce058[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key058[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input059[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e +}; +static const u8 enc_output059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, + 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 +}; +static const u8 enc_assoc059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce059[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key059[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input060[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d +}; +static const u8 enc_output060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, + 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a +}; +static const u8 enc_assoc060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce060[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key060[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input061[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d, + 0x7a, 0xda, 0x44, 0xc2, 0x42, 0x69, 0xbf, 0x7a, + 0x55, 0x27, 0xf2, 0xf0, 0xac, 0xf6, 0x85, 0x82, + 0xb7, 0x4c, 0x5a, 0x62, 0xe6, 0x0c, 0x05, 0x00, + 0x98, 0x1a, 0x49, 0xb8, 0x45, 0x93, 0x92, 0x44, + 0x9b, 0xb2, 0xf2, 0x04, 0xb6, 0x46, 0xef, 0x47, + 0xf3, 0xf0, 0xb1, 0xb6, 0x1d, 0xc3, 0x48, 0x6d, + 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, + 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 +}; +static const u8 enc_output061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, + 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a +}; +static const u8 enc_assoc061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce061[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key061[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input062[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 +}; +static const u8 enc_output062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, + 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 +}; +static const u8 enc_assoc062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce062[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key062[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input063[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 +}; +static const u8 enc_output063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, + 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c +}; +static const u8 enc_assoc063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce063[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key063[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input064[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2, + 0x85, 0x25, 0xbb, 0x3d, 0xbd, 0x96, 0x40, 0x85, + 0xaa, 0xd8, 0x0d, 0x0f, 0x53, 0x09, 0x7a, 0x7d, + 0x48, 0xb3, 0xa5, 0x9d, 0x19, 0xf3, 0xfa, 0xff, + 0x67, 0xe5, 0xb6, 0x47, 0xba, 0x6c, 0x6d, 0xbb, + 0x64, 0x4d, 0x0d, 0xfb, 0x49, 0xb9, 0x10, 0xb8, + 0x0c, 0x0f, 0x4e, 0x49, 0xe2, 0x3c, 0xb7, 0x92, + 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, + 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 +}; +static const u8 enc_output064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, + 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 +}; +static const u8 enc_assoc064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce064[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key064[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input065[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, + 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 +}; +static const u8 enc_assoc065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce065[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key065[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input066[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 +}; +static const u8 enc_output066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, + 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 +}; +static const u8 enc_assoc066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce066[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key066[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input067[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42, + 0x05, 0x25, 0xbb, 0xbd, 0x3d, 0x96, 0x40, 0x05, + 0x2a, 0xd8, 0x0d, 0x8f, 0xd3, 0x09, 0x7a, 0xfd, + 0xc8, 0xb3, 0xa5, 0x1d, 0x99, 0xf3, 0xfa, 0x7f, + 0xe7, 0xe5, 0xb6, 0xc7, 0x3a, 0x6c, 0x6d, 0x3b, + 0xe4, 0x4d, 0x0d, 0x7b, 0xc9, 0xb9, 0x10, 0x38, + 0x8c, 0x0f, 0x4e, 0xc9, 0x62, 0x3c, 0xb7, 0x12, + 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, + 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, + 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 +}; +static const u8 enc_assoc067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce067[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key067[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input068[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, + 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d +}; +static const u8 enc_assoc068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key068[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input069[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, + 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 +}; +static const u8 enc_assoc069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key069[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input070[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42, + 0x7a, 0xda, 0x44, 0x42, 0xbd, 0x96, 0x40, 0x05, + 0x55, 0x27, 0xf2, 0x70, 0x53, 0x09, 0x7a, 0xfd, + 0xb7, 0x4c, 0x5a, 0xe2, 0x19, 0xf3, 0xfa, 0x7f, + 0x98, 0x1a, 0x49, 0x38, 0xba, 0x6c, 0x6d, 0x3b, + 0x9b, 0xb2, 0xf2, 0x84, 0x49, 0xb9, 0x10, 0x38, + 0xf3, 0xf0, 0xb1, 0x36, 0xe2, 0x3c, 0xb7, 0x12, + 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, + 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, + 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 +}; +static const u8 enc_assoc070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key070[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input071[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, + 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 +}; +static const u8 enc_assoc071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce071[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key071[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input072[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, + 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 +}; +static const u8 enc_assoc072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce072[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key072[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input073[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd, + 0x85, 0x25, 0xbb, 0xbd, 0x42, 0x69, 0xbf, 0xfa, + 0xaa, 0xd8, 0x0d, 0x8f, 0xac, 0xf6, 0x85, 0x02, + 0x48, 0xb3, 0xa5, 0x1d, 0xe6, 0x0c, 0x05, 0x80, + 0x67, 0xe5, 0xb6, 0xc7, 0x45, 0x93, 0x92, 0xc4, + 0x64, 0x4d, 0x0d, 0x7b, 0xb6, 0x46, 0xef, 0xc7, + 0x0c, 0x0f, 0x4e, 0xc9, 0x1d, 0xc3, 0x48, 0xed, + 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, + 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, + 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f +}; +static const u8 enc_assoc073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce073[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key073[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input076[] __initconst = { + 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, + 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, + 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, + 0x52, 0x2c, 0xdd, 0x12, 0x06, 0xd2, 0x20, 0xb8, + 0xf8, 0xbd, 0x07, 0xd1, 0xf1, 0xf5, 0xa1, 0xbd, + 0x9a, 0x71, 0xd1, 0x1c, 0x7f, 0x57, 0x9b, 0x85, + 0x58, 0x18, 0xc0, 0x8d, 0x4d, 0xe0, 0x36, 0x39, + 0x31, 0x83, 0xb7, 0xf5, 0x90, 0xb3, 0x35, 0xae, + 0xd8, 0xde, 0x5b, 0x57, 0xb1, 0x3c, 0x5f, 0xed, + 0xe2, 0x44, 0x1c, 0x3e, 0x18, 0x4a, 0xa9, 0xd4, + 0x6e, 0x61, 0x59, 0x85, 0x06, 0xb3, 0xe1, 0x1c, + 0x43, 0xc6, 0x2c, 0xbc, 0xac, 0xec, 0xed, 0x33, + 0x19, 0x08, 0x75, 0xb0, 0x12, 0x21, 0x8b, 0x19, + 0x30, 0xfb, 0x7c, 0x38, 0xec, 0x45, 0xac, 0x11, + 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, + 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 +}; +static const u8 enc_output076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, + 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 +}; +static const u8 enc_assoc076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce076[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 +}; +static const u8 enc_key076[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input077[] __initconst = { + 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, + 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, + 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, + 0x4f, 0x01, 0x46, 0x49, 0x95, 0xa6, 0x9e, 0x61, + 0x76, 0xcb, 0x9e, 0x02, 0xb2, 0x24, 0x7e, 0xd2, + 0x99, 0x89, 0x2f, 0x91, 0x82, 0xa4, 0x5c, 0xaf, + 0x4c, 0x69, 0x40, 0x56, 0x11, 0x76, 0x6e, 0xdf, + 0xaf, 0xdc, 0x28, 0x55, 0x19, 0xea, 0x30, 0x48, + 0x0c, 0x44, 0xf0, 0x5e, 0x78, 0x1e, 0xac, 0xf8, + 0xfc, 0xec, 0xc7, 0x09, 0x0a, 0xbb, 0x28, 0xfa, + 0x5f, 0xd5, 0x85, 0xac, 0x8c, 0xda, 0x7e, 0x87, + 0x72, 0xe5, 0x94, 0xe4, 0xce, 0x6c, 0x88, 0x32, + 0x81, 0x93, 0x2e, 0x0f, 0x89, 0xf8, 0x77, 0xa1, + 0xf0, 0x4d, 0x9c, 0x32, 0xb0, 0x6c, 0xf9, 0x0b, + 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, + 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f +}; +static const u8 enc_output077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, + 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa +}; +static const u8 enc_assoc077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce077[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 +}; +static const u8 enc_key077[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input078[] __initconst = { + 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, + 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, + 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, + 0x0b, 0x96, 0xfb, 0xa0, 0x87, 0x1e, 0x93, 0x2d, + 0xb1, 0xfb, 0xf9, 0x07, 0x61, 0xbe, 0x25, 0xdf, + 0x8d, 0xfa, 0xf9, 0x31, 0xce, 0x57, 0x57, 0xe6, + 0x17, 0xb3, 0xd7, 0xa9, 0xf0, 0xbf, 0x0f, 0xfe, + 0x5d, 0x59, 0x1a, 0x33, 0xc1, 0x43, 0xb8, 0xf5, + 0x3f, 0xd0, 0xb5, 0xa1, 0x96, 0x09, 0xfd, 0x62, + 0xe5, 0xc2, 0x51, 0xa4, 0x28, 0x1a, 0x20, 0x0c, + 0xfd, 0xc3, 0x4f, 0x28, 0x17, 0x10, 0x40, 0x6f, + 0x4e, 0x37, 0x62, 0x54, 0x46, 0xff, 0x6e, 0xf2, + 0x24, 0x91, 0x3d, 0xeb, 0x0d, 0x89, 0xaf, 0x33, + 0x71, 0x28, 0xe3, 0xd1, 0x55, 0xd1, 0x6d, 0x3e, + 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, + 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 +}; +static const u8 enc_output078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, + 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d +}; +static const u8 enc_assoc078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce078[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 +}; +static const u8 enc_key078[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input079[] __initconst = { + 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, + 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, + 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, + 0xb4, 0x0d, 0x50, 0x4e, 0x6c, 0x94, 0x9f, 0x82, + 0xd1, 0x97, 0xc2, 0x5a, 0xd1, 0x7d, 0xc7, 0x21, + 0x65, 0x11, 0x25, 0x78, 0x2a, 0xc7, 0xa7, 0x12, + 0x47, 0xfe, 0xae, 0xf3, 0x2f, 0x1f, 0x25, 0x0c, + 0xe4, 0xbb, 0x8f, 0x79, 0xac, 0xaa, 0x17, 0x9d, + 0x45, 0xa7, 0xb0, 0x54, 0x5f, 0x09, 0x24, 0x32, + 0x5e, 0xfa, 0x87, 0xd5, 0xe4, 0x41, 0xd2, 0x84, + 0x78, 0xc6, 0x1f, 0x22, 0x23, 0xee, 0x67, 0xc3, + 0xb4, 0x1f, 0x43, 0x94, 0x53, 0x5e, 0x2a, 0x24, + 0x36, 0x9a, 0x2e, 0x16, 0x61, 0x3c, 0x45, 0x94, + 0x90, 0xc1, 0x4f, 0xb1, 0xd7, 0x55, 0xfe, 0x53, + 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, + 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd +}; +static const u8 enc_output079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, + 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e +}; +static const u8 enc_assoc079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce079[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a +}; +static const u8 enc_key079[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input080[] __initconst = { + 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, + 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, + 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, + 0x98, 0xd6, 0x69, 0xa8, 0xe6, 0x0e, 0x04, 0xfc, + 0x08, 0xd5, 0xfd, 0x06, 0x9c, 0x36, 0x26, 0x38, + 0xe3, 0x40, 0x0e, 0xf4, 0xcb, 0x24, 0x2e, 0x27, + 0xe2, 0x24, 0x5e, 0x68, 0xcb, 0x9e, 0xc5, 0x83, + 0xda, 0x53, 0x40, 0xb1, 0x2e, 0xdf, 0x42, 0x3b, + 0x73, 0x26, 0xad, 0x20, 0xfe, 0xeb, 0x57, 0xda, + 0xca, 0x2e, 0x04, 0x67, 0xa3, 0x28, 0x99, 0xb4, + 0x2d, 0xf8, 0xe5, 0x6d, 0x84, 0xe0, 0x06, 0xbc, + 0x8a, 0x7a, 0xcc, 0x73, 0x1e, 0x7c, 0x1f, 0x6b, + 0xec, 0xb5, 0x71, 0x9f, 0x70, 0x77, 0xf0, 0xd4, + 0xf4, 0xc6, 0x1a, 0xb1, 0x1e, 0xba, 0xc1, 0x00, + 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, + 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 +}; +static const u8 enc_output080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, + 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 +}; +static const u8 enc_assoc080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce080[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 +}; +static const u8 enc_key080[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input081[] __initconst = { + 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, + 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, + 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, + 0x0b, 0x2a, 0xfa, 0xad, 0x5e, 0xba, 0x35, 0x06, + 0x99, 0x25, 0xa6, 0x03, 0xfd, 0xbc, 0x34, 0x1a, + 0xae, 0xd4, 0x15, 0x05, 0xb1, 0x09, 0x41, 0xfa, + 0x38, 0x56, 0xa7, 0xe2, 0x47, 0xb1, 0x04, 0x07, + 0x09, 0x74, 0x6c, 0xfc, 0x20, 0x96, 0xca, 0xa6, + 0x31, 0xb2, 0xff, 0xf4, 0x1c, 0x25, 0x05, 0x06, + 0xd8, 0x89, 0xc1, 0xc9, 0x06, 0x71, 0xad, 0xe8, + 0x53, 0xee, 0x63, 0x94, 0xc1, 0x91, 0x92, 0xa5, + 0xcf, 0x37, 0x10, 0xd1, 0x07, 0x30, 0x99, 0xe5, + 0xbc, 0x94, 0x65, 0x82, 0xfc, 0x0f, 0xab, 0x9f, + 0x54, 0x3c, 0x71, 0x6a, 0xe2, 0x48, 0x6a, 0x86, + 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, + 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 +}; +static const u8 enc_output081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, + 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 +}; +static const u8 enc_assoc081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce081[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 +}; +static const u8 enc_key081[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input082[] __initconst = { + 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, + 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, + 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, + 0x6e, 0xa6, 0x5f, 0x81, 0xb5, 0x75, 0x74, 0x7d, + 0x03, 0x5b, 0x30, 0x40, 0xfe, 0xde, 0x1e, 0xb9, + 0x45, 0x97, 0x88, 0x66, 0x97, 0x88, 0x40, 0x8e, + 0x00, 0x41, 0x3b, 0x3e, 0x37, 0x6d, 0x15, 0x2d, + 0x20, 0x4a, 0xa2, 0xb7, 0xa8, 0x35, 0x58, 0xfc, + 0xd4, 0x8a, 0x0e, 0xf7, 0xa2, 0x6b, 0x1c, 0xd6, + 0xd3, 0x5d, 0x23, 0xb3, 0xf5, 0xdf, 0xe0, 0xca, + 0x77, 0xa4, 0xce, 0x32, 0xb9, 0x4a, 0xbf, 0x83, + 0xda, 0x2a, 0xef, 0xca, 0xf0, 0x68, 0x38, 0x08, + 0x79, 0xe8, 0x9f, 0xb0, 0xa3, 0x82, 0x95, 0x95, + 0xcf, 0x44, 0xc3, 0x85, 0x2a, 0xe2, 0xcc, 0x66, + 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, + 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 +}; +static const u8 enc_output082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, + 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae +}; +static const u8 enc_assoc082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce082[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 +}; +static const u8 enc_key082[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input083[] __initconst = { + 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, + 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, + 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, + 0xa9, 0x01, 0x58, 0x10, 0x85, 0x96, 0x44, 0xdf, + 0xc0, 0x21, 0x17, 0x4e, 0x0b, 0x06, 0x0a, 0x39, + 0x74, 0x48, 0xde, 0x8b, 0x48, 0x4a, 0x86, 0x03, + 0xbe, 0x68, 0x0a, 0x69, 0x34, 0xc0, 0x90, 0x6f, + 0x30, 0xdd, 0x17, 0xea, 0xe2, 0xd4, 0xc5, 0xfa, + 0xa7, 0x77, 0xf8, 0xca, 0x53, 0x37, 0x0e, 0x08, + 0x33, 0x1b, 0x88, 0xc3, 0x42, 0xba, 0xc9, 0x59, + 0x78, 0x7b, 0xbb, 0x33, 0x93, 0x0e, 0x3b, 0x56, + 0xbe, 0x86, 0xda, 0x7f, 0x2a, 0x6e, 0xb1, 0xf9, + 0x40, 0x89, 0xd1, 0xd1, 0x81, 0x07, 0x4d, 0x43, + 0x02, 0xf8, 0xe0, 0x55, 0x2d, 0x0d, 0xe1, 0xfa, + 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, + 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a +}; +static const u8 enc_output083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, + 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 +}; +static const u8 enc_assoc083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce083[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 +}; +static const u8 enc_key083[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input084[] __initconst = { + 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, + 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, + 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, + 0x4b, 0x72, 0x48, 0x4e, 0x79, 0x91, 0xd2, 0x5c, + 0x29, 0xaa, 0x07, 0x5e, 0xb1, 0xfc, 0x16, 0xde, + 0x93, 0xfe, 0x06, 0x90, 0x58, 0x11, 0x2a, 0xb2, + 0x84, 0xa3, 0xed, 0x18, 0x78, 0x03, 0x26, 0xd1, + 0x25, 0x8a, 0x47, 0x22, 0x2f, 0xa6, 0x33, 0xd8, + 0xb2, 0x9f, 0x3b, 0xd9, 0x15, 0x0b, 0x23, 0x9b, + 0x15, 0x46, 0xc2, 0xbb, 0x9b, 0x9f, 0x41, 0x0f, + 0xeb, 0xea, 0xd3, 0x96, 0x00, 0x0e, 0xe4, 0x77, + 0x70, 0x15, 0x32, 0xc3, 0xd0, 0xf5, 0xfb, 0xf8, + 0x95, 0xd2, 0x80, 0x19, 0x6d, 0x2f, 0x73, 0x7c, + 0x5e, 0x9f, 0xec, 0x50, 0xd9, 0x2b, 0xb0, 0xdf, + 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, + 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee +}; +static const u8 enc_output084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, + 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 +}; +static const u8 enc_assoc084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce084[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 +}; +static const u8 enc_key084[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input085[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x9c, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xd4, 0xd2, 0x06, 0x61, 0x6f, 0x92, 0x93, 0xf6, + 0x5b, 0x45, 0xdb, 0xbc, 0x74, 0xe7, 0xc2, 0xed, + 0xfb, 0xcb, 0xbf, 0x1c, 0xfb, 0x67, 0x9b, 0xb7, + 0x39, 0xa5, 0x86, 0x2d, 0xe2, 0xbc, 0xb9, 0x37, + 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, + 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b +}; +static const u8 enc_output085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, + 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 +}; +static const u8 enc_assoc085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce085[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key085[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input093[] __initconst = { + 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, + 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, + 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x5b, 0x8b, 0x94, 0x50, 0x9e, 0x2b, 0x74, 0xa3, + 0x6d, 0x34, 0x6e, 0x33, 0xd5, 0x72, 0x65, 0x9b, + 0xa9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, + 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c +}; +static const u8 enc_output093[] __initconst = { + 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, + 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, + 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a +}; +static const u8 enc_assoc093[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce093[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key093[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input094[] __initconst = { + 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, + 0xc0, 0x31, 0xf7, 0x52, 0x08, 0x50, 0x67, 0x45, + 0xae, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x49, 0x6d, 0xde, 0xb0, 0x55, 0x09, 0xc6, 0xef, + 0xff, 0xab, 0x75, 0xeb, 0x2d, 0xf4, 0xab, 0x09, + 0x76, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, + 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 +}; +static const u8 enc_output094[] __initconst = { + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, + 0x02, 0xf6, 0xe1, 0xd8, 0x34, 0x20, 0x7b, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, + 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc +}; +static const u8 enc_assoc094[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce094[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key094[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input095[] __initconst = { + 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, + 0x3a, 0x09, 0x57, 0xbe, 0x0f, 0xb8, 0x59, 0x46, + 0x80, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd1, 0x76, 0x9f, 0xe8, 0x06, 0xbb, 0xfe, 0xb6, + 0xf5, 0x90, 0x95, 0x0f, 0x2e, 0xac, 0x9e, 0x0a, + 0x58, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, + 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 +}; +static const u8 enc_output095[] __initconst = { + 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, + 0xf8, 0xce, 0x41, 0x34, 0x33, 0xc8, 0x45, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, + 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 +}; +static const u8 enc_assoc095[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce095[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key095[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input096[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, + 0xd4, 0x59, 0x6f, 0xf4, 0xb3, 0x40, 0x2c, 0x5c, + 0x00, 0xb9, 0xbb, 0x53, 0x70, 0x7a, 0xa6, 0x67, + 0xd3, 0x56, 0xfe, 0x50, 0xc7, 0x19, 0x96, 0x94, + 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, + 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 +}; +static const u8 enc_output096[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, + 0x16, 0x9e, 0x79, 0x7e, 0x8f, 0x30, 0x30, 0x1b, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, + 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 +}; +static const u8 enc_assoc096[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce096[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key096[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input097[] __initconst = { + 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, + 0xe3, 0x84, 0x07, 0xde, 0x2e, 0xc7, 0x5a, 0x47, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x2d, 0x2a, 0xf7, 0xcd, 0x6b, 0x08, 0x05, 0x01, + 0xd3, 0x1b, 0xa5, 0x4f, 0xb2, 0xeb, 0x75, 0x96, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, + 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f +}; +static const u8 enc_output097[] __initconst = { + 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, + 0x21, 0x43, 0x11, 0x54, 0x12, 0xb7, 0x46, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, + 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 +}; +static const u8 enc_assoc097[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce097[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key097[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input098[] __initconst = { + 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, + 0x9e, 0xfe, 0xff, 0x78, 0xd7, 0xd8, 0x6b, 0x45, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xc6, 0xf8, 0x07, 0x8c, 0xc8, 0xef, 0x12, 0xa0, + 0xff, 0x65, 0x7d, 0x6d, 0x08, 0xdb, 0x10, 0xb8, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, + 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 +}; +static const u8 enc_output098[] __initconst = { + 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, + 0x5c, 0x39, 0xe9, 0xf2, 0xeb, 0xa8, 0x77, 0x02, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, + 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b +}; +static const u8 enc_assoc098[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce098[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key098[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input099[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, + 0x8d, 0x7d, 0x1c, 0x57, 0x91, 0x92, 0xb6, 0x98, + 0x63, 0x41, 0x44, 0x15, 0xb6, 0x99, 0x68, 0x95, + 0x9a, 0x72, 0x91, 0xb7, 0xa5, 0xaf, 0x13, 0x48, + 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, + 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 +}; +static const u8 enc_output099[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, + 0x4f, 0xba, 0x0a, 0xdd, 0xad, 0xe2, 0xaa, 0xdf, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, + 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 +}; +static const u8 enc_assoc099[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce099[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key099[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input100[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, + 0xb5, 0x77, 0xf4, 0x41, 0x37, 0xd3, 0xd6, 0xbd, + 0x13, 0x35, 0xf5, 0xeb, 0x44, 0x49, 0x40, 0x77, + 0xb2, 0x64, 0x49, 0xa5, 0x4b, 0x6c, 0x7c, 0x75, + 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, + 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 +}; +static const u8 enc_output100[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, + 0x77, 0xb0, 0xe2, 0xcb, 0x0b, 0xa3, 0xca, 0xfa, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, + 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 +}; +static const u8 enc_assoc100[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce100[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key100[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input101[] __initconst = { + 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, + 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, + 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x16, 0xe9, 0x88, 0x4a, 0x11, 0x4f, 0x0e, 0x92, + 0x66, 0xce, 0xa3, 0x88, 0x5f, 0xe3, 0x6b, 0x9f, + 0xd6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, + 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 +}; +static const u8 enc_output101[] __initconst = { + 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, + 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, + 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 +}; +static const u8 enc_assoc101[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce101[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key101[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input102[] __initconst = { + 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, + 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, + 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x0c, 0x61, 0x2c, 0x5e, 0x8d, 0x89, 0xa8, 0x73, + 0xdb, 0xca, 0xad, 0x5b, 0x73, 0x46, 0x42, 0x9b, + 0xc5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, + 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c +}; +static const u8 enc_output102[] __initconst = { + 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, + 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, + 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 +}; +static const u8 enc_assoc102[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce102[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key102[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input103[] __initconst = { + 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, + 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, + 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x6b, 0x83, 0x94, 0x33, 0x09, 0x21, 0x48, 0x6c, + 0xa1, 0x1d, 0x29, 0x1c, 0x3e, 0x97, 0xee, 0x9a, + 0xd1, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, + 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d +}; +static const u8 enc_output103[] __initconst = { + 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, + 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, + 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 +}; +static const u8 enc_assoc103[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce103[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key103[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input104[] __initconst = { + 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, + 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, + 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9a, 0xde, 0x04, 0xe7, 0x5b, 0xb7, 0x01, 0xd9, + 0x66, 0x06, 0x01, 0xb3, 0x47, 0x65, 0xde, 0x98, + 0xd5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, + 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f +}; +static const u8 enc_output104[] __initconst = { + 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, + 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, + 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d +}; +static const u8 enc_assoc104[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce104[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key104[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input105[] __initconst = { + 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, + 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, + 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x41, 0x2d, 0x96, 0xaf, 0xbe, 0x80, 0xec, 0x3e, + 0x79, 0xd4, 0x51, 0xb0, 0x0a, 0x2d, 0xb2, 0x9a, + 0xc9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, + 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d +}; +static const u8 enc_output105[] __initconst = { + 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, + 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, + 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd +}; +static const u8 enc_assoc105[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce105[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key105[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input106[] __initconst = { + 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, + 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, + 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x17, 0x7c, 0x5f, 0xfe, 0x28, 0x75, 0xf4, 0x68, + 0xf6, 0xc2, 0x96, 0x57, 0x48, 0xf3, 0x59, 0x9a, + 0xd3, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, + 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d +}; +static const u8 enc_output106[] __initconst = { + 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, + 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, + 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a +}; +static const u8 enc_assoc106[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce106[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key106[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input107[] __initconst = { + 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, + 0x9b, 0xce, 0x9a, 0xbe, 0x27, 0x3a, 0xf1, 0x44, + 0xb4, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x48, 0x02, 0x5f, 0x41, 0xfa, 0x4e, 0x33, 0x6c, + 0x78, 0x69, 0x57, 0xa2, 0xa7, 0xc4, 0x93, 0x0a, + 0x6c, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, + 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 +}; +static const u8 enc_output107[] __initconst = { + 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, + 0x59, 0x09, 0x8c, 0x34, 0x1b, 0x4a, 0xed, 0x03, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, + 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 +}; +static const u8 enc_assoc107[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce107[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key107[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input108[] __initconst = { + 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, + 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, + 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x61, 0x95, 0x01, 0x93, 0xb1, 0xbf, 0x03, 0x11, + 0xff, 0x11, 0x79, 0x89, 0xae, 0xd9, 0xa9, 0x99, + 0xb0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, + 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e +}; +static const u8 enc_output108[] __initconst = { + 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, + 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, + 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a +}; +static const u8 enc_assoc108[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce108[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key108[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input109[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, + 0x08, 0xc0, 0x11, 0xd8, 0x50, 0xdd, 0x1d, 0xcb, + 0xcf, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x53, 0x40, 0xb8, 0x5a, 0x9a, 0xa0, 0x82, 0x96, + 0xb7, 0x7a, 0x5f, 0xc3, 0x96, 0x1f, 0x66, 0x0f, + 0x17, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, + 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 +}; +static const u8 enc_output109[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, + 0xca, 0x07, 0x07, 0x52, 0x6c, 0xad, 0x01, 0x8c, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, + 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb +}; +static const u8 enc_assoc109[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce109[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key109[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input110[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, + 0x88, 0x47, 0x7e, 0x99, 0x92, 0x5a, 0x56, 0xd9, + 0xd6, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xfa, 0x84, 0xf0, 0x64, 0x55, 0x36, 0x42, 0x1b, + 0x2b, 0xb9, 0x24, 0x6e, 0xc2, 0x19, 0xed, 0x0b, + 0x0e, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, + 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 +}; +static const u8 enc_output110[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, + 0x4a, 0x80, 0x68, 0x13, 0xae, 0x2a, 0x4a, 0x9e, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, + 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe +}; +static const u8 enc_assoc110[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce110[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key110[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input111[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, + 0xd6, 0x39, 0xa9, 0x2f, 0xc7, 0xec, 0xa6, 0x55, + 0xf7, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x63, 0x48, 0xb8, 0xfd, 0x29, 0xbf, 0x96, 0xd5, + 0x63, 0xa5, 0x17, 0xe2, 0x7d, 0x7b, 0xfc, 0x0f, + 0x2f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, + 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 +}; +static const u8 enc_output111[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, + 0x14, 0xfe, 0xbf, 0xa5, 0xfb, 0x9c, 0xba, 0x12, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, + 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 +}; +static const u8 enc_assoc111[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce111[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key111[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input112[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, + 0xf9, 0x33, 0x81, 0x1e, 0x65, 0x2d, 0xc6, 0x6a, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xb6, 0x71, 0xc8, 0xca, 0xc2, 0x70, 0xc2, 0x65, + 0xa0, 0xac, 0x2f, 0x53, 0x57, 0x99, 0x88, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, + 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 +}; +static const u8 enc_output112[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, + 0x3b, 0xf4, 0x97, 0x94, 0x59, 0x5d, 0xda, 0x2d, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, + 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce +}; +static const u8 enc_assoc112[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce112[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key112[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input113[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, + 0xa1, 0x2d, 0x79, 0x09, 0x8b, 0x6d, 0xe6, 0x0f, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xa7, 0xc7, 0x58, 0x99, 0xf3, 0xe6, 0x0a, 0xf1, + 0xfc, 0xb6, 0xc7, 0x30, 0x7d, 0x87, 0x59, 0x0f, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, + 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 +}; +static const u8 enc_output113[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, + 0x63, 0xea, 0x6f, 0x83, 0xb7, 0x1d, 0xfa, 0x48, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, + 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 +}; +static const u8 enc_assoc113[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce113[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key113[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input114[] __initconst = { + 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, + 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, + 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0xbe, 0x3f, 0xa6, 0x6b, 0x6c, 0xe7, 0x80, 0x8a, + 0xa3, 0xe4, 0x59, 0x49, 0xf9, 0x44, 0x64, 0x9f, + 0xd0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, + 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 +}; +static const u8 enc_output114[] __initconst = { + 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, + 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, + 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 +}; +static const u8 enc_assoc114[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce114[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key114[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input115[] __initconst = { + 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, + 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, + 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x06, 0x53, 0xe7, 0xa3, 0x31, 0x71, 0x88, 0x33, + 0xac, 0xc3, 0xb9, 0xad, 0xff, 0x1c, 0x31, 0x98, + 0xa6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, + 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f +}; +static const u8 enc_output115[] __initconst = { + 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, + 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, + 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 +}; +static const u8 enc_assoc115[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce115[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key115[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input116[] __initconst = { + 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, + 0xcd, 0xd4, 0x41, 0x9d, 0xe6, 0x4c, 0x8f, 0x44, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x77, 0xb5, 0xc9, 0x07, 0xd9, 0xc9, 0xe1, 0xea, + 0x51, 0x85, 0x1a, 0x20, 0x4a, 0xad, 0x9f, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, + 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 +}; +static const u8 enc_output116[] __initconst = { + 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, + 0x0f, 0x13, 0x57, 0x17, 0xda, 0x3c, 0x93, 0x03, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, + 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 +}; +static const u8 enc_assoc116[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce116[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key116[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input117[] __initconst = { + 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, + 0x8c, 0xce, 0x39, 0x2c, 0xc3, 0xee, 0xdb, 0x47, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x60, 0x1b, 0x5a, 0xd2, 0x06, 0x7f, 0x28, 0x06, + 0x6a, 0x8f, 0x32, 0x81, 0x71, 0x5b, 0xa8, 0x08, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, + 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 +}; +static const u8 enc_output117[] __initconst = { + 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, + 0x4e, 0x09, 0x2f, 0xa6, 0xff, 0x9e, 0xc7, 0x00, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, + 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 +}; +static const u8 enc_assoc117[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce117[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key117[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input118[] __initconst = { + 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, + 0xeb, 0xcc, 0x6a, 0xf5, 0x59, 0xec, 0x8b, 0x43, + 0x97, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd8, 0xc8, 0xc3, 0xfa, 0x1a, 0x9e, 0x47, 0x4a, + 0xbe, 0x52, 0xd0, 0x2c, 0x81, 0x87, 0xe9, 0x0f, + 0x4f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, + 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 +}; +static const u8 enc_output118[] __initconst = { + 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, + 0x29, 0x0b, 0x7c, 0x7f, 0x65, 0x9c, 0x97, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, + 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 +}; +static const u8 enc_assoc118[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce118[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key118[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_enc_vectors[] __initconst = { + { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, + sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, + { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, + sizeof(enc_input002), sizeof(enc_assoc002), sizeof(enc_nonce002) }, + { enc_input003, enc_output003, enc_assoc003, enc_nonce003, enc_key003, + sizeof(enc_input003), sizeof(enc_assoc003), sizeof(enc_nonce003) }, + { enc_input004, enc_output004, enc_assoc004, enc_nonce004, enc_key004, + sizeof(enc_input004), sizeof(enc_assoc004), sizeof(enc_nonce004) }, + { enc_input005, enc_output005, enc_assoc005, enc_nonce005, enc_key005, + sizeof(enc_input005), sizeof(enc_assoc005), sizeof(enc_nonce005) }, + { enc_input006, enc_output006, enc_assoc006, enc_nonce006, enc_key006, + sizeof(enc_input006), sizeof(enc_assoc006), sizeof(enc_nonce006) }, + { enc_input007, enc_output007, enc_assoc007, enc_nonce007, enc_key007, + sizeof(enc_input007), sizeof(enc_assoc007), sizeof(enc_nonce007) }, + { enc_input008, enc_output008, enc_assoc008, enc_nonce008, enc_key008, + sizeof(enc_input008), sizeof(enc_assoc008), sizeof(enc_nonce008) }, + { enc_input009, enc_output009, enc_assoc009, enc_nonce009, enc_key009, + sizeof(enc_input009), sizeof(enc_assoc009), sizeof(enc_nonce009) }, + { enc_input010, enc_output010, enc_assoc010, enc_nonce010, enc_key010, + sizeof(enc_input010), sizeof(enc_assoc010), sizeof(enc_nonce010) }, + { enc_input011, enc_output011, enc_assoc011, enc_nonce011, enc_key011, + sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, + { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, + sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, + { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, + sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, + { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, + sizeof(enc_input054), sizeof(enc_assoc054), sizeof(enc_nonce054) }, + { enc_input055, enc_output055, enc_assoc055, enc_nonce055, enc_key055, + sizeof(enc_input055), sizeof(enc_assoc055), sizeof(enc_nonce055) }, + { enc_input056, enc_output056, enc_assoc056, enc_nonce056, enc_key056, + sizeof(enc_input056), sizeof(enc_assoc056), sizeof(enc_nonce056) }, + { enc_input057, enc_output057, enc_assoc057, enc_nonce057, enc_key057, + sizeof(enc_input057), sizeof(enc_assoc057), sizeof(enc_nonce057) }, + { enc_input058, enc_output058, enc_assoc058, enc_nonce058, enc_key058, + sizeof(enc_input058), sizeof(enc_assoc058), sizeof(enc_nonce058) }, + { enc_input059, enc_output059, enc_assoc059, enc_nonce059, enc_key059, + sizeof(enc_input059), sizeof(enc_assoc059), sizeof(enc_nonce059) }, + { enc_input060, enc_output060, enc_assoc060, enc_nonce060, enc_key060, + sizeof(enc_input060), sizeof(enc_assoc060), sizeof(enc_nonce060) }, + { enc_input061, enc_output061, enc_assoc061, enc_nonce061, enc_key061, + sizeof(enc_input061), sizeof(enc_assoc061), sizeof(enc_nonce061) }, + { enc_input062, enc_output062, enc_assoc062, enc_nonce062, enc_key062, + sizeof(enc_input062), sizeof(enc_assoc062), sizeof(enc_nonce062) }, + { enc_input063, enc_output063, enc_assoc063, enc_nonce063, enc_key063, + sizeof(enc_input063), sizeof(enc_assoc063), sizeof(enc_nonce063) }, + { enc_input064, enc_output064, enc_assoc064, enc_nonce064, enc_key064, + sizeof(enc_input064), sizeof(enc_assoc064), sizeof(enc_nonce064) }, + { enc_input065, enc_output065, enc_assoc065, enc_nonce065, enc_key065, + sizeof(enc_input065), sizeof(enc_assoc065), sizeof(enc_nonce065) }, + { enc_input066, enc_output066, enc_assoc066, enc_nonce066, enc_key066, + sizeof(enc_input066), sizeof(enc_assoc066), sizeof(enc_nonce066) }, + { enc_input067, enc_output067, enc_assoc067, enc_nonce067, enc_key067, + sizeof(enc_input067), sizeof(enc_assoc067), sizeof(enc_nonce067) }, + { enc_input068, enc_output068, enc_assoc068, enc_nonce068, enc_key068, + sizeof(enc_input068), sizeof(enc_assoc068), sizeof(enc_nonce068) }, + { enc_input069, enc_output069, enc_assoc069, enc_nonce069, enc_key069, + sizeof(enc_input069), sizeof(enc_assoc069), sizeof(enc_nonce069) }, + { enc_input070, enc_output070, enc_assoc070, enc_nonce070, enc_key070, + sizeof(enc_input070), sizeof(enc_assoc070), sizeof(enc_nonce070) }, + { enc_input071, enc_output071, enc_assoc071, enc_nonce071, enc_key071, + sizeof(enc_input071), sizeof(enc_assoc071), sizeof(enc_nonce071) }, + { enc_input072, enc_output072, enc_assoc072, enc_nonce072, enc_key072, + sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, + { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, + sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, + { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, + sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, + { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, + sizeof(enc_input077), sizeof(enc_assoc077), sizeof(enc_nonce077) }, + { enc_input078, enc_output078, enc_assoc078, enc_nonce078, enc_key078, + sizeof(enc_input078), sizeof(enc_assoc078), sizeof(enc_nonce078) }, + { enc_input079, enc_output079, enc_assoc079, enc_nonce079, enc_key079, + sizeof(enc_input079), sizeof(enc_assoc079), sizeof(enc_nonce079) }, + { enc_input080, enc_output080, enc_assoc080, enc_nonce080, enc_key080, + sizeof(enc_input080), sizeof(enc_assoc080), sizeof(enc_nonce080) }, + { enc_input081, enc_output081, enc_assoc081, enc_nonce081, enc_key081, + sizeof(enc_input081), sizeof(enc_assoc081), sizeof(enc_nonce081) }, + { enc_input082, enc_output082, enc_assoc082, enc_nonce082, enc_key082, + sizeof(enc_input082), sizeof(enc_assoc082), sizeof(enc_nonce082) }, + { enc_input083, enc_output083, enc_assoc083, enc_nonce083, enc_key083, + sizeof(enc_input083), sizeof(enc_assoc083), sizeof(enc_nonce083) }, + { enc_input084, enc_output084, enc_assoc084, enc_nonce084, enc_key084, + sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, + { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, + sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, + { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, + sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, + { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, + sizeof(enc_input094), sizeof(enc_assoc094), sizeof(enc_nonce094) }, + { enc_input095, enc_output095, enc_assoc095, enc_nonce095, enc_key095, + sizeof(enc_input095), sizeof(enc_assoc095), sizeof(enc_nonce095) }, + { enc_input096, enc_output096, enc_assoc096, enc_nonce096, enc_key096, + sizeof(enc_input096), sizeof(enc_assoc096), sizeof(enc_nonce096) }, + { enc_input097, enc_output097, enc_assoc097, enc_nonce097, enc_key097, + sizeof(enc_input097), sizeof(enc_assoc097), sizeof(enc_nonce097) }, + { enc_input098, enc_output098, enc_assoc098, enc_nonce098, enc_key098, + sizeof(enc_input098), sizeof(enc_assoc098), sizeof(enc_nonce098) }, + { enc_input099, enc_output099, enc_assoc099, enc_nonce099, enc_key099, + sizeof(enc_input099), sizeof(enc_assoc099), sizeof(enc_nonce099) }, + { enc_input100, enc_output100, enc_assoc100, enc_nonce100, enc_key100, + sizeof(enc_input100), sizeof(enc_assoc100), sizeof(enc_nonce100) }, + { enc_input101, enc_output101, enc_assoc101, enc_nonce101, enc_key101, + sizeof(enc_input101), sizeof(enc_assoc101), sizeof(enc_nonce101) }, + { enc_input102, enc_output102, enc_assoc102, enc_nonce102, enc_key102, + sizeof(enc_input102), sizeof(enc_assoc102), sizeof(enc_nonce102) }, + { enc_input103, enc_output103, enc_assoc103, enc_nonce103, enc_key103, + sizeof(enc_input103), sizeof(enc_assoc103), sizeof(enc_nonce103) }, + { enc_input104, enc_output104, enc_assoc104, enc_nonce104, enc_key104, + sizeof(enc_input104), sizeof(enc_assoc104), sizeof(enc_nonce104) }, + { enc_input105, enc_output105, enc_assoc105, enc_nonce105, enc_key105, + sizeof(enc_input105), sizeof(enc_assoc105), sizeof(enc_nonce105) }, + { enc_input106, enc_output106, enc_assoc106, enc_nonce106, enc_key106, + sizeof(enc_input106), sizeof(enc_assoc106), sizeof(enc_nonce106) }, + { enc_input107, enc_output107, enc_assoc107, enc_nonce107, enc_key107, + sizeof(enc_input107), sizeof(enc_assoc107), sizeof(enc_nonce107) }, + { enc_input108, enc_output108, enc_assoc108, enc_nonce108, enc_key108, + sizeof(enc_input108), sizeof(enc_assoc108), sizeof(enc_nonce108) }, + { enc_input109, enc_output109, enc_assoc109, enc_nonce109, enc_key109, + sizeof(enc_input109), sizeof(enc_assoc109), sizeof(enc_nonce109) }, + { enc_input110, enc_output110, enc_assoc110, enc_nonce110, enc_key110, + sizeof(enc_input110), sizeof(enc_assoc110), sizeof(enc_nonce110) }, + { enc_input111, enc_output111, enc_assoc111, enc_nonce111, enc_key111, + sizeof(enc_input111), sizeof(enc_assoc111), sizeof(enc_nonce111) }, + { enc_input112, enc_output112, enc_assoc112, enc_nonce112, enc_key112, + sizeof(enc_input112), sizeof(enc_assoc112), sizeof(enc_nonce112) }, + { enc_input113, enc_output113, enc_assoc113, enc_nonce113, enc_key113, + sizeof(enc_input113), sizeof(enc_assoc113), sizeof(enc_nonce113) }, + { enc_input114, enc_output114, enc_assoc114, enc_nonce114, enc_key114, + sizeof(enc_input114), sizeof(enc_assoc114), sizeof(enc_nonce114) }, + { enc_input115, enc_output115, enc_assoc115, enc_nonce115, enc_key115, + sizeof(enc_input115), sizeof(enc_assoc115), sizeof(enc_nonce115) }, + { enc_input116, enc_output116, enc_assoc116, enc_nonce116, enc_key116, + sizeof(enc_input116), sizeof(enc_assoc116), sizeof(enc_nonce116) }, + { enc_input117, enc_output117, enc_assoc117, enc_nonce117, enc_key117, + sizeof(enc_input117), sizeof(enc_assoc117), sizeof(enc_nonce117) }, + { enc_input118, enc_output118, enc_assoc118, enc_nonce118, enc_key118, + sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } +}; + +static const u8 dec_input001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 dec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 dec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 dec_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 dec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 dec_input002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 dec_output002[] __initconst = { }; +static const u8 dec_assoc002[] __initconst = { }; +static const u8 dec_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 dec_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 dec_input003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 dec_output003[] __initconst = { }; +static const u8 dec_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 dec_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 dec_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 dec_input004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 dec_output004[] __initconst = { + 0xa4 +}; +static const u8 dec_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 dec_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 dec_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 dec_input005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 dec_output005[] __initconst = { + 0x2d +}; +static const u8 dec_assoc005[] __initconst = { }; +static const u8 dec_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 dec_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 dec_input006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 dec_output006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 dec_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 dec_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 dec_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 dec_input007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 dec_output007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 dec_assoc007[] __initconst = { }; +static const u8 dec_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 dec_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 dec_input008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 dec_output008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 dec_assoc008[] __initconst = { }; +static const u8 dec_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 dec_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 dec_input009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 dec_output009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 dec_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 dec_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 dec_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 dec_input010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 dec_output010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 dec_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 dec_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 dec_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 dec_input011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 dec_output011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 dec_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 dec_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 dec_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 dec_input012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 dec_output012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const u8 dec_input013[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd7 +}; +static const u8 dec_output013[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc013[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce013[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key013[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_dec_vectors[] __initconst = { + { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, + sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, + { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, + sizeof(dec_input002), sizeof(dec_assoc002), sizeof(dec_nonce002) }, + { dec_input003, dec_output003, dec_assoc003, dec_nonce003, dec_key003, + sizeof(dec_input003), sizeof(dec_assoc003), sizeof(dec_nonce003) }, + { dec_input004, dec_output004, dec_assoc004, dec_nonce004, dec_key004, + sizeof(dec_input004), sizeof(dec_assoc004), sizeof(dec_nonce004) }, + { dec_input005, dec_output005, dec_assoc005, dec_nonce005, dec_key005, + sizeof(dec_input005), sizeof(dec_assoc005), sizeof(dec_nonce005) }, + { dec_input006, dec_output006, dec_assoc006, dec_nonce006, dec_key006, + sizeof(dec_input006), sizeof(dec_assoc006), sizeof(dec_nonce006) }, + { dec_input007, dec_output007, dec_assoc007, dec_nonce007, dec_key007, + sizeof(dec_input007), sizeof(dec_assoc007), sizeof(dec_nonce007) }, + { dec_input008, dec_output008, dec_assoc008, dec_nonce008, dec_key008, + sizeof(dec_input008), sizeof(dec_assoc008), sizeof(dec_nonce008) }, + { dec_input009, dec_output009, dec_assoc009, dec_nonce009, dec_key009, + sizeof(dec_input009), sizeof(dec_assoc009), sizeof(dec_nonce009) }, + { dec_input010, dec_output010, dec_assoc010, dec_nonce010, dec_key010, + sizeof(dec_input010), sizeof(dec_assoc010), sizeof(dec_nonce010) }, + { dec_input011, dec_output011, dec_assoc011, dec_nonce011, dec_key011, + sizeof(dec_input011), sizeof(dec_assoc011), sizeof(dec_nonce011) }, + { dec_input012, dec_output012, dec_assoc012, dec_nonce012, dec_key012, + sizeof(dec_input012), sizeof(dec_assoc012), sizeof(dec_nonce012) }, + { dec_input013, dec_output013, dec_assoc013, dec_nonce013, dec_key013, + sizeof(dec_input013), sizeof(dec_assoc013), sizeof(dec_nonce013), + true } +}; + +static const u8 xenc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xenc_output001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xenc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xenc_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xenc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_enc_vectors[] __initconst = { + { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, + sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } +}; + +static const u8 xdec_input001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xdec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xdec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xdec_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xdec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_dec_vectors[] __initconst = { + { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, + sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } +}; + +static void __init +chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src, + const size_t src_len, const u8 *ad, + const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + + simd_get(&simd_context); + chacha20_init(&chacha20_state, key, 0); + chacha20_state.counter[1] = get_unaligned_le32(nonce + 0); + chacha20_state.counter[2] = get_unaligned_le32(nonce + 4); + chacha20_state.counter[3] = get_unaligned_le32(nonce + 8); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + &simd_context); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + &simd_context); + chacha20(&chacha20_state, dst, src, src_len, &simd_context); + poly1305_update(&poly1305_state, dst, src_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + &simd_context); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + &simd_context); + poly1305_final(&poly1305_state, dst + src_len, &simd_context); + simd_put(&simd_context); + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +static void __init +chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + if (nonce_len == 8) + chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_selftest_encrypt_bignonce(dst, src, src_len, + ad, ad_len, nonce, + key); + else + BUG(); +} + +static bool __init +decryption_success(bool func_ret, bool expect_failure, int memcmp_result) +{ + if (expect_failure) + return !func_ret; + return func_ret && !memcmp_result; +} + +static bool __init chacha20poly1305_selftest(void) +{ + enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; + bool success = true, ret; + simd_context_t simd_context; + struct scatterlist sg_src[3]; + + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { + pr_err("chacha20poly1305 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + chacha20poly1305_enc_vectors[i].nonce, + chacha20poly1305_enc_vectors[i].nlen, + chacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + if (chacha20poly1305_enc_vectors[i].nlen != 8) + continue; + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE); + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), + chacha20poly1305_enc_vectors[i].key, + &simd_context); + if (!ret || memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = chacha20poly1305_decrypt(computed_output, + chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_dec_vectors[i].ilen); + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key, &simd_context); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + xchacha20poly1305_encrypt(computed_output, + xchacha20poly1305_enc_vectors[i].input, + xchacha20poly1305_enc_vectors[i].ilen, + xchacha20poly1305_enc_vectors[i].assoc, + xchacha20poly1305_enc_vectors[i].alen, + xchacha20poly1305_enc_vectors[i].nonce, + xchacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = xchacha20poly1305_decrypt(computed_output, + xchacha20poly1305_dec_vectors[i].input, + xchacha20poly1305_dec_vectors[i].ilen, + xchacha20poly1305_dec_vectors[i].assoc, + xchacha20poly1305_dec_vectors[i].alen, + xchacha20poly1305_dec_vectors[i].nonce, + xchacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + xchacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + xchacha20poly1305_dec_vectors[i].output, + xchacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + + simd_get(&simd_context); + for (total_len = POLY1305_MAC_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + sg_init_table(sg_src, 3); + sg_set_buf(&sg_src[0], input, i); + sg_set_buf(&sg_src[1], input + i, j - i); + sg_set_buf(&sg_src[2], input + j, total_len - j); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_MAC_SIZE, NULL, 0, + 0, enc_key001, &simd_context)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_MAC_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail;; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001, + &simd_context)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + simd_put(&simd_context); + +out: + kfree(computed_output); + kfree(input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/curve25519.c b/net/wireguard/crypto/zinc/selftest/curve25519.c new file mode 100644 index 0000000000000..0e3e3af06ba44 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/curve25519.c @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct curve25519_test_vector { + u8 private[CURVE25519_KEY_SIZE]; + u8 public[CURVE25519_KEY_SIZE]; + u8 result[CURVE25519_KEY_SIZE]; + bool valid; +}; +static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = { + { + .private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d, + 0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45, + 0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a, + 0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a }, + .public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4, + 0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37, + 0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d, + 0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b, + 0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6, + 0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd, + 0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb }, + .public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54, + 0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a, + 0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4, + 0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64, + 0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d, + 0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98, + 0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f, + 0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d, + 0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3, + 0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 }, + .valid = true + }, + { + .private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + { + .private = { 1, 2, 3, 4 }, + .public = { 0 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 2, 4, 6, 8 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f }, + .result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2, + 0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57, + 0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05, + 0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 }, + .valid = true + }, + { + .private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 }, + .result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d, + 0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12, + 0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99, + 0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c }, + .valid = true + }, + /* wycheproof - normal case */ + { + .private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda, + 0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66, + 0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3, + 0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba }, + .public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5, + 0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9, + 0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e, + 0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a }, + .result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5, + 0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38, + 0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e, + 0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4, + 0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5, + 0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49, + 0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 }, + .public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5, + 0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8, + 0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3, + 0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 }, + .result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff, + 0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d, + 0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe, + 0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9, + 0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39, + 0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5, + 0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 }, + .public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f, + 0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b, + 0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c, + 0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 }, + .result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53, + 0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57, + 0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0, + 0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc, + 0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d, + 0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67, + 0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c }, + .public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97, + 0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f, + 0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45, + 0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a }, + .result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93, + 0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2, + 0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44, + 0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1, + 0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95, + 0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99, + 0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d }, + .public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27, + 0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07, + 0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae, + 0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c }, + .result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73, + 0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2, + 0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f, + 0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9, + 0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd, + 0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b, + 0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 }, + .public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5, + 0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52, + 0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8, + 0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 }, + .result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86, + 0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4, + 0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6, + 0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 }, + .valid = true + }, + /* wycheproof - public key = 0 */ + { + .private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11, + 0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac, + 0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b, + 0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key = 1 */ + { + .private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61, + 0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea, + 0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f, + 0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04, + 0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77, + 0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90, + 0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97, + 0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9, + 0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7, + 0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36, + 0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd, + 0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c, + 0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e, + 0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b, + 0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e, + 0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed, + 0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e, + 0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd, + 0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 }, + .public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 }, + .result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f, + 0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1, + 0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10, + 0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3, + 0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d, + 0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00, + 0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 }, + .public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f }, + .result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8, + 0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4, + 0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70, + 0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3, + 0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a, + 0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e, + 0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57, + 0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c, + 0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59, + 0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f, + 0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42, + 0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9, + 0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c, + 0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5, + 0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65, + 0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6, + 0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4, + 0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8, + 0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7, + 0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca, + 0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f, + 0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa, + 0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3, + 0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52, + 0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3, + 0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e, + 0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75, + 0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26, + 0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea, + 0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00, + 0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8, + 0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32, + 0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87, + 0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c, + 0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6, + 0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb, + 0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 }, + .public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f }, + .result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85, + 0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f, + 0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0, + 0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38, + 0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b, + 0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c, + 0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b, + 0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81, + 0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3, + 0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d, + 0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42, + 0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98, + 0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c, + 0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9, + 0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89, + 0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29, + 0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6, + 0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c, + 0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75, + 0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89, + 0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c, + 0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f }, + .valid = true + }, + /* wycheproof - public key with low order */ + { + .private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30, + 0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69, + 0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14, + 0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3, + 0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b, + 0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef, + 0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20, + 0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf, + 0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43, + 0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f, + 0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65, + 0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06, + 0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe, + 0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9, + 0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f, + 0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8, + 0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85, + 0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c, + 0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8, + 0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d, + 0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0, + 0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a, + 0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b, + 0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67, + 0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46, + 0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02, + 0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3, + 0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30, + 0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1, + 0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6, + 0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f, + 0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77, + 0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0, + 0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e, + 0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f, + 0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77, + 0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc, + 0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1, + 0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d, + 0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09, + 0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde, + 0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1, + 0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81, + 0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a, + 0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99, + 0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17, + 0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35, + 0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55, + 0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11, + 0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b, + 0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9, + 0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53, + 0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e, + 0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6, + 0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78, + 0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2, + 0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd, + 0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb, + 0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40, + 0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2, + 0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9, + 0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60, + 0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13, + 0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c, + 0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3, + 0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65, + 0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a, + 0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7, + 0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11, + 0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82, + 0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4, + 0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c, + 0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e, + 0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a, + 0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d, + 0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2, + 0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60, + 0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25, + 0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb, + 0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97, + 0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c, + 0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 }, + .public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23, + 0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8, + 0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69, + 0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a, + 0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23, + 0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b, + 0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 }, + .public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b, + 0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44, + 0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37, + 0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80, + 0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d, + 0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b, + 0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 }, + .public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63, + 0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae, + 0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f, + 0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0, + 0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd, + 0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49, + 0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41, + 0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0, + 0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf, + 0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9, + 0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa, + 0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5, + 0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47, + 0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3, + 0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b, + 0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8, + 0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98, + 0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0, + 0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0, + 0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1, + 0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a, + 0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02, + 0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4, + 0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68, + 0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f, + 0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2, + 0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95, + 0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7, + 0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06, + 0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9, + 0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5, + 0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0, + 0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80, + 0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd, + 0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4, + 0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04, + 0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0, + 0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac, + 0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48, + 0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, + 0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, + 0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4, + 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d }, + .public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, + 0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, + 0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e, + 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 }, + .result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, + 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, + 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, + 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde, + 0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8, + 0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4, + 0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 }, + .result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d, + 0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64, + 0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd, + 0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 }, + .result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8, + 0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf, + 0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94, + 0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d }, + .result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84, + 0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62, + 0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e, + 0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 }, + .result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8, + 0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58, + 0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02, + 0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 }, + .result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9, + 0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a, + 0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44, + 0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b }, + .result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd, + 0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22, + 0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56, + 0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b }, + .result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53, + 0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f, + 0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18, + 0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f }, + .result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55, + 0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b, + 0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79, + 0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f }, + .result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39, + 0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c, + 0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb, + 0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e }, + .result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04, + 0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10, + 0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58, + 0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c }, + .result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3, + 0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c, + 0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88, + 0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 }, + .result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a, + 0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49, + 0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a, + 0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca, + 0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c, + 0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb, + 0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58, + 0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7, + 0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01, + 0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d }, + .result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d, + 0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27, + 0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b, + 0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26, + 0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2, + 0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44, + 0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e }, + .result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6, + 0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d, + 0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e, + 0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61, + 0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67, + 0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e, + 0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c }, + .result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65, + 0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce, + 0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0, + 0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee, + 0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d, + 0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14, + 0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 }, + .result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e, + 0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc, + 0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5, + 0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4, + 0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5, + 0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c, + 0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 }, + .result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b, + 0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93, + 0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f, + 0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 }, + .valid = true + }, + /* wycheproof - private key == -1 (mod order) */ + { + .private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8, + 0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 }, + .public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .valid = true + }, + /* wycheproof - private key == 1 (mod order) on twist */ + { + .private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef, + 0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f }, + .public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .valid = true + } +}; + +static bool __init curve25519_selftest(void) +{ + bool success = true, ret, ret2; + size_t i = 0, j; + u8 in[CURVE25519_KEY_SIZE]; + u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE]; + + for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) { + memset(out, 0, CURVE25519_KEY_SIZE); + ret = curve25519(out, curve25519_test_vectors[i].private, + curve25519_test_vectors[i].public); + if (ret != curve25519_test_vectors[i].valid || + memcmp(out, curve25519_test_vectors[i].result, + CURVE25519_KEY_SIZE)) { + pr_err("curve25519 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < 5; ++i) { + get_random_bytes(in, sizeof(in)); + ret = curve25519_generate_public(out, in); + ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + if (ret != ret2 || memcmp(out, out2, CURVE25519_KEY_SIZE)) { + pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x", + i + 1); + for (j = CURVE25519_KEY_SIZE; j-- > 0;) + printk(KERN_CONT "%02x", in[j]); + printk(KERN_CONT "\n"); + success = false; + } + } + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/poly1305.c b/net/wireguard/crypto/zinc/selftest/poly1305.c new file mode 100644 index 0000000000000..b4d7a9c2f6ec7 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/poly1305.c @@ -0,0 +1,1107 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct poly1305_testvec { + const u8 *input, *output, *key; + size_t ilen; +}; + +/* RFC7539 */ +static const u8 input01[] __initconst = { + 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x67, 0x72, + 0x61, 0x70, 0x68, 0x69, 0x63, 0x20, 0x46, 0x6f, + 0x72, 0x75, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x20, 0x47, 0x72, 0x6f, + 0x75, 0x70 +}; +static const u8 output01[] __initconst = { + 0xa8, 0x06, 0x1d, 0xc1, 0x30, 0x51, 0x36, 0xc6, + 0xc2, 0x2b, 0x8b, 0xaf, 0x0c, 0x01, 0x27, 0xa9 +}; +static const u8 key01[] __initconst = { + 0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33, + 0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8, + 0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd, + 0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b +}; + +/* "The Poly1305-AES message-authentication code" */ +static const u8 input02[] __initconst = { + 0xf3, 0xf6 +}; +static const u8 output02[] __initconst = { + 0xf4, 0xc6, 0x33, 0xc3, 0x04, 0x4f, 0xc1, 0x45, + 0xf8, 0x4f, 0x33, 0x5c, 0xb8, 0x19, 0x53, 0xde +}; +static const u8 key02[] __initconst = { + 0x85, 0x1f, 0xc4, 0x0c, 0x34, 0x67, 0xac, 0x0b, + 0xe0, 0x5c, 0xc2, 0x04, 0x04, 0xf3, 0xf7, 0x00, + 0x58, 0x0b, 0x3b, 0x0f, 0x94, 0x47, 0xbb, 0x1e, + 0x69, 0xd0, 0x95, 0xb5, 0x92, 0x8b, 0x6d, 0xbc +}; + +static const u8 input03[] __initconst = { }; +static const u8 output03[] __initconst = { + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; +static const u8 key03[] __initconst = { + 0xa0, 0xf3, 0x08, 0x00, 0x00, 0xf4, 0x64, 0x00, + 0xd0, 0xc7, 0xe9, 0x07, 0x6c, 0x83, 0x44, 0x03, + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; + +static const u8 input04[] __initconst = { + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output04[] __initconst = { + 0x0e, 0xe1, 0xc1, 0x6b, 0xb7, 0x3f, 0x0f, 0x4f, + 0xd1, 0x98, 0x81, 0x75, 0x3c, 0x01, 0xcd, 0xbe +}; +static const u8 key04[] __initconst = { + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef +}; + +static const u8 input05[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9 +}; +static const u8 output05[] __initconst = { + 0x51, 0x54, 0xad, 0x0d, 0x2c, 0xb2, 0x6e, 0x01, + 0x27, 0x4f, 0xc5, 0x11, 0x48, 0x49, 0x1f, 0x1b +}; +static const u8 key05[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* self-generated vectors exercise "significant" lengths, such that they + * are handled by different code paths */ +static const u8 input06[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf +}; +static const u8 output06[] __initconst = { + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 key06[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input07[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67 +}; +static const u8 output07[] __initconst = { + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 key07[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input08[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output08[] __initconst = { + 0xbb, 0xb6, 0x13, 0xb2, 0xb6, 0xd7, 0x53, 0xba, + 0x07, 0x39, 0x5b, 0x91, 0x6a, 0xae, 0xce, 0x15 +}; +static const u8 key08[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input09[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24 +}; +static const u8 output09[] __initconst = { + 0xc7, 0x94, 0xd7, 0x05, 0x7d, 0x17, 0x78, 0xc4, + 0xbb, 0xee, 0x0a, 0x39, 0xb3, 0xd9, 0x73, 0x42 +}; +static const u8 key09[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input10[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output10[] __initconst = { + 0xff, 0xbc, 0xb9, 0xb3, 0x71, 0x42, 0x31, 0x52, + 0xd7, 0xfc, 0xa5, 0xad, 0x04, 0x2f, 0xba, 0xa9 +}; +static const u8 key10[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input11[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 output11[] __initconst = { + 0x06, 0x9e, 0xd6, 0xb8, 0xef, 0x0f, 0x20, 0x7b, + 0x3e, 0x24, 0x3b, 0xb1, 0x01, 0x9f, 0xe6, 0x32 +}; +static const u8 key11[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input12[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output12[] __initconst = { + 0xcc, 0xa3, 0x39, 0xd9, 0xa4, 0x5f, 0xa2, 0x36, + 0x8c, 0x2c, 0x68, 0xb3, 0xa4, 0x17, 0x91, 0x33 +}; +static const u8 key12[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input13[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output13[] __initconst = { + 0x53, 0xf6, 0xe8, 0x28, 0xa2, 0xf0, 0xfe, 0x0e, + 0xe8, 0x15, 0xbf, 0x0b, 0xd5, 0x84, 0x1a, 0x34 +}; +static const u8 key13[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input14[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output14[] __initconst = { + 0xb8, 0x46, 0xd4, 0x4e, 0x9b, 0xbd, 0x53, 0xce, + 0xdf, 0xfb, 0xfb, 0xb6, 0xb7, 0xfa, 0x49, 0x33 +}; +static const u8 key14[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* 4th power of the key spills to 131th bit in SIMD key setup */ +static const u8 input15[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output15[] __initconst = { + 0x07, 0x14, 0x5a, 0x4c, 0x02, 0xfe, 0x5f, 0xa3, + 0x20, 0x36, 0xde, 0x68, 0xfa, 0xbe, 0x90, 0x66 +}; +static const u8 key15[] __initconst = { + 0xad, 0x62, 0x81, 0x07, 0xe8, 0x35, 0x1d, 0x0f, + 0x2c, 0x23, 0x1a, 0x05, 0xdc, 0x4a, 0x41, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* OpenSSL's poly1305_ieee754.c failed this in final stage */ +static const u8 input16[] __initconst = { + 0x84, 0x23, 0x64, 0xe1, 0x56, 0x33, 0x6c, 0x09, + 0x98, 0xb9, 0x33, 0xa6, 0x23, 0x77, 0x26, 0x18, + 0x0d, 0x9e, 0x3f, 0xdc, 0xbd, 0xe4, 0xcd, 0x5d, + 0x17, 0x08, 0x0f, 0xc3, 0xbe, 0xb4, 0x96, 0x14, + 0xd7, 0x12, 0x2c, 0x03, 0x74, 0x63, 0xff, 0x10, + 0x4d, 0x73, 0xf1, 0x9c, 0x12, 0x70, 0x46, 0x28, + 0xd4, 0x17, 0xc4, 0xc5, 0x4a, 0x3f, 0xe3, 0x0d, + 0x3c, 0x3d, 0x77, 0x14, 0x38, 0x2d, 0x43, 0xb0, + 0x38, 0x2a, 0x50, 0xa5, 0xde, 0xe5, 0x4b, 0xe8, + 0x44, 0xb0, 0x76, 0xe8, 0xdf, 0x88, 0x20, 0x1a, + 0x1c, 0xd4, 0x3b, 0x90, 0xeb, 0x21, 0x64, 0x3f, + 0xa9, 0x6f, 0x39, 0xb5, 0x18, 0xaa, 0x83, 0x40, + 0xc9, 0x42, 0xff, 0x3c, 0x31, 0xba, 0xf7, 0xc9, + 0xbd, 0xbf, 0x0f, 0x31, 0xae, 0x3f, 0xa0, 0x96, + 0xbf, 0x8c, 0x63, 0x03, 0x06, 0x09, 0x82, 0x9f, + 0xe7, 0x2e, 0x17, 0x98, 0x24, 0x89, 0x0b, 0xc8, + 0xe0, 0x8c, 0x31, 0x5c, 0x1c, 0xce, 0x2a, 0x83, + 0x14, 0x4d, 0xbb, 0xff, 0x09, 0xf7, 0x4e, 0x3e, + 0xfc, 0x77, 0x0b, 0x54, 0xd0, 0x98, 0x4a, 0x8f, + 0x19, 0xb1, 0x47, 0x19, 0xe6, 0x36, 0x35, 0x64, + 0x1d, 0x6b, 0x1e, 0xed, 0xf6, 0x3e, 0xfb, 0xf0, + 0x80, 0xe1, 0x78, 0x3d, 0x32, 0x44, 0x54, 0x12, + 0x11, 0x4c, 0x20, 0xde, 0x0b, 0x83, 0x7a, 0x0d, + 0xfa, 0x33, 0xd6, 0xb8, 0x28, 0x25, 0xff, 0xf4, + 0x4c, 0x9a, 0x70, 0xea, 0x54, 0xce, 0x47, 0xf0, + 0x7d, 0xf6, 0x98, 0xe6, 0xb0, 0x33, 0x23, 0xb5, + 0x30, 0x79, 0x36, 0x4a, 0x5f, 0xc3, 0xe9, 0xdd, + 0x03, 0x43, 0x92, 0xbd, 0xde, 0x86, 0xdc, 0xcd, + 0xda, 0x94, 0x32, 0x1c, 0x5e, 0x44, 0x06, 0x04, + 0x89, 0x33, 0x6c, 0xb6, 0x5b, 0xf3, 0x98, 0x9c, + 0x36, 0xf7, 0x28, 0x2c, 0x2f, 0x5d, 0x2b, 0x88, + 0x2c, 0x17, 0x1e, 0x74 +}; +static const u8 output16[] __initconst = { + 0xf2, 0x48, 0x31, 0x2e, 0x57, 0x8d, 0x9d, 0x58, + 0xf8, 0xb7, 0xbb, 0x4d, 0x19, 0x10, 0x54, 0x31 +}; +static const u8 key16[] __initconst = { + 0x95, 0xd5, 0xc0, 0x05, 0x50, 0x3e, 0x51, 0x0d, + 0x8c, 0xd0, 0xaa, 0x07, 0x2c, 0x4a, 0x4d, 0x06, + 0x6e, 0xab, 0xc5, 0x2d, 0x11, 0x65, 0x3d, 0xf4, + 0x7f, 0xbf, 0x63, 0xab, 0x19, 0x8b, 0xcc, 0x26 +}; + +/* AVX2 in OpenSSL's poly1305-x86.pl failed this with 176+32 split */ +static const u8 input17[] __initconst = { + 0x24, 0x8a, 0xc3, 0x10, 0x85, 0xb6, 0xc2, 0xad, + 0xaa, 0xa3, 0x82, 0x59, 0xa0, 0xd7, 0x19, 0x2c, + 0x5c, 0x35, 0xd1, 0xbb, 0x4e, 0xf3, 0x9a, 0xd9, + 0x4c, 0x38, 0xd1, 0xc8, 0x24, 0x79, 0xe2, 0xdd, + 0x21, 0x59, 0xa0, 0x77, 0x02, 0x4b, 0x05, 0x89, + 0xbc, 0x8a, 0x20, 0x10, 0x1b, 0x50, 0x6f, 0x0a, + 0x1a, 0xd0, 0xbb, 0xab, 0x76, 0xe8, 0x3a, 0x83, + 0xf1, 0xb9, 0x4b, 0xe6, 0xbe, 0xae, 0x74, 0xe8, + 0x74, 0xca, 0xb6, 0x92, 0xc5, 0x96, 0x3a, 0x75, + 0x43, 0x6b, 0x77, 0x61, 0x21, 0xec, 0x9f, 0x62, + 0x39, 0x9a, 0x3e, 0x66, 0xb2, 0xd2, 0x27, 0x07, + 0xda, 0xe8, 0x19, 0x33, 0xb6, 0x27, 0x7f, 0x3c, + 0x85, 0x16, 0xbc, 0xbe, 0x26, 0xdb, 0xbd, 0x86, + 0xf3, 0x73, 0x10, 0x3d, 0x7c, 0xf4, 0xca, 0xd1, + 0x88, 0x8c, 0x95, 0x21, 0x18, 0xfb, 0xfb, 0xd0, + 0xd7, 0xb4, 0xbe, 0xdc, 0x4a, 0xe4, 0x93, 0x6a, + 0xff, 0x91, 0x15, 0x7e, 0x7a, 0xa4, 0x7c, 0x54, + 0x44, 0x2e, 0xa7, 0x8d, 0x6a, 0xc2, 0x51, 0xd3, + 0x24, 0xa0, 0xfb, 0xe4, 0x9d, 0x89, 0xcc, 0x35, + 0x21, 0xb6, 0x6d, 0x16, 0xe9, 0xc6, 0x6a, 0x37, + 0x09, 0x89, 0x4e, 0x4e, 0xb0, 0xa4, 0xee, 0xdc, + 0x4a, 0xe1, 0x94, 0x68, 0xe6, 0x6b, 0x81, 0xf2, + 0x71, 0x35, 0x1b, 0x1d, 0x92, 0x1e, 0xa5, 0x51, + 0x04, 0x7a, 0xbc, 0xc6, 0xb8, 0x7a, 0x90, 0x1f, + 0xde, 0x7d, 0xb7, 0x9f, 0xa1, 0x81, 0x8c, 0x11, + 0x33, 0x6d, 0xbc, 0x07, 0x24, 0x4a, 0x40, 0xeb +}; +static const u8 output17[] __initconst = { + 0xbc, 0x93, 0x9b, 0xc5, 0x28, 0x14, 0x80, 0xfa, + 0x99, 0xc6, 0xd6, 0x8c, 0x25, 0x8e, 0xc4, 0x2f +}; +static const u8 key17[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* test vectors from Google */ +static const u8 input18[] __initconst = { }; +static const u8 output18[] __initconst = { + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; +static const u8 key18[] __initconst = { + 0xc8, 0xaf, 0xaa, 0xc3, 0x31, 0xee, 0x37, 0x2c, + 0xd6, 0x08, 0x2d, 0xe1, 0x34, 0x94, 0x3b, 0x17, + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; + +static const u8 input19[] __initconst = { + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, + 0x72, 0x6c, 0x64, 0x21 +}; +static const u8 output19[] __initconst = { + 0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, + 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0 +}; +static const u8 key19[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input20[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output20[] __initconst = { + 0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, + 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07 +}; +static const u8 key20[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input21[] __initconst = { + 0x89, 0xda, 0xb8, 0x0b, 0x77, 0x17, 0xc1, 0xdb, + 0x5d, 0xb4, 0x37, 0x86, 0x0a, 0x3f, 0x70, 0x21, + 0x8e, 0x93, 0xe1, 0xb8, 0xf4, 0x61, 0xfb, 0x67, + 0x7f, 0x16, 0xf3, 0x5f, 0x6f, 0x87, 0xe2, 0xa9, + 0x1c, 0x99, 0xbc, 0x3a, 0x47, 0xac, 0xe4, 0x76, + 0x40, 0xcc, 0x95, 0xc3, 0x45, 0xbe, 0x5e, 0xcc, + 0xa5, 0xa3, 0x52, 0x3c, 0x35, 0xcc, 0x01, 0x89, + 0x3a, 0xf0, 0xb6, 0x4a, 0x62, 0x03, 0x34, 0x27, + 0x03, 0x72, 0xec, 0x12, 0x48, 0x2d, 0x1b, 0x1e, + 0x36, 0x35, 0x61, 0x69, 0x8a, 0x57, 0x8b, 0x35, + 0x98, 0x03, 0x49, 0x5b, 0xb4, 0xe2, 0xef, 0x19, + 0x30, 0xb1, 0x7a, 0x51, 0x90, 0xb5, 0x80, 0xf1, + 0x41, 0x30, 0x0d, 0xf3, 0x0a, 0xdb, 0xec, 0xa2, + 0x8f, 0x64, 0x27, 0xa8, 0xbc, 0x1a, 0x99, 0x9f, + 0xd5, 0x1c, 0x55, 0x4a, 0x01, 0x7d, 0x09, 0x5d, + 0x8c, 0x3e, 0x31, 0x27, 0xda, 0xf9, 0xf5, 0x95 +}; +static const u8 output21[] __initconst = { + 0xc8, 0x5d, 0x15, 0xed, 0x44, 0xc3, 0x78, 0xd6, + 0xb0, 0x0e, 0x23, 0x06, 0x4c, 0x7b, 0xcd, 0x51 +}; +static const u8 key21[] __initconst = { + 0x2d, 0x77, 0x3b, 0xe3, 0x7a, 0xdb, 0x1e, 0x4d, + 0x68, 0x3b, 0xf0, 0x07, 0x5e, 0x79, 0xc4, 0xee, + 0x03, 0x79, 0x18, 0x53, 0x5a, 0x7f, 0x99, 0xcc, + 0xb7, 0x04, 0x0f, 0xb5, 0xf5, 0xf4, 0x3a, 0xea +}; + +static const u8 input22[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, + 0x17, 0x03, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x06, 0xdb, 0x1f, 0x1f, 0x36, 0x8d, 0x69, 0x6a, + 0x81, 0x0a, 0x34, 0x9c, 0x0c, 0x71, 0x4c, 0x9a, + 0x5e, 0x78, 0x50, 0xc2, 0x40, 0x7d, 0x72, 0x1a, + 0xcd, 0xed, 0x95, 0xe0, 0x18, 0xd7, 0xa8, 0x52, + 0x66, 0xa6, 0xe1, 0x28, 0x9c, 0xdb, 0x4a, 0xeb, + 0x18, 0xda, 0x5a, 0xc8, 0xa2, 0xb0, 0x02, 0x6d, + 0x24, 0xa5, 0x9a, 0xd4, 0x85, 0x22, 0x7f, 0x3e, + 0xae, 0xdb, 0xb2, 0xe7, 0xe3, 0x5e, 0x1c, 0x66, + 0xcd, 0x60, 0xf9, 0xab, 0xf7, 0x16, 0xdc, 0xc9, + 0xac, 0x42, 0x68, 0x2d, 0xd7, 0xda, 0xb2, 0x87, + 0xa7, 0x02, 0x4c, 0x4e, 0xef, 0xc3, 0x21, 0xcc, + 0x05, 0x74, 0xe1, 0x67, 0x93, 0xe3, 0x7c, 0xec, + 0x03, 0xc5, 0xbd, 0xa4, 0x2b, 0x54, 0xc1, 0x14, + 0xa8, 0x0b, 0x57, 0xaf, 0x26, 0x41, 0x6c, 0x7b, + 0xe7, 0x42, 0x00, 0x5e, 0x20, 0x85, 0x5c, 0x73, + 0xe2, 0x1d, 0xc8, 0xe2, 0xed, 0xc9, 0xd4, 0x35, + 0xcb, 0x6f, 0x60, 0x59, 0x28, 0x00, 0x11, 0xc2, + 0x70, 0xb7, 0x15, 0x70, 0x05, 0x1c, 0x1c, 0x9b, + 0x30, 0x52, 0x12, 0x66, 0x20, 0xbc, 0x1e, 0x27, + 0x30, 0xfa, 0x06, 0x6c, 0x7a, 0x50, 0x9d, 0x53, + 0xc6, 0x0e, 0x5a, 0xe1, 0xb4, 0x0a, 0xa6, 0xe3, + 0x9e, 0x49, 0x66, 0x92, 0x28, 0xc9, 0x0e, 0xec, + 0xb4, 0xa5, 0x0d, 0xb3, 0x2a, 0x50, 0xbc, 0x49, + 0xe9, 0x0b, 0x4f, 0x4b, 0x35, 0x9a, 0x1d, 0xfd, + 0x11, 0x74, 0x9c, 0xd3, 0x86, 0x7f, 0xcf, 0x2f, + 0xb7, 0xbb, 0x6c, 0xd4, 0x73, 0x8f, 0x6a, 0x4a, + 0xd6, 0xf7, 0xca, 0x50, 0x58, 0xf7, 0x61, 0x88, + 0x45, 0xaf, 0x9f, 0x02, 0x0f, 0x6c, 0x3b, 0x96, + 0x7b, 0x8f, 0x4c, 0xd4, 0xa9, 0x1e, 0x28, 0x13, + 0xb5, 0x07, 0xae, 0x66, 0xf2, 0xd3, 0x5c, 0x18, + 0x28, 0x4f, 0x72, 0x92, 0x18, 0x60, 0x62, 0xe1, + 0x0f, 0xd5, 0x51, 0x0d, 0x18, 0x77, 0x53, 0x51, + 0xef, 0x33, 0x4e, 0x76, 0x34, 0xab, 0x47, 0x43, + 0xf5, 0xb6, 0x8f, 0x49, 0xad, 0xca, 0xb3, 0x84, + 0xd3, 0xfd, 0x75, 0xf7, 0x39, 0x0f, 0x40, 0x06, + 0xef, 0x2a, 0x29, 0x5c, 0x8c, 0x7a, 0x07, 0x6a, + 0xd5, 0x45, 0x46, 0xcd, 0x25, 0xd2, 0x10, 0x7f, + 0xbe, 0x14, 0x36, 0xc8, 0x40, 0x92, 0x4a, 0xae, + 0xbe, 0x5b, 0x37, 0x08, 0x93, 0xcd, 0x63, 0xd1, + 0x32, 0x5b, 0x86, 0x16, 0xfc, 0x48, 0x10, 0x88, + 0x6b, 0xc1, 0x52, 0xc5, 0x32, 0x21, 0xb6, 0xdf, + 0x37, 0x31, 0x19, 0x39, 0x32, 0x55, 0xee, 0x72, + 0xbc, 0xaa, 0x88, 0x01, 0x74, 0xf1, 0x71, 0x7f, + 0x91, 0x84, 0xfa, 0x91, 0x64, 0x6f, 0x17, 0xa2, + 0x4a, 0xc5, 0x5d, 0x16, 0xbf, 0xdd, 0xca, 0x95, + 0x81, 0xa9, 0x2e, 0xda, 0x47, 0x92, 0x01, 0xf0, + 0xed, 0xbf, 0x63, 0x36, 0x00, 0xd6, 0x06, 0x6d, + 0x1a, 0xb3, 0x6d, 0x5d, 0x24, 0x15, 0xd7, 0x13, + 0x51, 0xbb, 0xcd, 0x60, 0x8a, 0x25, 0x10, 0x8d, + 0x25, 0x64, 0x19, 0x92, 0xc1, 0xf2, 0x6c, 0x53, + 0x1c, 0xf9, 0xf9, 0x02, 0x03, 0xbc, 0x4c, 0xc1, + 0x9f, 0x59, 0x27, 0xd8, 0x34, 0xb0, 0xa4, 0x71, + 0x16, 0xd3, 0x88, 0x4b, 0xbb, 0x16, 0x4b, 0x8e, + 0xc8, 0x83, 0xd1, 0xac, 0x83, 0x2e, 0x56, 0xb3, + 0x91, 0x8a, 0x98, 0x60, 0x1a, 0x08, 0xd1, 0x71, + 0x88, 0x15, 0x41, 0xd5, 0x94, 0xdb, 0x39, 0x9c, + 0x6a, 0xe6, 0x15, 0x12, 0x21, 0x74, 0x5a, 0xec, + 0x81, 0x4c, 0x45, 0xb0, 0xb0, 0x5b, 0x56, 0x54, + 0x36, 0xfd, 0x6f, 0x13, 0x7a, 0xa1, 0x0a, 0x0c, + 0x0b, 0x64, 0x37, 0x61, 0xdb, 0xd6, 0xf9, 0xa9, + 0xdc, 0xb9, 0x9b, 0x1a, 0x6e, 0x69, 0x08, 0x54, + 0xce, 0x07, 0x69, 0xcd, 0xe3, 0x97, 0x61, 0xd8, + 0x2f, 0xcd, 0xec, 0x15, 0xf0, 0xd9, 0x2d, 0x7d, + 0x8e, 0x94, 0xad, 0xe8, 0xeb, 0x83, 0xfb, 0xe0 +}; +static const u8 output22[] __initconst = { + 0x26, 0x37, 0x40, 0x8f, 0xe1, 0x30, 0x86, 0xea, + 0x73, 0xf9, 0x71, 0xe3, 0x42, 0x5e, 0x28, 0x20 +}; +static const u8 key22[] __initconst = { + 0x99, 0xe5, 0x82, 0x2d, 0xd4, 0x17, 0x3c, 0x99, + 0x5e, 0x3d, 0xae, 0x0d, 0xde, 0xfb, 0x97, 0x74, + 0x3f, 0xde, 0x3b, 0x08, 0x01, 0x34, 0xb3, 0x9f, + 0x76, 0xe9, 0xbf, 0x8d, 0x0e, 0x88, 0xd5, 0x46 +}; + +/* test vectors from Hanno Böck */ +static const u8 input23[] __initconst = { + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0x80, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xc5, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe3, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xac, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe6, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x00, 0x00, 0x00, + 0xaf, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xff, 0xff, 0xff, 0xf5, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xe7, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x71, 0x92, 0x05, 0xa8, 0x52, 0x1d, + 0xfc +}; +static const u8 output23[] __initconst = { + 0x85, 0x59, 0xb8, 0x76, 0xec, 0xee, 0xd6, 0x6e, + 0xb3, 0x77, 0x98, 0xc0, 0x45, 0x7b, 0xaf, 0xf9 +}; +static const u8 key23[] __initconst = { + 0x7f, 0x1b, 0x02, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc +}; + +static const u8 input24[] __initconst = { + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x64 +}; +static const u8 output24[] __initconst = { + 0x00, 0xbd, 0x12, 0x58, 0x97, 0x8e, 0x20, 0x54, + 0x44, 0xc9, 0xaa, 0xaa, 0x82, 0x00, 0x6f, 0xed +}; +static const u8 key24[] __initconst = { + 0xe0, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa +}; + +static const u8 input25[] __initconst = { + 0x02, 0xfc +}; +static const u8 output25[] __initconst = { + 0x06, 0x12, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; +static const u8 key25[] __initconst = { + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; + +static const u8 input26[] __initconst = { + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x5c, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x6e, 0x7b, 0x00, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x5c, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x6e, 0x7b, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x7a, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc +}; +static const u8 output26[] __initconst = { + 0x33, 0x20, 0x5b, 0xbf, 0x9e, 0x9f, 0x8f, 0x72, + 0x12, 0xab, 0x9e, 0x2a, 0xb9, 0xb7, 0xe4, 0xa5 +}; +static const u8 key26[] __initconst = { + 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x7b +}; + +static const u8 input27[] __initconst = { + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0xff, 0xff, 0xff, 0xe9, + 0xe9, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x00, 0x00, 0xac, 0xac, + 0xec, 0x01, 0x00, 0xac, 0xac, 0xac, 0x2c, 0xac, + 0xa2, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x64, 0xf2 +}; +static const u8 output27[] __initconst = { + 0x02, 0xee, 0x7c, 0x8c, 0x54, 0x6d, 0xde, 0xb1, + 0xa4, 0x67, 0xe4, 0xc3, 0x98, 0x11, 0x58, 0xb9 +}; +static const u8 key27[] __initconst = { + 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x7f, + 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xcf, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77 +}; + +/* nacl */ +static const u8 input28[] __initconst = { + 0x8e, 0x99, 0x3b, 0x9f, 0x48, 0x68, 0x12, 0x73, + 0xc2, 0x96, 0x50, 0xba, 0x32, 0xfc, 0x76, 0xce, + 0x48, 0x33, 0x2e, 0xa7, 0x16, 0x4d, 0x96, 0xa4, + 0x47, 0x6f, 0xb8, 0xc5, 0x31, 0xa1, 0x18, 0x6a, + 0xc0, 0xdf, 0xc1, 0x7c, 0x98, 0xdc, 0xe8, 0x7b, + 0x4d, 0xa7, 0xf0, 0x11, 0xec, 0x48, 0xc9, 0x72, + 0x71, 0xd2, 0xc2, 0x0f, 0x9b, 0x92, 0x8f, 0xe2, + 0x27, 0x0d, 0x6f, 0xb8, 0x63, 0xd5, 0x17, 0x38, + 0xb4, 0x8e, 0xee, 0xe3, 0x14, 0xa7, 0xcc, 0x8a, + 0xb9, 0x32, 0x16, 0x45, 0x48, 0xe5, 0x26, 0xae, + 0x90, 0x22, 0x43, 0x68, 0x51, 0x7a, 0xcf, 0xea, + 0xbd, 0x6b, 0xb3, 0x73, 0x2b, 0xc0, 0xe9, 0xda, + 0x99, 0x83, 0x2b, 0x61, 0xca, 0x01, 0xb6, 0xde, + 0x56, 0x24, 0x4a, 0x9e, 0x88, 0xd5, 0xf9, 0xb3, + 0x79, 0x73, 0xf6, 0x22, 0xa4, 0x3d, 0x14, 0xa6, + 0x59, 0x9b, 0x1f, 0x65, 0x4c, 0xb4, 0x5a, 0x74, + 0xe3, 0x55, 0xa5 +}; +static const u8 output28[] __initconst = { + 0xf3, 0xff, 0xc7, 0x70, 0x3f, 0x94, 0x00, 0xe5, + 0x2a, 0x7d, 0xfb, 0x4b, 0x3d, 0x33, 0x05, 0xd9 +}; +static const u8 key28[] __initconst = { + 0xee, 0xa6, 0xa7, 0x25, 0x1c, 0x1e, 0x72, 0x91, + 0x6d, 0x11, 0xc2, 0xcb, 0x21, 0x4d, 0x3c, 0x25, + 0x25, 0x39, 0x12, 0x1d, 0x8e, 0x23, 0x4e, 0x65, + 0x2d, 0x65, 0x1f, 0xa4, 0xc8, 0xcf, 0xf8, 0x80 +}; + +/* wrap 2^130-5 */ +static const u8 input29[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output29[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key29[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* wrap 2^128 */ +static const u8 input30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output30[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +/* limb carry */ +static const u8 input31[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output31[] __initconst = { + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key31[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-5 */ +static const u8 input32[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfb, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; +static const u8 output32[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key32[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-6 */ +static const u8 input33[] __initconst = { + 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output33[] __initconst = { + 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 key33[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction intermediate */ +static const u8 input34[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output34[] __initconst = { + 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key34[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction final */ +static const u8 input35[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output35[] __initconst = { + 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key35[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const struct poly1305_testvec poly1305_testvecs[] __initconst = { + { input01, output01, key01, sizeof(input01) }, + { input02, output02, key02, sizeof(input02) }, + { input03, output03, key03, sizeof(input03) }, + { input04, output04, key04, sizeof(input04) }, + { input05, output05, key05, sizeof(input05) }, + { input06, output06, key06, sizeof(input06) }, + { input07, output07, key07, sizeof(input07) }, + { input08, output08, key08, sizeof(input08) }, + { input09, output09, key09, sizeof(input09) }, + { input10, output10, key10, sizeof(input10) }, + { input11, output11, key11, sizeof(input11) }, + { input12, output12, key12, sizeof(input12) }, + { input13, output13, key13, sizeof(input13) }, + { input14, output14, key14, sizeof(input14) }, + { input15, output15, key15, sizeof(input15) }, + { input16, output16, key16, sizeof(input16) }, + { input17, output17, key17, sizeof(input17) }, + { input18, output18, key18, sizeof(input18) }, + { input19, output19, key19, sizeof(input19) }, + { input20, output20, key20, sizeof(input20) }, + { input21, output21, key21, sizeof(input21) }, + { input22, output22, key22, sizeof(input22) }, + { input23, output23, key23, sizeof(input23) }, + { input24, output24, key24, sizeof(input24) }, + { input25, output25, key25, sizeof(input25) }, + { input26, output26, key26, sizeof(input26) }, + { input27, output27, key27, sizeof(input27) }, + { input28, output28, key28, sizeof(input28) }, + { input29, output29, key29, sizeof(input29) }, + { input30, output30, key30, sizeof(input30) }, + { input31, output31, key31, sizeof(input31) }, + { input32, output32, key32, sizeof(input32) }, + { input33, output33, key33, sizeof(input33) }, + { input34, output34, key34, sizeof(input34) }, + { input35, output35, key35, sizeof(input35) } +}; + +static bool __init poly1305_selftest(void) +{ + simd_context_t simd_context; + bool success = true; + size_t i, j; + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(poly1305_testvecs); ++i) { + struct poly1305_ctx poly1305; + u8 out[POLY1305_MAC_SIZE]; + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + poly1305_testvecs[i].ilen, &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu: FAIL\n", i + 1); + success = false; + } + simd_relax(&simd_context); + + if (poly1305_testvecs[i].ilen <= 1) + continue; + + for (j = 1; j < poly1305_testvecs[i].ilen - 1; ++j) { + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu): FAIL\n", + i + 1, j); + success = false; + } + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + DONT_USE_SIMD); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n", + i + 1, j); + success = false; + } + simd_relax(&simd_context); + } + } + simd_put(&simd_context); + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/run.h b/net/wireguard/crypto/zinc/selftest/run.h new file mode 100644 index 0000000000000..4ffaf6089eeaf --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/run.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_SELFTEST_RUN_H +#define _ZINC_SELFTEST_RUN_H + +#include +#include +#include + +static inline bool selftest_run(const char *name, bool (*selftest)(void), + bool *const nobs[], unsigned int nobs_len) +{ + unsigned long set = 0, subset = 0, largest_subset = 0; + unsigned int i; + + BUILD_BUG_ON(!__builtin_constant_p(nobs_len) || + nobs_len >= BITS_PER_LONG); + + if (!IS_ENABLED(CONFIG_ZINC_SELFTEST)) + return true; + + for (i = 0; i < nobs_len; ++i) + set |= ((unsigned long)*nobs[i]) << i; + + do { + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & subset; + if (selftest()) + largest_subset = max(subset, largest_subset); + else + pr_err("%s self-test combination 0x%lx: FAIL\n", name, + subset); + subset = (subset - set) & set; + } while (subset); + + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & largest_subset; + + if (largest_subset == set) + pr_info("%s self-tests: pass\n", name); + + return !WARN_ON(largest_subset != set); +} + +#endif diff --git a/net/wireguard/device.c b/net/wireguard/device.c new file mode 100644 index 0000000000000..e888ac37f9a89 --- /dev/null +++ b/net/wireguard/device.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "socket.h" +#include "timers.h" +#include "device.h" +#include "ratelimiter.h" +#include "peer.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(device_list); + +static int wg_open(struct net_device *dev) +{ + struct in_device *dev_v4 = __in_dev_get_rtnl(dev); +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + struct inet6_dev *dev_v6 = __in6_dev_get(dev); +#endif + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + int ret; + + if (dev_v4) { + /* At some point we might put this check near the ip_rt_send_ + * redirect call of ip_forward in net/ipv4/ip_forward.c, similar + * to the current secpath check. + */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + if (dev_v6) +#ifndef COMPAT_CANNOT_USE_DEV_CNF + dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#else + dev_v6->addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#endif +#endif + + ret = wg_socket_init(wg, wg->incoming_port); + if (ret < 0) + return ret; + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_send_staged_packets(peer); + if (peer->persistent_keepalive_interval) + wg_packet_send_keepalive(peer); + } + mutex_unlock(&wg->device_update_lock); + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int wg_pm_notification(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct wg_device *wg; + struct wg_peer *peer; + + /* If the machine is constantly suspending and resuming, as part of + * its normal operation rather than as a somewhat rare event, then we + * don't actually want to clear keys. + */ + if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) + return 0; + + if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) + return 0; + + rtnl_lock(); + list_for_each_entry(wg, &device_list, device_list) { + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + del_timer(&peer->timer_zero_key_material); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + } + mutex_unlock(&wg->device_update_lock); + } + rtnl_unlock(); + rcu_barrier(); + return 0; +} + +static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; +#endif + +static int wg_stop(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_purge_staged_packets(peer); + wg_timers_stop(peer); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + } + mutex_unlock(&wg->device_update_lock); + skb_queue_purge(&wg->incoming_handshakes); + wg_socket_reinit(wg, NULL, NULL); + return 0; +} + +static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct sk_buff_head packets; + struct wg_peer *peer; + struct sk_buff *next; + sa_family_t family; + u32 mtu; + int ret; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) { + ret = -EPROTONOSUPPORT; + net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); + goto err; + } + + peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); + if (unlikely(!peer)) { + ret = -ENOKEY; + if (skb->protocol == htons(ETH_P_IP)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", + dev->name, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", + dev->name, &ipv6_hdr(skb)->daddr); + goto err; + } + + family = READ_ONCE(peer->endpoint.addr.sa_family); + if (unlikely(family != AF_INET && family != AF_INET6)) { + ret = -EDESTADDRREQ; + net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", + dev->name, peer->internal_id); + goto err_peer; + } + + mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; + + __skb_queue_head_init(&packets); + if (!skb_is_gso(skb)) { + skb_mark_not_on_list(skb); + } else { + struct sk_buff *segs = skb_gso_segment(skb, 0); + + if (unlikely(IS_ERR(segs))) { + ret = PTR_ERR(segs); + goto err_peer; + } + dev_kfree_skb(skb); + skb = segs; + } + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + + skb = skb_share_check(skb, GFP_ATOMIC); + if (unlikely(!skb)) + continue; + + /* We only need to keep the original dst around for icmp, + * so at this point we're in a position to drop it. + */ + skb_dst_drop(skb); + + PACKET_CB(skb)->mtu = mtu; + + __skb_queue_tail(&packets, skb); + } + + spin_lock_bh(&peer->staged_packet_queue.lock); + /* If the queue is getting too big, we start removing the oldest packets + * until it's small again. We do this before adding the new packet, so + * we don't remove GSO segments that are in excess. + */ + while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { + dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); + ++dev->stats.tx_dropped; + } + skb_queue_splice_tail(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + wg_packet_send_staged_packets(peer); + + wg_peer_put(peer); + return NETDEV_TX_OK; + +err_peer: + wg_peer_put(peer); +err: + ++dev->stats.tx_errors; + if (skb->protocol == htons(ETH_P_IP)) + icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); + else if (skb->protocol == htons(ETH_P_IPV6)) + icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); + kfree_skb(skb); + return ret; +} + +static const struct net_device_ops netdev_ops = { + .ndo_open = wg_open, + .ndo_stop = wg_stop, + .ndo_start_xmit = wg_xmit, + .ndo_get_stats64 = ip_tunnel_get_stats64 +}; + +static void wg_destruct(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + + rtnl_lock(); + list_del(&wg->device_list); + rtnl_unlock(); + mutex_lock(&wg->device_update_lock); + wg->incoming_port = 0; + wg_socket_reinit(wg, NULL, NULL); + /* The final references are cleared in the below calls to destroy_workqueue. */ + wg_peer_remove_all(wg); + destroy_workqueue(wg->handshake_receive_wq); + destroy_workqueue(wg->handshake_send_wq); + destroy_workqueue(wg->packet_crypt_wq); + wg_packet_queue_free(&wg->decrypt_queue, true); + wg_packet_queue_free(&wg->encrypt_queue, true); + rcu_barrier(); /* Wait for all the peers to be actually freed. */ + wg_ratelimiter_uninit(); + memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); + skb_queue_purge(&wg->incoming_handshakes); + free_percpu(dev->tstats); + free_percpu(wg->incoming_handshakes_worker); + if (wg->have_creating_net_ref) + put_net(wg->creating_net); + kvfree(wg->index_hashtable); + kvfree(wg->peer_hashtable); + mutex_unlock(&wg->device_update_lock); + + pr_debug("%s: Interface deleted\n", dev->name); + free_netdev(dev); +} + +static const struct device_type device_type = { .name = KBUILD_MODNAME }; + +static void wg_setup(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | + NETIF_F_SG | NETIF_F_GSO | + NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; + + dev->netdev_ops = &netdev_ops; + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->needed_headroom = DATA_PACKET_HEAD_ROOM; + dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); + dev->type = ARPHRD_NONE; + dev->flags = IFF_POINTOPOINT | IFF_NOARP; +#ifndef COMPAT_CANNOT_USE_IFF_NO_QUEUE + dev->priv_flags |= IFF_NO_QUEUE; +#else + dev->tx_queue_len = 0; +#endif + dev->features |= NETIF_F_LLTX; + dev->features |= WG_NETDEV_FEATURES; + dev->hw_features |= WG_NETDEV_FEATURES; + dev->hw_enc_features |= WG_NETDEV_FEATURES; + dev->mtu = ETH_DATA_LEN - MESSAGE_MINIMUM_LENGTH - + sizeof(struct udphdr) - + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + + SET_NETDEV_DEVTYPE(dev, &device_type); + + /* We need to keep the dst around in case of icmp replies. */ + netif_keep_dst(dev); + + memset(wg, 0, sizeof(*wg)); + wg->dev = dev; +} + +static int wg_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct wg_device *wg = netdev_priv(dev); + int ret = -ENOMEM; + + wg->creating_net = src_net; + init_rwsem(&wg->static_identity.lock); + mutex_init(&wg->socket_update_lock); + mutex_init(&wg->device_update_lock); + skb_queue_head_init(&wg->incoming_handshakes); + wg_allowedips_init(&wg->peer_allowedips); + wg_cookie_checker_init(&wg->cookie_checker, wg); + INIT_LIST_HEAD(&wg->peer_list); + wg->device_update_gen = 1; + + wg->peer_hashtable = wg_pubkey_hashtable_alloc(); + if (!wg->peer_hashtable) + return ret; + + wg->index_hashtable = wg_index_hashtable_alloc(); + if (!wg->index_hashtable) + goto err_free_peer_hashtable; + + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_free_index_hashtable; + + wg->incoming_handshakes_worker = + wg_packet_percpu_multicore_worker_alloc( + wg_packet_handshake_receive_worker, wg); + if (!wg->incoming_handshakes_worker) + goto err_free_tstats; + + wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", + WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_receive_wq) + goto err_free_incoming_handshakes; + + wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", + WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_send_wq) + goto err_destroy_handshake_receive; + + wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); + if (!wg->packet_crypt_wq) + goto err_destroy_handshake_send; + + ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_destroy_packet_crypt; + + ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_free_encrypt_queue; + + ret = wg_ratelimiter_init(); + if (ret < 0) + goto err_free_decrypt_queue; + + ret = register_netdevice(dev); + if (ret < 0) + goto err_uninit_ratelimiter; + + list_add(&wg->device_list, &device_list); + + /* We wait until the end to assign priv_destructor, so that + * register_netdevice doesn't call it for us if it fails. + */ + dev->priv_destructor = wg_destruct; + + pr_debug("%s: Interface created\n", dev->name); + return ret; + +err_uninit_ratelimiter: + wg_ratelimiter_uninit(); +err_free_decrypt_queue: + wg_packet_queue_free(&wg->decrypt_queue, true); +err_free_encrypt_queue: + wg_packet_queue_free(&wg->encrypt_queue, true); +err_destroy_packet_crypt: + destroy_workqueue(wg->packet_crypt_wq); +err_destroy_handshake_send: + destroy_workqueue(wg->handshake_send_wq); +err_destroy_handshake_receive: + destroy_workqueue(wg->handshake_receive_wq); +err_free_incoming_handshakes: + free_percpu(wg->incoming_handshakes_worker); +err_free_tstats: + free_percpu(dev->tstats); +err_free_index_hashtable: + kvfree(wg->index_hashtable); +err_free_peer_hashtable: + kvfree(wg->peer_hashtable); + return ret; +} + +static struct rtnl_link_ops link_ops __read_mostly = { + .kind = KBUILD_MODNAME, + .priv_size = sizeof(struct wg_device), + .setup = wg_setup, + .newlink = wg_newlink, +}; + +static int wg_netdevice_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct net_device *dev = ((struct netdev_notifier_info *)data)->dev; + struct wg_device *wg = netdev_priv(dev); + + ASSERT_RTNL(); + + if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops) + return 0; + + if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) { + put_net(wg->creating_net); + wg->have_creating_net_ref = false; + } else if (dev_net(dev) != wg->creating_net && + !wg->have_creating_net_ref) { + wg->have_creating_net_ref = true; + get_net(wg->creating_net); + } + return 0; +} + +static struct notifier_block netdevice_notifier = { + .notifier_call = wg_netdevice_notification +}; + +int __init wg_device_init(void) +{ + int ret; + +#ifdef CONFIG_PM_SLEEP + ret = register_pm_notifier(&pm_notifier); + if (ret) + return ret; +#endif + + ret = register_netdevice_notifier(&netdevice_notifier); + if (ret) + goto error_pm; + + ret = rtnl_link_register(&link_ops); + if (ret) + goto error_netdevice; + + return 0; + +error_netdevice: + unregister_netdevice_notifier(&netdevice_notifier); +error_pm: +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + return ret; +} + +void wg_device_uninit(void) +{ + rtnl_link_unregister(&link_ops); + unregister_netdevice_notifier(&netdevice_notifier); +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + rcu_barrier(); +} diff --git a/net/wireguard/device.h b/net/wireguard/device.h new file mode 100644 index 0000000000000..c91f3051c5c78 --- /dev/null +++ b/net/wireguard/device.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_DEVICE_H +#define _WG_DEVICE_H + +#include "noise.h" +#include "allowedips.h" +#include "peerlookup.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include + +struct wg_device; + +struct multicore_worker { + void *ptr; + struct work_struct work; +}; + +struct crypt_queue { + struct ptr_ring ring; + union { + struct { + struct multicore_worker __percpu *worker; + int last_cpu; + }; + struct work_struct work; + }; +}; + +struct wg_device { + struct net_device *dev; + struct crypt_queue encrypt_queue, decrypt_queue; + struct sock __rcu *sock4, *sock6; + struct net *creating_net; + struct noise_static_identity static_identity; + struct workqueue_struct *handshake_receive_wq, *handshake_send_wq; + struct workqueue_struct *packet_crypt_wq; + struct sk_buff_head incoming_handshakes; + int incoming_handshake_cpu; + struct multicore_worker __percpu *incoming_handshakes_worker; + struct cookie_checker cookie_checker; + struct pubkey_hashtable *peer_hashtable; + struct index_hashtable *index_hashtable; + struct allowedips peer_allowedips; + struct mutex device_update_lock, socket_update_lock; + struct list_head device_list, peer_list; + unsigned int num_peers, device_update_gen; + u32 fwmark; + u16 incoming_port; + bool have_creating_net_ref; +}; + +int wg_device_init(void); +void wg_device_uninit(void); + +/* Later after the dust settles, this can be moved into include/linux/skbuff.h, + * where virtually all code that deals with GSO segs can benefit, around ~30 + * drivers as of writing. + */ +#define skb_list_walk_safe(first, skb, next) \ + for (skb = first, next = skb->next; skb; \ + skb = next, next = skb ? skb->next : NULL) + +#endif /* _WG_DEVICE_H */ diff --git a/net/wireguard/main.c b/net/wireguard/main.c new file mode 100644 index 0000000000000..9ca4195bc9850 --- /dev/null +++ b/net/wireguard/main.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "version.h" +#include "device.h" +#include "noise.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "netlink.h" +#include "uapi/wireguard.h" +#include "crypto/zinc.h" + +#include +#include +#include +#include + +static int __init mod_init(void) +{ + int ret; + + if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) || + (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) || + (ret = curve25519_mod_init())) + return ret; + +#ifdef DEBUG + if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || + !wg_ratelimiter_selftest()) + return -ENOTRECOVERABLE; +#endif + wg_noise_init(); + + ret = wg_device_init(); + if (ret < 0) + goto err_device; + + ret = wg_genetlink_init(); + if (ret < 0) + goto err_netlink; + + pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); + pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + + return 0; + +err_netlink: + wg_device_uninit(); +err_device: + return ret; +} + +static void __exit mod_exit(void) +{ + wg_genetlink_uninit(); + wg_device_uninit(); +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("WireGuard secure network tunnel"); +MODULE_AUTHOR("Jason A. Donenfeld "); +MODULE_VERSION(WIREGUARD_VERSION); +MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); +MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); diff --git a/net/wireguard/messages.h b/net/wireguard/messages.h new file mode 100644 index 0000000000000..f415cdd2e90a9 --- /dev/null +++ b/net/wireguard/messages.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_MESSAGES_H +#define _WG_MESSAGES_H + +#include +#include +#include + +#include +#include +#include + +enum noise_lengths { + NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, + NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, + NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32), + NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, + NOISE_HASH_LEN = BLAKE2S_HASH_SIZE +}; + +#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN) + +enum cookie_values { + COOKIE_SECRET_MAX_AGE = 2 * 60, + COOKIE_SECRET_LATENCY = 5, + COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, + COOKIE_LEN = 16 +}; + +enum counter_values { + COUNTER_BITS_TOTAL = 2048, + COUNTER_REDUNDANT_BITS = BITS_PER_LONG, + COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS +}; + +enum limits { + REKEY_AFTER_MESSAGES = 1ULL << 60, + REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1, + REKEY_TIMEOUT = 5, + REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3, + REKEY_AFTER_TIME = 120, + REJECT_AFTER_TIME = 180, + INITIATIONS_PER_SECOND = 50, + MAX_PEERS_PER_DEVICE = 1U << 20, + KEEPALIVE_TIMEOUT = 10, + MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT, + MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */ + MAX_STAGED_PACKETS = 128, + MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */ +}; + +enum message_type { + MESSAGE_INVALID = 0, + MESSAGE_HANDSHAKE_INITIATION = 1, + MESSAGE_HANDSHAKE_RESPONSE = 2, + MESSAGE_HANDSHAKE_COOKIE = 3, + MESSAGE_DATA = 4 +}; + +struct message_header { + /* The actual layout of this that we want is: + * u8 type + * u8 reserved_zero[3] + * + * But it turns out that by encoding this as little endian, + * we achieve the same thing, and it makes checking faster. + */ + __le32 type; +}; + +struct message_macs { + u8 mac1[COOKIE_LEN]; + u8 mac2[COOKIE_LEN]; +}; + +struct message_handshake_initiation { + struct message_header header; + __le32 sender_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; + u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; + struct message_macs macs; +}; + +struct message_handshake_response { + struct message_header header; + __le32 sender_index; + __le32 receiver_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_nothing[noise_encrypted_len(0)]; + struct message_macs macs; +}; + +struct message_handshake_cookie { + struct message_header header; + __le32 receiver_index; + u8 nonce[COOKIE_NONCE_LEN]; + u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)]; +}; + +struct message_data { + struct message_header header; + __le32 key_idx; + __le64 counter; + u8 encrypted_data[]; +}; + +#define message_data_len(plain_len) \ + (noise_encrypted_len(plain_len) + sizeof(struct message_data)) + +enum message_alignments { + MESSAGE_PADDING_MULTIPLE = 16, + MESSAGE_MINIMUM_LENGTH = message_data_len(0) +}; + +#define SKB_HEADER_LEN \ + (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ + sizeof(struct udphdr) + NET_SKB_PAD) +#define DATA_PACKET_HEAD_ROOM \ + ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4) + +enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ }; + +#endif /* _WG_MESSAGES_H */ diff --git a/net/wireguard/netlink.c b/net/wireguard/netlink.c new file mode 100644 index 0000000000000..9bf2f84edda81 --- /dev/null +++ b/net/wireguard/netlink.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "netlink.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" +#include "uapi/wireguard.h" +#include +#include +#include +#include + +static struct genl_family genl_family; + +static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { + [WGDEVICE_A_IFINDEX] = { .type = NLA_U32 }, + [WGDEVICE_A_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, + [WGDEVICE_A_PRIVATE_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, + [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, + [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, + [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } +}; + +static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { + [WGPEER_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGPEER_A_PRESHARED_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN }, + [WGPEER_A_FLAGS] = { .type = NLA_U32 }, + [WGPEER_A_ENDPOINT] = { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) }, + [WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL] = { .type = NLA_U16 }, + [WGPEER_A_LAST_HANDSHAKE_TIME] = { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) }, + [WGPEER_A_RX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_TX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_ALLOWEDIPS] = { .type = NLA_NESTED }, + [WGPEER_A_PROTOCOL_VERSION] = { .type = NLA_U32 } +}; + +static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { + [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, + [WGALLOWEDIP_A_IPADDR] = { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) }, + [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } +}; + +static struct wg_device *lookup_interface(struct nlattr **attrs, + struct sk_buff *skb) +{ + struct net_device *dev = NULL; + + if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) + return ERR_PTR(-EBADR); + if (attrs[WGDEVICE_A_IFINDEX]) + dev = dev_get_by_index(sock_net(skb->sk), + nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); + else if (attrs[WGDEVICE_A_IFNAME]) + dev = dev_get_by_name(sock_net(skb->sk), + nla_data(attrs[WGDEVICE_A_IFNAME])); + if (!dev) + return ERR_PTR(-ENODEV); + if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || + strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) { + dev_put(dev); + return ERR_PTR(-EOPNOTSUPP); + } + return netdev_priv(dev); +} + +static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, + int family) +{ + struct nlattr *allowedip_nest; + + allowedip_nest = nla_nest_start(skb, 0); + if (!allowedip_nest) + return -EMSGSIZE; + + if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || + nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || + nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? + sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { + nla_nest_cancel(skb, allowedip_nest); + return -EMSGSIZE; + } + + nla_nest_end(skb, allowedip_nest); + return 0; +} + +struct dump_ctx { + struct wg_device *wg; + struct wg_peer *next_peer; + u64 allowedips_seq; + struct allowedips_node *next_allowedip; +}; + +#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) + +static int +get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) +{ + + struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); + struct allowedips_node *allowedips_node = ctx->next_allowedip; + bool fail; + + if (!peer_nest) + return -EMSGSIZE; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, + peer->handshake.remote_static); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (!allowedips_node) { + const struct __kernel_timespec last_handshake = { + .tv_sec = peer->walltime_last_handshake.tv_sec, + .tv_nsec = peer->walltime_last_handshake.tv_nsec + }; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, + NOISE_SYMMETRIC_KEY_LEN, + peer->handshake.preshared_key); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, + sizeof(last_handshake), &last_handshake) || + nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + peer->persistent_keepalive_interval) || + nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) + goto err; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr4), + &peer->endpoint.addr4); + else if (peer->endpoint.addr.sa_family == AF_INET6) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr6), + &peer->endpoint.addr6); + read_unlock_bh(&peer->endpoint_lock); + if (fail) + goto err; + allowedips_node = + list_first_entry_or_null(&peer->allowedips_list, + struct allowedips_node, peer_list); + } + if (!allowedips_node) + goto no_allowedips; + if (!ctx->allowedips_seq) + ctx->allowedips_seq = peer->device->peer_allowedips.seq; + else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq) + goto no_allowedips; + + allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS); + if (!allowedips_nest) + goto err; + + list_for_each_entry_from(allowedips_node, &peer->allowedips_list, + peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family; + + family = wg_allowedips_read_node(allowedips_node, ip, &cidr); + if (get_allowedips(skb, ip, cidr, family)) { + nla_nest_end(skb, allowedips_nest); + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = allowedips_node; + return -EMSGSIZE; + } + } + nla_nest_end(skb, allowedips_nest); +no_allowedips: + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = NULL; + ctx->allowedips_seq = 0; + return 0; +err: + nla_nest_cancel(skb, peer_nest); + return -EMSGSIZE; +} + +static int wg_get_device_start(struct netlink_callback *cb) +{ + struct wg_device *wg; + + wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); + if (IS_ERR(wg)) + return PTR_ERR(wg); + DUMP_CTX(cb)->wg = wg; + return 0; +} + +static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct wg_peer *peer, *next_peer_cursor; + struct dump_ctx *ctx = DUMP_CTX(cb); + struct wg_device *wg = ctx->wg; + struct nlattr *peers_nest; + int ret = -EMSGSIZE; + bool done = true; + void *hdr; + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + cb->seq = wg->device_update_gen; + next_peer_cursor = ctx->next_peer; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); + if (!hdr) + goto out; + genl_dump_check_consistent(cb, hdr); + + if (!ctx->next_peer) { + if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, + wg->incoming_port) || + nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || + nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || + nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) + goto out; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity) { + if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_private) || + nla_put(skb, WGDEVICE_A_PUBLIC_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_public)) { + up_read(&wg->static_identity.lock); + goto out; + } + } + up_read(&wg->static_identity.lock); + } + + peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); + if (!peers_nest) + goto out; + ret = 0; + /* If the last cursor was removed via list_del_init in peer_remove, then + * we just treat this the same as there being no more peers left. The + * reason is that seq_nr should indicate to userspace that this isn't a + * coherent dump anyway, so they'll try again. + */ + if (list_empty(&wg->peer_list) || + (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) { + nla_nest_cancel(skb, peers_nest); + goto out; + } + lockdep_assert_held(&wg->device_update_lock); + peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list); + list_for_each_entry_continue(peer, &wg->peer_list, peer_list) { + if (get_peer(peer, skb, ctx)) { + done = false; + break; + } + next_peer_cursor = peer; + } + nla_nest_end(skb, peers_nest); + +out: + if (!ret && !done && next_peer_cursor) + wg_peer_get(next_peer_cursor); + wg_peer_put(ctx->next_peer); + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + + if (ret) { + genlmsg_cancel(skb, hdr); + return ret; + } + genlmsg_end(skb, hdr); + if (done) { + ctx->next_peer = NULL; + return 0; + } + ctx->next_peer = next_peer_cursor; + return skb->len; + + /* At this point, we can't really deal ourselves with safely zeroing out + * the private key material after usage. This will need an additional API + * in the kernel for marking skbs as zero_on_free. + */ +} + +static int wg_get_device_done(struct netlink_callback *cb) +{ + struct dump_ctx *ctx = DUMP_CTX(cb); + + if (ctx->wg) + dev_put(ctx->wg->dev); + wg_peer_put(ctx->next_peer); + return 0; +} + +static int set_port(struct wg_device *wg, u16 port) +{ + struct wg_peer *peer; + + if (wg->incoming_port == port) + return 0; + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + if (!netif_running(wg->dev)) { + wg->incoming_port = port; + return 0; + } + return wg_socket_init(wg, port); +} + +static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) +{ + int ret = -EINVAL; + u16 family; + u8 cidr; + + if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || + !attrs[WGALLOWEDIP_A_CIDR_MASK]) + return ret; + family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); + cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); + + if (family == AF_INET && cidr <= 32 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) + ret = wg_allowedips_insert_v4( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else if (family == AF_INET6 && cidr <= 128 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) + ret = wg_allowedips_insert_v6( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + + return ret; +} + +static int set_peer(struct wg_device *wg, struct nlattr **attrs) +{ + u8 *public_key = NULL, *preshared_key = NULL; + struct wg_peer *peer = NULL; + u32 flags = 0; + int ret; + + ret = -EINVAL; + if (attrs[WGPEER_A_PUBLIC_KEY] && + nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) + public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]); + else + goto out; + if (attrs[WGPEER_A_PRESHARED_KEY] && + nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN) + preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]); + + if (attrs[WGPEER_A_FLAGS]) + flags = nla_get_u32(attrs[WGPEER_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGPEER_F_ALL) + goto out; + + ret = -EPFNOSUPPORT; + if (attrs[WGPEER_A_PROTOCOL_VERSION]) { + if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1) + goto out; + } + + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + nla_data(attrs[WGPEER_A_PUBLIC_KEY])); + ret = 0; + if (!peer) { /* Peer doesn't exist yet. Add a new one. */ + if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) + goto out; + + /* The peer is new, so there aren't allowed IPs to remove. */ + flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity && + !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), + wg->static_identity.static_public, + NOISE_PUBLIC_KEY_LEN)) { + /* We silently ignore peers that have the same public + * key as the device. The reason we do it silently is + * that we'd like for people to be able to reuse the + * same set of API calls across peers. + */ + up_read(&wg->static_identity.lock); + ret = 0; + goto out; + } + up_read(&wg->static_identity.lock); + + peer = wg_peer_create(wg, public_key, preshared_key); + if (IS_ERR(peer)) { + /* Similar to the above, if the key is invalid, we skip + * it without fanfare, so that services don't need to + * worry about doing key validation themselves. + */ + ret = PTR_ERR(peer) == -EKEYREJECTED ? 0 : PTR_ERR(peer); + peer = NULL; + goto out; + } + /* Take additional reference, as though we've just been + * looked up. + */ + wg_peer_get(peer); + } + + if (flags & WGPEER_F_REMOVE_ME) { + wg_peer_remove(peer); + goto out; + } + + if (preshared_key) { + down_write(&peer->handshake.lock); + memcpy(&peer->handshake.preshared_key, preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + up_write(&peer->handshake.lock); + } + + if (attrs[WGPEER_A_ENDPOINT]) { + struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); + size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); + + if ((len == sizeof(struct sockaddr_in) && + addr->sa_family == AF_INET) || + (len == sizeof(struct sockaddr_in6) && + addr->sa_family == AF_INET6)) { + struct endpoint endpoint = { { { 0 } } }; + + memcpy(&endpoint.addr, addr, len); + wg_socket_set_peer_endpoint(peer, &endpoint); + } + } + + if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) + wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, + &wg->device_update_lock); + + if (attrs[WGPEER_A_ALLOWEDIPS]) { + struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { + ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, + attr, allowedip_policy, NULL); + if (ret < 0) + goto out; + ret = set_allowedip(peer, allowedip); + if (ret < 0) + goto out; + } + } + + if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { + const u16 persistent_keepalive_interval = nla_get_u16( + attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); + const bool send_keepalive = + !peer->persistent_keepalive_interval && + persistent_keepalive_interval && + netif_running(wg->dev); + + peer->persistent_keepalive_interval = persistent_keepalive_interval; + if (send_keepalive) + wg_packet_send_keepalive(peer); + } + + if (netif_running(wg->dev)) + wg_packet_send_staged_packets(peer); + +out: + wg_peer_put(peer); + if (attrs[WGPEER_A_PRESHARED_KEY]) + memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), + nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + return ret; +} + +static int wg_set_device(struct sk_buff *skb, struct genl_info *info) +{ + struct wg_device *wg = lookup_interface(info->attrs, skb); + u32 flags = 0; + int ret; + + if (IS_ERR(wg)) { + ret = PTR_ERR(wg); + goto out_nodev; + } + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + + if (info->attrs[WGDEVICE_A_FLAGS]) + flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGDEVICE_F_ALL) + goto out; + + ret = -EPERM; + if ((info->attrs[WGDEVICE_A_LISTEN_PORT] || + info->attrs[WGDEVICE_A_FWMARK]) && + !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN)) + goto out; + + ++wg->device_update_gen; + + if (info->attrs[WGDEVICE_A_FWMARK]) { + struct wg_peer *peer; + + wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + } + + if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { + ret = set_port(wg, + nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); + if (ret) + goto out; + } + + if (flags & WGDEVICE_F_REPLACE_PEERS) + wg_peer_remove_all(wg); + + if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == + NOISE_PUBLIC_KEY_LEN) { + u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); + u8 public_key[NOISE_PUBLIC_KEY_LEN]; + struct wg_peer *peer, *temp; + + if (!crypto_memneq(wg->static_identity.static_private, + private_key, NOISE_PUBLIC_KEY_LEN)) + goto skip_set_private_key; + + /* We remove before setting, to prevent race, which means doing + * two 25519-genpub ops. + */ + if (curve25519_generate_public(public_key, private_key)) { + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + public_key); + if (peer) { + wg_peer_put(peer); + wg_peer_remove(peer); + } + } + + down_write(&wg->static_identity.lock); + wg_noise_set_static_identity_private_key(&wg->static_identity, + private_key); + list_for_each_entry_safe(peer, temp, &wg->peer_list, + peer_list) { + if (wg_noise_precompute_static_static(peer)) + wg_noise_expire_current_peer_keypairs(peer); + else + wg_peer_remove(peer); + } + wg_cookie_checker_precompute_device_keys(&wg->cookie_checker); + up_write(&wg->static_identity.lock); + } +skip_set_private_key: + + if (info->attrs[WGDEVICE_A_PEERS]) { + struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { + ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, + peer_policy, NULL); + if (ret < 0) + goto out; + ret = set_peer(wg, peer); + if (ret < 0) + goto out; + } + } + ret = 0; + +out: + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + dev_put(wg->dev); +out_nodev: + if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) + memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + return ret; +} + +#ifndef COMPAT_CANNOT_USE_CONST_GENL_OPS +static const +#else +static +#endif +struct genl_ops genl_ops[] = { + { + .cmd = WG_CMD_GET_DEVICE, +#ifndef COMPAT_CANNOT_USE_NETLINK_START + .start = wg_get_device_start, +#endif + .dumpit = wg_get_device_dump, + .done = wg_get_device_done, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + }, { + .cmd = WG_CMD_SET_DEVICE, + .doit = wg_set_device, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + } +}; + +static struct genl_family genl_family +#ifndef COMPAT_CANNOT_USE_GENL_NOPS +__ro_after_init = { + .ops = genl_ops, + .n_ops = ARRAY_SIZE(genl_ops), +#else += { +#endif + .name = WG_GENL_NAME, + .version = WG_GENL_VERSION, + .maxattr = WGDEVICE_A_MAX, + .module = THIS_MODULE, +#ifndef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .netnsok = true +}; + +int __init wg_genetlink_init(void) +{ + return genl_register_family(&genl_family); +} + +void __exit wg_genetlink_uninit(void) +{ + genl_unregister_family(&genl_family); +} diff --git a/net/wireguard/netlink.h b/net/wireguard/netlink.h new file mode 100644 index 0000000000000..15100d92e2e32 --- /dev/null +++ b/net/wireguard/netlink.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_NETLINK_H +#define _WG_NETLINK_H + +int wg_genetlink_init(void); +void wg_genetlink_uninit(void); + +#endif /* _WG_NETLINK_H */ diff --git a/net/wireguard/noise.c b/net/wireguard/noise.c new file mode 100644 index 0000000000000..269b69faf11a4 --- /dev/null +++ b/net/wireguard/noise.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "noise.h" +#include "device.h" +#include "peer.h" +#include "messages.h" +#include "queueing.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +/* This implements Noise_IKpsk2: + * + * <- s + * ****** + * -> e, es, s, ss, {t} + * <- e, ee, se, psk, {} + */ + +static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; +static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; +static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; +static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; +static atomic64_t keypair_counter = ATOMIC64_INIT(0); + +void __init wg_noise_init(void) +{ + struct blake2s_state blake; + + blake2s(handshake_init_chaining_key, handshake_name, NULL, + NOISE_HASH_LEN, sizeof(handshake_name), 0); + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s_update(&blake, identifier_name, sizeof(identifier_name)); + blake2s_final(&blake, handshake_init_hash); +} + +/* Must hold peer->handshake.static_identity->lock */ +bool wg_noise_precompute_static_static(struct wg_peer *peer) +{ + bool ret = true; + + down_write(&peer->handshake.lock); + if (peer->handshake.static_identity->has_identity) + ret = curve25519( + peer->handshake.precomputed_static_static, + peer->handshake.static_identity->static_private, + peer->handshake.remote_static); + else + memset(peer->handshake.precomputed_static_static, 0, + NOISE_PUBLIC_KEY_LEN); + up_write(&peer->handshake.lock); + return ret; +} + +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer) +{ + memset(handshake, 0, sizeof(*handshake)); + init_rwsem(&handshake->lock); + handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; + handshake->entry.peer = peer; + memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); + if (peer_preshared_key) + memcpy(handshake->preshared_key, peer_preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + handshake->static_identity = static_identity; + handshake->state = HANDSHAKE_ZEROED; + return wg_noise_precompute_static_static(peer); +} + +static void handshake_zero(struct noise_handshake *handshake) +{ + memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->hash, 0, NOISE_HASH_LEN); + memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); + handshake->remote_index = 0; + handshake->state = HANDSHAKE_ZEROED; +} + +void wg_noise_handshake_clear(struct noise_handshake *handshake) +{ + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + down_write(&handshake->lock); + handshake_zero(handshake); + up_write(&handshake->lock); + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); +} + +static struct noise_keypair *keypair_create(struct wg_peer *peer) +{ + struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); + + if (unlikely(!keypair)) + return NULL; + keypair->internal_id = atomic64_inc_return(&keypair_counter); + keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; + keypair->entry.peer = peer; + kref_init(&keypair->refcount); + return keypair; +} + +static void keypair_free_rcu(struct rcu_head *rcu) +{ + kzfree(container_of(rcu, struct noise_keypair, rcu)); +} + +static void keypair_free_kref(struct kref *kref) +{ + struct noise_keypair *keypair = + container_of(kref, struct noise_keypair, refcount); + + net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", + keypair->entry.peer->device->dev->name, + keypair->internal_id, + keypair->entry.peer->internal_id); + wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, + &keypair->entry); + call_rcu(&keypair->rcu, keypair_free_rcu); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) +{ + if (unlikely(!keypair)) + return; + if (unlikely(unreference_now)) + wg_index_hashtable_remove( + keypair->entry.peer->device->index_hashtable, + &keypair->entry); + kref_put(&keypair->refcount, keypair_free_kref); +} + +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking noise keypair reference without holding the RCU BH read lock"); + if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) + return NULL; + return keypair; +} + +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) +{ + struct noise_keypair *old; + + spin_lock_bh(&keypairs->keypair_update_lock); + + /* We zero the next_keypair before zeroing the others, so that + * wg_noise_received_with_keypair returns early before subsequent ones + * are zeroed. + */ + old = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->current_keypair, NULL); + wg_noise_keypair_put(old, true); + + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + + wg_noise_handshake_clear(&peer->handshake); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + + spin_lock_bh(&peer->keypairs.keypair_update_lock); + keypair = rcu_dereference_protected(peer->keypairs.next_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + keypair = rcu_dereference_protected(peer->keypairs.current_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + spin_unlock_bh(&peer->keypairs.keypair_update_lock); +} + +static void add_new_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *new_keypair) +{ + struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; + + spin_lock_bh(&keypairs->keypair_update_lock); + previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + next_keypair = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + current_keypair = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + if (new_keypair->i_am_the_initiator) { + /* If we're the initiator, it means we've sent a handshake, and + * received a confirmation response, which means this new + * keypair can now be used. + */ + if (next_keypair) { + /* If there already was a next keypair pending, we + * demote it to be the previous keypair, and free the + * existing current. Note that this means KCI can result + * in this transition. It would perhaps be more sound to + * always just get rid of the unused next keypair + * instead of putting it in the previous slot, but this + * might be a bit less robust. Something to think about + * for the future. + */ + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + rcu_assign_pointer(keypairs->previous_keypair, + next_keypair); + wg_noise_keypair_put(current_keypair, true); + } else /* If there wasn't an existing next keypair, we replace + * the previous with the current one. + */ + rcu_assign_pointer(keypairs->previous_keypair, + current_keypair); + /* At this point we can get rid of the old previous keypair, and + * set up the new keypair. + */ + wg_noise_keypair_put(previous_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, new_keypair); + } else { + /* If we're the responder, it means we can't use the new keypair + * until we receive confirmation via the first data packet, so + * we get rid of the existing previous one, the possibly + * existing next one, and slide in the new next one. + */ + rcu_assign_pointer(keypairs->next_keypair, new_keypair); + wg_noise_keypair_put(next_keypair, true); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(previous_keypair, true); + } + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair) +{ + struct noise_keypair *old_keypair; + bool key_is_new; + + /* We first check without taking the spinlock. */ + key_is_new = received_keypair == + rcu_access_pointer(keypairs->next_keypair); + if (likely(!key_is_new)) + return false; + + spin_lock_bh(&keypairs->keypair_update_lock); + /* After locking, we double check that things didn't change from + * beneath us. + */ + if (unlikely(received_keypair != + rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)))) { + spin_unlock_bh(&keypairs->keypair_update_lock); + return false; + } + + /* When we've finally received the confirmation, we slide the next + * into the current, the current into the previous, and get rid of + * the old previous. + */ + old_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + rcu_assign_pointer(keypairs->previous_keypair, + rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock))); + wg_noise_keypair_put(old_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, received_keypair); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + + spin_unlock_bh(&keypairs->keypair_update_lock); + return true; +} + +/* Must hold static_identity->lock */ +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(static_identity->static_private, private_key, + NOISE_PUBLIC_KEY_LEN); + curve25519_clamp_secret(static_identity->static_private); + static_identity->has_identity = curve25519_generate_public( + static_identity->static_public, private_key); +} + +/* This is Hugo Krawczyk's HKDF: + * - https://eprint.iacr.org/2010/264.pdf + * - https://tools.ietf.org/html/rfc5869 + */ +static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, + size_t first_len, size_t second_len, size_t third_len, + size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) +{ + u8 output[BLAKE2S_HASH_SIZE + 1]; + u8 secret[BLAKE2S_HASH_SIZE]; + + WARN_ON(IS_ENABLED(DEBUG) && + (first_len > BLAKE2S_HASH_SIZE || + second_len > BLAKE2S_HASH_SIZE || + third_len > BLAKE2S_HASH_SIZE || + ((second_len || second_dst || third_len || third_dst) && + (!first_len || !first_dst)) || + ((third_len || third_dst) && (!second_len || !second_dst)))); + + /* Extract entropy from data into secret */ + blake2s_hmac(secret, data, chaining_key, BLAKE2S_HASH_SIZE, data_len, + NOISE_HASH_LEN); + + if (!first_dst || !first_len) + goto out; + + /* Expand first key: key = secret, data = 0x1 */ + output[0] = 1; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, 1, + BLAKE2S_HASH_SIZE); + memcpy(first_dst, output, first_len); + + if (!second_dst || !second_len) + goto out; + + /* Expand second key: key = secret, data = first-key || 0x2 */ + output[BLAKE2S_HASH_SIZE] = 2; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(second_dst, output, second_len); + + if (!third_dst || !third_len) + goto out; + + /* Expand third key: key = secret, data = second-key || 0x3 */ + output[BLAKE2S_HASH_SIZE] = 3; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(third_dst, output, third_len); + +out: + /* Clear sensitive data from stack */ + memzero_explicit(secret, BLAKE2S_HASH_SIZE); + memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); +} + +static void symmetric_key_init(struct noise_symmetric_key *key) +{ + spin_lock_init(&key->counter.receive.lock); + atomic64_set(&key->counter.counter, 0); + memset(key->counter.receive.backtrack, 0, + sizeof(key->counter.receive.backtrack)); + key->birthdate = ktime_get_coarse_boottime_ns(); + key->is_valid = true; +} + +static void derive_keys(struct noise_symmetric_key *first_dst, + struct noise_symmetric_key *second_dst, + const u8 chaining_key[NOISE_HASH_LEN]) +{ + kdf(first_dst->key, second_dst->key, NULL, NULL, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, + chaining_key); + symmetric_key_init(first_dst); + symmetric_key_init(second_dst); +} + +static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 private[NOISE_PUBLIC_KEY_LEN], + const u8 public[NOISE_PUBLIC_KEY_LEN]) +{ + u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; + + if (unlikely(!curve25519(dh_calculation, private, public))) + return false; + kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); + memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); + return true; +} + +static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, hash, NOISE_HASH_LEN); + blake2s_update(&blake, src, src_len); + blake2s_final(&blake, hash); +} + +static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) +{ + u8 temp_hash[NOISE_HASH_LEN]; + + kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); + mix_hash(hash, temp_hash, NOISE_HASH_LEN); + memzero_explicit(temp_hash, NOISE_HASH_LEN); +} + +static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN], + const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); + mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); +} + +static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, + NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key); + mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); +} + +static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, + hash, NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key)) + return false; + mix_hash(hash, src_ciphertext, src_len); + return true; +} + +static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], + const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], + u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (ephemeral_dst != ephemeral_src) + memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, + NOISE_PUBLIC_KEY_LEN, chaining_key); +} + +static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) +{ + struct timespec64 now; + + ktime_get_real_ts64(&now); + + /* In order to prevent some sort of infoleak from precise timers, we + * round down the nanoseconds part to the closest rounded-down power of + * two to the maximum initiations per second allowed anyway by the + * implementation. + */ + now.tv_nsec = ALIGN_DOWN(now.tv_nsec, + rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); + + /* https://cr.yp.to/libtai/tai64.html */ + *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); + *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); +} + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake) +{ + u8 timestamp[NOISE_TIMESTAMP_LEN]; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (unlikely(!handshake->static_identity->has_identity)) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); + + handshake_init(handshake->chaining_key, handshake->hash, + handshake->remote_static); + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* es */ + if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* s */ + message_encrypt(dst->encrypted_static, + handshake->static_identity->static_public, + NOISE_PUBLIC_KEY_LEN, key, handshake->hash); + + /* ss */ + kdf(handshake->chaining_key, key, NULL, + handshake->precomputed_static_static, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + handshake->chaining_key); + + /* {t} */ + tai64n_now(timestamp); + message_encrypt(dst->encrypted_timestamp, timestamp, + NOISE_TIMESTAMP_LEN, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_INITIATION; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + bool replay_attack, flood_attack; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 s[NOISE_PUBLIC_KEY_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 t[NOISE_TIMESTAMP_LEN]; + u64 initiation_consumption; + + down_read(&wg->static_identity.lock); + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake_init(chaining_key, hash, wg->static_identity.static_public); + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* es */ + if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) + goto out; + + /* s */ + if (!message_decrypt(s, src->encrypted_static, + sizeof(src->encrypted_static), key, hash)) + goto out; + + /* Lookup which peer we're actually talking to */ + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); + if (!peer) + goto out; + handshake = &peer->handshake; + + /* ss */ + kdf(chaining_key, key, NULL, handshake->precomputed_static_static, + NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + chaining_key); + + /* {t} */ + if (!message_decrypt(t, src->encrypted_timestamp, + sizeof(src->encrypted_timestamp), key, hash)) + goto out; + + down_read(&handshake->lock); + replay_attack = memcmp(t, handshake->latest_timestamp, + NOISE_TIMESTAMP_LEN) <= 0; + flood_attack = (s64)handshake->last_initiation_consumption + + NSEC_PER_SEC / INITIATIONS_PER_SECOND > + (s64)ktime_get_coarse_boottime_ns(); + up_read(&handshake->lock); + if (replay_attack || flood_attack) + goto out; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) + memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + if ((s64)(handshake->last_initiation_consumption - + (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0) + handshake->last_initiation_consumption = initiation_consumption; + handshake->state = HANDSHAKE_CONSUMED_INITIATION; + up_write(&handshake->lock); + ret_peer = peer; + +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + up_read(&wg->static_identity.lock); + if (!ret_peer) + wg_peer_put(peer); + return ret_peer; +} + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake) +{ + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); + dst->receiver_index = handshake->remote_index; + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* ee */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_ephemeral)) + goto out; + + /* se */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* psk */ + mix_psk(handshake->chaining_key, handshake->hash, key, + handshake->preshared_key); + + /* {} */ + message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_RESPONSE; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg) +{ + enum noise_handshake_state state = HANDSHAKE_ZEROED; + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + + down_read(&wg->static_identity.lock); + + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake = (struct noise_handshake *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, + src->receiver_index, &peer); + if (unlikely(!handshake)) + goto out; + + down_read(&handshake->lock); + state = handshake->state; + memcpy(hash, handshake->hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); + memcpy(ephemeral_private, handshake->ephemeral_private, + NOISE_PUBLIC_KEY_LEN); + up_read(&handshake->lock); + + if (state != HANDSHAKE_CREATED_INITIATION) + goto fail; + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* ee */ + if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + goto fail; + + /* se */ + if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) + goto fail; + + /* psk */ + mix_psk(chaining_key, hash, key, handshake->preshared_key); + + /* {} */ + if (!message_decrypt(NULL, src->encrypted_nothing, + sizeof(src->encrypted_nothing), key, hash)) + goto fail; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + /* It's important to check that the state is still the same, while we + * have an exclusive lock. + */ + if (handshake->state != state) { + up_write(&handshake->lock); + goto fail; + } + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + handshake->state = HANDSHAKE_CONSUMED_RESPONSE; + up_write(&handshake->lock); + ret_peer = peer; + goto out; + +fail: + wg_peer_put(peer); +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); + memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); + up_read(&wg->static_identity.lock); + return ret_peer; +} + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs) +{ + struct noise_keypair *new_keypair; + bool ret = false; + + down_write(&handshake->lock); + if (handshake->state != HANDSHAKE_CREATED_RESPONSE && + handshake->state != HANDSHAKE_CONSUMED_RESPONSE) + goto out; + + new_keypair = keypair_create(handshake->entry.peer); + if (!new_keypair) + goto out; + new_keypair->i_am_the_initiator = handshake->state == + HANDSHAKE_CONSUMED_RESPONSE; + new_keypair->remote_index = handshake->remote_index; + + if (new_keypair->i_am_the_initiator) + derive_keys(&new_keypair->sending, &new_keypair->receiving, + handshake->chaining_key); + else + derive_keys(&new_keypair->receiving, &new_keypair->sending, + handshake->chaining_key); + + handshake_zero(handshake); + rcu_read_lock_bh(); + if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, + handshake)->is_dead))) { + add_new_keypair(keypairs, new_keypair); + net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", + handshake->entry.peer->device->dev->name, + new_keypair->internal_id, + handshake->entry.peer->internal_id); + ret = wg_index_hashtable_replace( + handshake->entry.peer->device->index_hashtable, + &handshake->entry, &new_keypair->entry); + } else { + kzfree(new_keypair); + } + rcu_read_unlock_bh(); + +out: + up_write(&handshake->lock); + return ret; +} diff --git a/net/wireguard/noise.h b/net/wireguard/noise.h new file mode 100644 index 0000000000000..138a07bb817ce --- /dev/null +++ b/net/wireguard/noise.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ +#ifndef _WG_NOISE_H +#define _WG_NOISE_H + +#include "messages.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +union noise_counter { + struct { + u64 counter; + unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG]; + spinlock_t lock; + } receive; + atomic64_t counter; +}; + +struct noise_symmetric_key { + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + union noise_counter counter; + u64 birthdate; + bool is_valid; +}; + +struct noise_keypair { + struct index_hashtable_entry entry; + struct noise_symmetric_key sending; + struct noise_symmetric_key receiving; + __le32 remote_index; + bool i_am_the_initiator; + struct kref refcount; + struct rcu_head rcu; + u64 internal_id; +}; + +struct noise_keypairs { + struct noise_keypair __rcu *current_keypair; + struct noise_keypair __rcu *previous_keypair; + struct noise_keypair __rcu *next_keypair; + spinlock_t keypair_update_lock; +}; + +struct noise_static_identity { + u8 static_public[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + struct rw_semaphore lock; + bool has_identity; +}; + +enum noise_handshake_state { + HANDSHAKE_ZEROED, + HANDSHAKE_CREATED_INITIATION, + HANDSHAKE_CONSUMED_INITIATION, + HANDSHAKE_CREATED_RESPONSE, + HANDSHAKE_CONSUMED_RESPONSE +}; + +struct noise_handshake { + struct index_hashtable_entry entry; + + enum noise_handshake_state state; + u64 last_initiation_consumption; + + struct noise_static_identity *static_identity; + + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 remote_static[NOISE_PUBLIC_KEY_LEN]; + u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN]; + + u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; + + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + + u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; + __le32 remote_index; + + /* Protects all members except the immutable (after noise_handshake_ + * init): remote_static, precomputed_static_static, static_identity. + */ + struct rw_semaphore lock; +}; + +struct wg_device; + +void wg_noise_init(void); +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer); +void wg_noise_handshake_clear(struct noise_handshake *handshake); +static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns) +{ + atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() - + (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now); +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair); +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs); +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair); +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer); + +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]); +bool wg_noise_precompute_static_static(struct wg_peer *peer); + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg); + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg); + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs); + +#endif /* _WG_NOISE_H */ diff --git a/net/wireguard/peer.c b/net/wireguard/peer.c new file mode 100644 index 0000000000000..071eedf33f5aa --- /dev/null +++ b/net/wireguard/peer.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peer.h" +#include "device.h" +#include "queueing.h" +#include "timers.h" +#include "peerlookup.h" +#include "noise.h" + +#include +#include +#include +#include + +static atomic64_t peer_counter = ATOMIC64_INIT(0); + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) +{ + struct wg_peer *peer; + int ret = -ENOMEM; + + lockdep_assert_held(&wg->device_update_lock); + + if (wg->num_peers >= MAX_PEERS_PER_DEVICE) + return ERR_PTR(ret); + + peer = kzalloc(sizeof(*peer), GFP_KERNEL); + if (unlikely(!peer)) + return ERR_PTR(ret); + peer->device = wg; + + if (!wg_noise_handshake_init(&peer->handshake, &wg->static_identity, + public_key, preshared_key, peer)) { + ret = -EKEYREJECTED; + goto err_1; + } + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) + goto err_1; + if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, + MAX_QUEUED_PACKETS)) + goto err_2; + if (wg_packet_queue_init(&peer->rx_queue, NULL, false, + MAX_QUEUED_PACKETS)) + goto err_3; + + peer->internal_id = atomic64_inc_return(&peer_counter); + peer->serial_work_cpu = nr_cpumask_bits; + wg_cookie_init(&peer->latest_cookie); + wg_timers_init(peer); + wg_cookie_checker_precompute_peer_keys(peer); + spin_lock_init(&peer->keypairs.keypair_update_lock); + INIT_WORK(&peer->transmit_handshake_work, + wg_packet_handshake_send_worker); + rwlock_init(&peer->endpoint_lock); + kref_init(&peer->refcount); + skb_queue_head_init(&peer->staged_packet_queue); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); + netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, + NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); + list_add_tail(&peer->peer_list, &wg->peer_list); + INIT_LIST_HEAD(&peer->allowedips_list); + wg_pubkey_hashtable_add(wg->peer_hashtable, peer); + ++wg->num_peers; + pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); + return peer; + +err_3: + wg_packet_queue_free(&peer->tx_queue, false); +err_2: + dst_cache_destroy(&peer->endpoint_cache); +err_1: + kfree(peer); + return ERR_PTR(ret); +} + +struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking peer reference without holding the RCU read lock"); + if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) + return NULL; + return peer; +} + +static void peer_make_dead(struct wg_peer *peer) +{ + /* Remove from configuration-time lookup structures. */ + list_del_init(&peer->peer_list); + wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, + &peer->device->device_update_lock); + wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); + + /* Mark as dead, so that we don't allow jumping contexts after. */ + WRITE_ONCE(peer->is_dead, true); + + /* The caller must now synchronize_rcu() for this to take effect. */ +} + +static void peer_remove_after_dead(struct wg_peer *peer) +{ + WARN_ON(!peer->is_dead); + + /* No more keypairs can be created for this peer, since is_dead protects + * add_new_keypair, so we can now destroy existing ones. + */ + wg_noise_keypairs_clear(&peer->keypairs); + + /* Destroy all ongoing timers that were in-flight at the beginning of + * this function. + */ + wg_timers_stop(peer); + + /* The transition between packet encryption/decryption queues isn't + * guarded by is_dead, but each reference's life is strictly bounded by + * two generations: once for parallel crypto and once for serial + * ingestion, so we can simply flush twice, and be sure that we no + * longer have references inside these queues. + */ + + /* a) For encrypt/decrypt. */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.1) For send (but not receive, since that's napi). */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.2.1) For receive (but not send, since that's wq). */ + napi_disable(&peer->napi); + /* b.2.1) It's now safe to remove the napi struct, which must be done + * here from process context. + */ + netif_napi_del(&peer->napi); + + /* Ensure any workstructs we own (like transmit_handshake_work or + * clear_peer_work) no longer are in use. + */ + flush_workqueue(peer->device->handshake_send_wq); + + /* After the above flushes, a peer might still be active in a few + * different contexts: 1) from xmit(), before hitting is_dead and + * returning, 2) from wg_packet_consume_data(), before hitting is_dead + * and returning, 3) from wg_receive_handshake_packet() after a point + * where it has processed an incoming handshake packet, but where + * all calls to pass it off to timers fails because of is_dead. We won't + * have new references in (1) eventually, because we're removed from + * allowedips; we won't have new references in (2) eventually, because + * wg_index_hashtable_lookup will always return NULL, since we removed + * all existing keypairs and no more can be created; we won't have new + * references in (3) eventually, because we're removed from the pubkey + * hash table, which allows for a maximum of one handshake response, + * via the still-uncleared index hashtable entry, but not more than one, + * and in wg_cookie_message_consume, the lookup eventually gets a peer + * with a refcount of zero, so no new reference is taken. + */ + + --peer->device->num_peers; + wg_peer_put(peer); +} + +/* We have a separate "remove" function make sure that all active places where + * a peer is currently operating will eventually come to an end and not pass + * their reference onto another context. + */ +void wg_peer_remove(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + lockdep_assert_held(&peer->device->device_update_lock); + + peer_make_dead(peer); + synchronize_rcu(); + peer_remove_after_dead(peer); +} + +void wg_peer_remove_all(struct wg_device *wg) +{ + struct wg_peer *peer, *temp; + LIST_HEAD(dead_peers); + + lockdep_assert_held(&wg->device_update_lock); + + /* Avoid having to traverse individually for each one. */ + wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); + + list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { + peer_make_dead(peer); + list_add_tail(&peer->peer_list, &dead_peers); + } + synchronize_rcu(); + list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) + peer_remove_after_dead(peer); +} + +static void rcu_release(struct rcu_head *rcu) +{ + struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); + + dst_cache_destroy(&peer->endpoint_cache); + wg_packet_queue_free(&peer->rx_queue, false); + wg_packet_queue_free(&peer->tx_queue, false); + + /* The final zeroing takes care of clearing any remaining handshake key + * material and other potentially sensitive information. + */ + kzfree(peer); +} + +static void kref_release(struct kref *refcount) +{ + struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); + + pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + /* Remove ourself from dynamic runtime lookup structures, now that the + * last reference is gone. + */ + wg_index_hashtable_remove(peer->device->index_hashtable, + &peer->handshake.entry); + + /* Remove any lingering packets that didn't have a chance to be + * transmitted. + */ + wg_packet_purge_staged_packets(peer); + + /* Free the memory used. */ + call_rcu(&peer->rcu, rcu_release); +} + +void wg_peer_put(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + kref_put(&peer->refcount, kref_release); +} diff --git a/net/wireguard/peer.h b/net/wireguard/peer.h new file mode 100644 index 0000000000000..23af409229972 --- /dev/null +++ b/net/wireguard/peer.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEER_H +#define _WG_PEER_H + +#include "device.h" +#include "noise.h" +#include "cookie.h" + +#include +#include +#include +#include +#include + +struct wg_device; + +struct endpoint { + union { + struct sockaddr addr; + struct sockaddr_in addr4; + struct sockaddr_in6 addr6; + }; + union { + struct { + struct in_addr src4; + /* Essentially the same as addr6->scope_id */ + int src_if4; + }; + struct in6_addr src6; + }; +}; + +struct wg_peer { + struct wg_device *device; + struct crypt_queue tx_queue, rx_queue; + struct sk_buff_head staged_packet_queue; + int serial_work_cpu; + struct noise_keypairs keypairs; + struct endpoint endpoint; + struct dst_cache endpoint_cache; + rwlock_t endpoint_lock; + struct noise_handshake handshake; + atomic64_t last_sent_handshake; + struct work_struct transmit_handshake_work, clear_peer_work; + struct cookie latest_cookie; + struct hlist_node pubkey_hash; + u64 rx_bytes, tx_bytes; + struct timer_list timer_retransmit_handshake, timer_send_keepalive; + struct timer_list timer_new_handshake, timer_zero_key_material; + struct timer_list timer_persistent_keepalive; + unsigned int timer_handshake_attempts; + u16 persistent_keepalive_interval; + bool timer_need_another_keepalive; + bool sent_lastminute_handshake; + struct timespec64 walltime_last_handshake; + struct kref refcount; + struct rcu_head rcu; + struct list_head peer_list; + struct list_head allowedips_list; + u64 internal_id; + struct napi_struct napi; + bool is_dead; +}; + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); + +struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer); +static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) +{ + kref_get(&peer->refcount); + return peer; +} +void wg_peer_put(struct wg_peer *peer); +void wg_peer_remove(struct wg_peer *peer); +void wg_peer_remove_all(struct wg_device *wg); + +#endif /* _WG_PEER_H */ diff --git a/net/wireguard/peerlookup.c b/net/wireguard/peerlookup.c new file mode 100644 index 0000000000000..e4deb331476b3 --- /dev/null +++ b/net/wireguard/peerlookup.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peerlookup.h" +#include "peer.h" +#include "noise.h" + +static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + /* siphash gives us a secure 64bit number based on a random key. Since + * the bits are uniformly distributed, we can then mask off to get the + * bits we need. + */ + const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key); + + return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)]; +} + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void) +{ + struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + get_random_bytes(&table->key, sizeof(table->key)); + hash_init(table->hashtable); + mutex_init(&table->lock); + return table; +} + +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_add_head_rcu(&peer->pubkey_hash, + pubkey_bucket(table, peer->handshake.remote_static)); + mutex_unlock(&table->lock); +} + +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_del_init_rcu(&peer->pubkey_hash); + mutex_unlock(&table->lock); +} + +/* Returns a strong reference to a peer */ +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + struct wg_peer *iter_peer, *peer = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey), + pubkey_hash) { + if (!memcmp(pubkey, iter_peer->handshake.remote_static, + NOISE_PUBLIC_KEY_LEN)) { + peer = iter_peer; + break; + } + } + peer = wg_peer_get_maybe_zero(peer); + rcu_read_unlock_bh(); + return peer; +} + +static struct hlist_head *index_bucket(struct index_hashtable *table, + const __le32 index) +{ + /* Since the indices are random and thus all bits are uniformly + * distributed, we can find its bucket simply by masking. + */ + return &table->hashtable[(__force u32)index & + (HASH_SIZE(table->hashtable) - 1)]; +} + +struct index_hashtable *wg_index_hashtable_alloc(void) +{ + struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + hash_init(table->hashtable); + spin_lock_init(&table->lock); + return table; +} + +/* At the moment, we limit ourselves to 2^20 total peers, which generally might + * amount to 2^20*3 items in this hashtable. The algorithm below works by + * picking a random number and testing it. We can see that these limits mean we + * usually succeed pretty quickly: + * + * >>> def calculation(tries, size): + * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) + * ... + * >>> calculation(1, 2**20 * 3) + * 0.999267578125 + * >>> calculation(2, 2**20 * 3) + * 0.0007318854331970215 + * >>> calculation(3, 2**20 * 3) + * 5.360489012673497e-07 + * >>> calculation(4, 2**20 * 3) + * 3.9261394135792216e-10 + * + * At the moment, we don't do any masking, so this algorithm isn't exactly + * constant time in either the random guessing or in the hash list lookup. We + * could require a minimum of 3 tries, which would successfully mask the + * guessing. this would not, however, help with the growing hash lengths, which + * is another thing to consider moving forward. + */ + +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + struct index_hashtable_entry *existing_entry; + + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); + + rcu_read_lock_bh(); + +search_unused_slot: + /* First we try to find an unused slot, randomly, while unlocked. */ + entry->index = (__force __le32)get_random_u32(); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) + /* If it's already in use, we continue searching. */ + goto search_unused_slot; + } + + /* Once we've found an unused slot, we lock it, and then double-check + * that nobody else stole it from us. + */ + spin_lock_bh(&table->lock); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) { + spin_unlock_bh(&table->lock); + /* If it was stolen, we start over. */ + goto search_unused_slot; + } + } + /* Otherwise, we know we have it exclusively (since we're locked), + * so we insert. + */ + hlist_add_head_rcu(&entry->index_hash, + index_bucket(table, entry->index)); + spin_unlock_bh(&table->lock); + + rcu_read_unlock_bh(); + + return entry->index; +} + +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new) +{ + if (unlikely(hlist_unhashed(&old->index_hash))) + return false; + spin_lock_bh(&table->lock); + new->index = old->index; + hlist_replace_rcu(&old->index_hash, &new->index_hash); + + /* Calling init here NULLs out index_hash, and in fact after this + * function returns, it's theoretically possible for this to get + * reinserted elsewhere. That means the RCU lookup below might either + * terminate early or jump between buckets, in which case the packet + * simply gets dropped, which isn't terrible. + */ + INIT_HLIST_NODE(&old->index_hash); + spin_unlock_bh(&table->lock); + return true; +} + +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); +} + +/* Returns a strong reference to a entry->peer */ +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer) +{ + struct index_hashtable_entry *iter_entry, *entry = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index), + index_hash) { + if (iter_entry->index == index) { + if (likely(iter_entry->type & type_mask)) + entry = iter_entry; + break; + } + } + if (likely(entry)) { + entry->peer = wg_peer_get_maybe_zero(entry->peer); + if (likely(entry->peer)) + *peer = entry->peer; + else + entry = NULL; + } + rcu_read_unlock_bh(); + return entry; +} diff --git a/net/wireguard/peerlookup.h b/net/wireguard/peerlookup.h new file mode 100644 index 0000000000000..ced8117976800 --- /dev/null +++ b/net/wireguard/peerlookup.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEERLOOKUP_H +#define _WG_PEERLOOKUP_H + +#include "messages.h" + +#include +#include +#include + +struct wg_peer; + +struct pubkey_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 11); + siphash_key_t key; + struct mutex lock; +}; + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void); +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer); +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer); +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]); + +struct index_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 13); + spinlock_t lock; +}; + +enum index_hashtable_type { + INDEX_HASHTABLE_HANDSHAKE = 1U << 0, + INDEX_HASHTABLE_KEYPAIR = 1U << 1 +}; + +struct index_hashtable_entry { + struct wg_peer *peer; + struct hlist_node index_hash; + enum index_hashtable_type type; + __le32 index; +}; + +struct index_hashtable *wg_index_hashtable_alloc(void); +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry); +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new); +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry); +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer); + +#endif /* _WG_PEERLOOKUP_H */ diff --git a/net/wireguard/queueing.c b/net/wireguard/queueing.c new file mode 100644 index 0000000000000..5c964fcb994ec --- /dev/null +++ b/net/wireguard/queueing.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" + +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) +{ + int cpu; + struct multicore_worker __percpu *worker = + alloc_percpu(struct multicore_worker); + + if (!worker) + return NULL; + + for_each_possible_cpu(cpu) { + per_cpu_ptr(worker, cpu)->ptr = ptr; + INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function); + } + return worker; +} + +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len) +{ + int ret; + + memset(queue, 0, sizeof(*queue)); + ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); + if (ret) + return ret; + if (function) { + if (multicore) { + queue->worker = wg_packet_percpu_multicore_worker_alloc( + function, queue); + if (!queue->worker) + return -ENOMEM; + } else { + INIT_WORK(&queue->work, function); + } + } + return 0; +} + +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) +{ + if (multicore) + free_percpu(queue->worker); + WARN_ON(!__ptr_ring_empty(&queue->ring)); + ptr_ring_cleanup(&queue->ring, NULL); +} diff --git a/net/wireguard/queueing.h b/net/wireguard/queueing.h new file mode 100644 index 0000000000000..e49a464238fde --- /dev/null +++ b/net/wireguard/queueing.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_QUEUEING_H +#define _WG_QUEUEING_H + +#include "peer.h" +#include +#include +#include +#include + +struct wg_device; +struct wg_peer; +struct multicore_worker; +struct crypt_queue; +struct sk_buff; + +/* queueing.c APIs: */ +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len); +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); + +/* receive.c APIs: */ +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); +void wg_packet_handshake_receive_worker(struct work_struct *work); +/* NAPI poll function: */ +int wg_packet_rx_poll(struct napi_struct *napi, int budget); +/* Workqueue worker: */ +void wg_packet_decrypt_worker(struct work_struct *work); + +/* send.c APIs: */ +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry); +void wg_packet_send_handshake_response(struct wg_peer *peer); +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index); +void wg_packet_send_keepalive(struct wg_peer *peer); +void wg_packet_purge_staged_packets(struct wg_peer *peer); +void wg_packet_send_staged_packets(struct wg_peer *peer); +/* Workqueue workers: */ +void wg_packet_handshake_send_worker(struct work_struct *work); +void wg_packet_tx_worker(struct work_struct *work); +void wg_packet_encrypt_worker(struct work_struct *work); + +enum packet_state { + PACKET_STATE_UNCRYPTED, + PACKET_STATE_CRYPTED, + PACKET_STATE_DEAD +}; + +struct packet_cb { + u64 nonce; + struct noise_keypair *keypair; + atomic_t state; + u32 mtu; + u8 ds; +}; + +#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) +#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer) + +/* Returns either the correct skb->protocol value, or 0 if invalid. */ +static inline __be16 wg_skb_examine_untrusted_ip_hdr(struct sk_buff *skb) +{ + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct iphdr)) <= + skb_tail_pointer(skb) && + ip_hdr(skb)->version == 4) + return htons(ETH_P_IP); + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct ipv6hdr)) <= + skb_tail_pointer(skb) && + ipv6_hdr(skb)->version == 6) + return htons(ETH_P_IPV6); + return 0; +} + +static inline void wg_reset_packet(struct sk_buff *skb) +{ + const int pfmemalloc = skb->pfmemalloc; + + skb_scrub_packet(skb, true); + memset(&skb->headers_start, 0, + offsetof(struct sk_buff, headers_end) - + offsetof(struct sk_buff, headers_start)); + skb->pfmemalloc = pfmemalloc; + skb->queue_mapping = 0; + skb->nohdr = 0; + skb->peeked = 0; + skb->mac_len = 0; + skb->dev = NULL; +#ifdef CONFIG_NET_SCHED + skb->tc_index = 0; + skb_reset_tc(skb); +#endif + skb->hdr_len = skb_headroom(skb); + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb_probe_transport_header(skb); + skb_reset_inner_headers(skb); +} + +static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) +{ + unsigned int cpu = *stored_cpu, cpu_index, i; + + if (unlikely(cpu == nr_cpumask_bits || + !cpumask_test_cpu(cpu, cpu_online_mask))) { + cpu_index = id % cpumask_weight(cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < cpu_index; ++i) + cpu = cpumask_next(cpu, cpu_online_mask); + *stored_cpu = cpu; + } + return cpu; +} + +/* This function is racy, in the sense that next is unlocked, so it could return + * the same CPU twice. A race-free version of this would be to instead store an + * atomic sequence number, do an increment-and-return, and then iterate through + * every possible CPU until we get to that index -- choose_cpu. However that's + * a bit slower, and it doesn't seem like this potential race actually + * introduces any performance loss, so we live with it. + */ +static inline int wg_cpumask_next_online(int *next) +{ + int cpu = *next; + + while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask))) + cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + return cpu; +} + +static inline int wg_queue_enqueue_per_device_and_peer( + struct crypt_queue *device_queue, struct crypt_queue *peer_queue, + struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) +{ + int cpu; + + atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); + /* We first queue this up for the peer ingestion, but the consumer + * will wait for the state to change to CRYPTED or DEAD before. + */ + if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) + return -ENOSPC; + /* Then we queue it up in the device queue, which consumes the + * packet as soon as it can. + */ + cpu = wg_cpumask_next_online(next_cpu); + if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) + return -EPIPE; + queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); + return 0; +} + +static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, + struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, + peer->internal_id), + peer->device->packet_crypt_wq, &queue->work); + wg_peer_put(peer); +} + +static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + wg_peer_put(peer); +} + +#ifdef DEBUG +bool wg_packet_counter_selftest(void); +#endif + +#endif /* _WG_QUEUEING_H */ diff --git a/net/wireguard/ratelimiter.c b/net/wireguard/ratelimiter.c new file mode 100644 index 0000000000000..e33ec72a96421 --- /dev/null +++ b/net/wireguard/ratelimiter.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU +/* We normally alias all non-_bh functions to the _bh ones in the compat layer, + * but that's not appropriate here, where we actually do want non-_bh ones. + */ +#undef synchronize_rcu +#define synchronize_rcu old_synchronize_rcu +#undef call_rcu +#define call_rcu old_call_rcu +#undef rcu_barrier +#define rcu_barrier old_rcu_barrier +#endif + +#include "ratelimiter.h" +#include +#include +#include +#include + +static struct kmem_cache *entry_cache; +static hsiphash_key_t key; +static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); +static DEFINE_MUTEX(init_lock); +static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ +static atomic_t total_entries = ATOMIC_INIT(0); +static unsigned int max_entries, table_size; +static void wg_ratelimiter_gc_entries(struct work_struct *); +static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); +static struct hlist_head *table_v4; +#if IS_ENABLED(CONFIG_IPV6) +static struct hlist_head *table_v6; +#endif + +struct ratelimiter_entry { + u64 last_time_ns, tokens, ip; + void *net; + spinlock_t lock; + struct hlist_node hash; + struct rcu_head rcu; +}; + +enum { + PACKETS_PER_SECOND = 20, + PACKETS_BURSTABLE = 5, + PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, + TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE +}; + +static void entry_free(struct rcu_head *rcu) +{ + kmem_cache_free(entry_cache, + container_of(rcu, struct ratelimiter_entry, rcu)); + atomic_dec(&total_entries); +} + +static void entry_uninit(struct ratelimiter_entry *entry) +{ + hlist_del_rcu(&entry->hash); + call_rcu(&entry->rcu, entry_free); +} + +/* Calling this function with a NULL work uninits all entries. */ +static void wg_ratelimiter_gc_entries(struct work_struct *work) +{ + const u64 now = ktime_get_coarse_boottime_ns(); + struct ratelimiter_entry *entry; + struct hlist_node *temp; + unsigned int i; + + for (i = 0; i < table_size; ++i) { + spin_lock(&table_lock); + hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#if IS_ENABLED(CONFIG_IPV6) + hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#endif + spin_unlock(&table_lock); + if (likely(work)) + cond_resched(); + } + if (likely(work)) + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); +} + +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) +{ + /* We only take the bottom half of the net pointer, so that we can hash + * 3 words in the end. This way, siphash's len param fits into the final + * u32, and we don't incur an extra round. + */ + const u32 net_word = (unsigned long)net; + struct ratelimiter_entry *entry; + struct hlist_head *bucket; + u64 ip; + + if (skb->protocol == htons(ETH_P_IP)) { + ip = (u64 __force)ip_hdr(skb)->saddr; + bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & + (table_size - 1)]; + } +#if IS_ENABLED(CONFIG_IPV6) + else if (skb->protocol == htons(ETH_P_IPV6)) { + /* Only use 64 bits, so as to ratelimit the whole /64. */ + memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); + bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & + (table_size - 1)]; + } +#endif + else + return false; + rcu_read_lock(); + hlist_for_each_entry_rcu(entry, bucket, hash) { + if (entry->net == net && entry->ip == ip) { + u64 now, tokens; + bool ret; + /* Quasi-inspired by nft_limit.c, but this is actually a + * slightly different algorithm. Namely, we incorporate + * the burst as part of the maximum tokens, rather than + * as part of the rate. + */ + spin_lock(&entry->lock); + now = ktime_get_coarse_boottime_ns(); + tokens = min_t(u64, TOKEN_MAX, + entry->tokens + now - + entry->last_time_ns); + entry->last_time_ns = now; + ret = tokens >= PACKET_COST; + entry->tokens = ret ? tokens - PACKET_COST : tokens; + spin_unlock(&entry->lock); + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + if (atomic_inc_return(&total_entries) > max_entries) + goto err_oom; + + entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); + if (unlikely(!entry)) + goto err_oom; + + entry->net = net; + entry->ip = ip; + INIT_HLIST_NODE(&entry->hash); + spin_lock_init(&entry->lock); + entry->last_time_ns = ktime_get_coarse_boottime_ns(); + entry->tokens = TOKEN_MAX - PACKET_COST; + spin_lock(&table_lock); + hlist_add_head_rcu(&entry->hash, bucket); + spin_unlock(&table_lock); + return true; + +err_oom: + atomic_dec(&total_entries); + return false; +} + +int wg_ratelimiter_init(void) +{ + mutex_lock(&init_lock); + if (++init_refcnt != 1) + goto out; + + entry_cache = KMEM_CACHE(ratelimiter_entry, 0); + if (!entry_cache) + goto err; + + /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, + * but what it shares in common is that it uses a massive hashtable. So, + * we borrow their wisdom about good table sizes on different systems + * dependent on RAM. This calculation here comes from there. + */ + table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : + max_t(unsigned long, 16, roundup_pow_of_two( + (totalram_pages() << PAGE_SHIFT) / + (1U << 14) / sizeof(struct hlist_head))); + max_entries = table_size * 8; + + table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL); + if (unlikely(!table_v4)) + goto err_kmemcache; + +#if IS_ENABLED(CONFIG_IPV6) + table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL); + if (unlikely(!table_v6)) { + kvfree(table_v4); + goto err_kmemcache; + } +#endif + + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); + get_random_bytes(&key, sizeof(key)); +out: + mutex_unlock(&init_lock); + return 0; + +err_kmemcache: + kmem_cache_destroy(entry_cache); +err: + --init_refcnt; + mutex_unlock(&init_lock); + return -ENOMEM; +} + +void wg_ratelimiter_uninit(void) +{ + mutex_lock(&init_lock); + if (!init_refcnt || --init_refcnt) + goto out; + + cancel_delayed_work_sync(&gc_work); + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + kvfree(table_v4); +#if IS_ENABLED(CONFIG_IPV6) + kvfree(table_v6); +#endif + kmem_cache_destroy(entry_cache); +out: + mutex_unlock(&init_lock); +} + +#include "selftest/ratelimiter.c" diff --git a/net/wireguard/ratelimiter.h b/net/wireguard/ratelimiter.h new file mode 100644 index 0000000000000..83067f71ea99e --- /dev/null +++ b/net/wireguard/ratelimiter.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_RATELIMITER_H +#define _WG_RATELIMITER_H + +#include + +int wg_ratelimiter_init(void); +void wg_ratelimiter_uninit(void); +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net); + +#ifdef DEBUG +bool wg_ratelimiter_selftest(void); +#endif + +#endif /* _WG_RATELIMITER_H */ diff --git a/net/wireguard/receive.c b/net/wireguard/receive.c new file mode 100644 index 0000000000000..e00f0f4b09827 --- /dev/null +++ b/net/wireguard/receive.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "device.h" +#include "peer.h" +#include "timers.h" +#include "messages.h" +#include "cookie.h" +#include "socket.h" + +#include +#include +#include +#include +#include + +/* Must be called with bh disabled. */ +static void update_rx_stats(struct wg_peer *peer, size_t len) +{ + struct pcpu_sw_netstats *tstats = + get_cpu_ptr(peer->device->dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + ++tstats->rx_packets; + tstats->rx_bytes += len; + peer->rx_bytes += len; + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); +} + +#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) + +static size_t validate_header_len(struct sk_buff *skb) +{ + if (unlikely(skb->len < sizeof(struct message_header))) + return 0; + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && + skb->len >= MESSAGE_MINIMUM_LENGTH) + return sizeof(struct message_data); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && + skb->len == sizeof(struct message_handshake_initiation)) + return sizeof(struct message_handshake_initiation); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && + skb->len == sizeof(struct message_handshake_response)) + return sizeof(struct message_handshake_response); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && + skb->len == sizeof(struct message_handshake_cookie)) + return sizeof(struct message_handshake_cookie); + return 0; +} + +static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) +{ + size_t data_offset, data_len, header_len; + struct udphdr *udp; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol || + skb_transport_header(skb) < skb->head || + (skb_transport_header(skb) + sizeof(struct udphdr)) > + skb_tail_pointer(skb))) + return -EINVAL; /* Bogus IP header */ + udp = udp_hdr(skb); + data_offset = (u8 *)udp - skb->data; + if (unlikely(data_offset > U16_MAX || + data_offset + sizeof(struct udphdr) > skb->len)) + /* Packet has offset at impossible location or isn't big enough + * to have UDP fields. + */ + return -EINVAL; + data_len = ntohs(udp->len); + if (unlikely(data_len < sizeof(struct udphdr) || + data_len > skb->len - data_offset)) + /* UDP packet is reporting too small of a size or lying about + * its size. + */ + return -EINVAL; + data_len -= sizeof(struct udphdr); + data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; + if (unlikely(!pskb_may_pull(skb, + data_offset + sizeof(struct message_header)) || + pskb_trim(skb, data_len + data_offset) < 0)) + return -EINVAL; + skb_pull(skb, data_offset); + if (unlikely(skb->len != data_len)) + /* Final len does not agree with calculated len */ + return -EINVAL; + header_len = validate_header_len(skb); + if (unlikely(!header_len)) + return -EINVAL; + __skb_push(skb, data_offset); + if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) + return -EINVAL; + __skb_pull(skb, data_offset); + return 0; +} + +static void wg_receive_handshake_packet(struct wg_device *wg, + struct sk_buff *skb) +{ + enum cookie_mac_state mac_state; + struct wg_peer *peer = NULL; + /* This is global, so that our load calculation applies to the whole + * system. We don't care about races with it at all. + */ + static u64 last_under_load; + bool packet_needs_cookie; + bool under_load; + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { + net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", + wg->dev->name, skb); + wg_cookie_message_consume( + (struct message_handshake_cookie *)skb->data, wg); + return; + } + + under_load = skb_queue_len(&wg->incoming_handshakes) >= + MAX_QUEUED_INCOMING_HANDSHAKES / 8; + if (under_load) + last_under_load = ktime_get_coarse_boottime_ns(); + else if (last_under_load) + under_load = !wg_birthdate_has_expired(last_under_load, 1); + mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, + under_load); + if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || + (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { + packet_needs_cookie = false; + } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { + packet_needs_cookie = true; + } else { + net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", + wg->dev->name, skb); + return; + } + + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { + struct message_handshake_initiation *message = + (struct message_handshake_initiation *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_initiation(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + wg_packet_send_handshake_response(peer); + break; + } + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { + struct message_handshake_response *message = + (struct message_handshake_response *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_response(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_handshake_complete(peer); + /* Calling this function will either send any existing + * packets in the queue and not send a keepalive, which + * is the best case, Or, if there's nothing in the + * queue, it will send a keepalive, in order to give + * immediate confirmation of the session. + */ + wg_packet_send_keepalive(peer); + } + break; + } + } + + if (unlikely(!peer)) { + WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); + return; + } + + local_bh_disable(); + update_rx_stats(peer, skb->len); + local_bh_enable(); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_peer_put(peer); +} + +void wg_packet_handshake_receive_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *skb; + + while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { + wg_receive_handshake_packet(wg, skb); + dev_kfree_skb(skb); + cond_resched(); + } +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + if (peer->sent_lastminute_handshake) + return; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) + send = true; + rcu_read_unlock_bh(); + + if (send) { + peer->sent_lastminute_handshake = true; + wg_packet_send_queued_handshake_initiation(peer, false); + } +} + +static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key, + simd_context_t *simd_context) +{ + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct sk_buff *trailer; + unsigned int offset; + int num_frags; + + if (unlikely(!key)) + return false; + + if (unlikely(!READ_ONCE(key->is_valid) || + wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || + key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { + WRITE_ONCE(key->is_valid, false); + return false; + } + + PACKET_CB(skb)->nonce = + le64_to_cpu(((struct message_data *)skb->data)->counter); + + /* We ensure that the network header is part of the packet before we + * call skb_cow_data, so that there's no chance that data is removed + * from the skb, so that later we can extract the original endpoint. + */ + offset = skb->data - skb_network_header(skb); + skb_push(skb, offset); + num_frags = skb_cow_data(skb, 0, &trailer); + offset += sizeof(struct message_data); + skb_pull(skb, offset); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) + return false; + + if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, + PACKET_CB(skb)->nonce, key->key, + simd_context)) + return false; + + /* Another ugly situation of pushing and pulling the header so as to + * keep endpoint information intact. + */ + skb_push(skb, offset); + if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) + return false; + skb_pull(skb, offset); + + return true; +} + +/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ +static bool counter_validate(union noise_counter *counter, u64 their_counter) +{ + unsigned long index, index_current, top, i; + bool ret = false; + + spin_lock_bh(&counter->receive.lock); + + if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || + their_counter >= REJECT_AFTER_MESSAGES)) + goto out; + + ++their_counter; + + if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < + counter->receive.counter)) + goto out; + + index = their_counter >> ilog2(BITS_PER_LONG); + + if (likely(their_counter > counter->receive.counter)) { + index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); + top = min_t(unsigned long, index - index_current, + COUNTER_BITS_TOTAL / BITS_PER_LONG); + for (i = 1; i <= top; ++i) + counter->receive.backtrack[(i + index_current) & + ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; + counter->receive.counter = their_counter; + } + + index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; + ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), + &counter->receive.backtrack[index]); + +out: + spin_unlock_bh(&counter->receive.lock); + return ret; +} + +#include "selftest/counter.c" + +static void wg_packet_consume_data_done(struct wg_peer *peer, + struct sk_buff *skb, + struct endpoint *endpoint) +{ + struct net_device *dev = peer->device->dev; + unsigned int len, len_before_trim; + struct wg_peer *routed_peer; + + wg_socket_set_peer_endpoint(peer, endpoint); + + if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, + PACKET_CB(skb)->keypair))) { + wg_timers_handshake_complete(peer); + wg_packet_send_staged_packets(peer); + } + + keep_key_fresh(peer); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + + /* A packet with length 0 is a keepalive packet */ + if (unlikely(!skb->len)) { + update_rx_stats(peer, message_data_len(0)); + net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + goto packet_processed; + } + + wg_timers_data_received(peer); + + if (unlikely(skb_network_header(skb) < skb->head)) + goto dishonest_packet_size; + if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && + (ip_hdr(skb)->version == 4 || + (ip_hdr(skb)->version == 6 && + pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) + goto dishonest_packet_type; + + skb->dev = dev; + /* We've already verified the Poly1305 auth tag, which means this packet + * was not modified in transit. We can therefore tell the networking + * stack that all checksums of every layer of encapsulation have already + * been checked "by the hardware" and therefore is unnecessary to check + * again in software. + */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +#ifndef COMPAT_CANNOT_USE_CSUM_LEVEL + skb->csum_level = ~0; /* All levels */ +#endif + skb->protocol = wg_skb_examine_untrusted_ip_hdr(skb); + if (skb->protocol == htons(ETH_P_IP)) { + len = ntohs(ip_hdr(skb)->tot_len); + if (unlikely(len < sizeof(struct iphdr))) + goto dishonest_packet_size; + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP_ECN_set_ce(ip_hdr(skb)); + } else if (skb->protocol == htons(ETH_P_IPV6)) { + len = ntohs(ipv6_hdr(skb)->payload_len) + + sizeof(struct ipv6hdr); + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP6_ECN_set_ce(skb, ipv6_hdr(skb)); + } else { + goto dishonest_packet_type; + } + + if (unlikely(len > skb->len)) + goto dishonest_packet_size; + len_before_trim = skb->len; + if (unlikely(pskb_trim(skb, len))) + goto packet_processed; + + routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, + skb); + wg_peer_put(routed_peer); /* We don't need the extra reference. */ + + if (unlikely(routed_peer != peer)) + goto dishonest_packet_peer; + + if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { + ++dev->stats.rx_dropped; + net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + } else { + update_rx_stats(peer, message_data_len(len_before_trim)); + } + return; + +dishonest_packet_peer: + net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", + dev->name, skb, peer->internal_id, + &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_type: + net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_size: + net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_length_errors; + goto packet_processed; +packet_processed: + dev_kfree_skb(skb); +} + +int wg_packet_rx_poll(struct napi_struct *napi, int budget) +{ + struct wg_peer *peer = container_of(napi, struct wg_peer, napi); + struct crypt_queue *queue = &peer->rx_queue; + struct noise_keypair *keypair; + struct endpoint endpoint; + enum packet_state state; + struct sk_buff *skb; + int work_done = 0; + bool free; + + if (unlikely(budget <= 0)) + return 0; + + while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(skb); + keypair = PACKET_CB(skb)->keypair; + free = true; + + if (unlikely(state != PACKET_STATE_CRYPTED)) + goto next; + + if (unlikely(!counter_validate(&keypair->receiving.counter, + PACKET_CB(skb)->nonce))) { + net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", + peer->device->dev->name, + PACKET_CB(skb)->nonce, + keypair->receiving.counter.receive.counter); + goto next; + } + + if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) + goto next; + + wg_reset_packet(skb); + wg_packet_consume_data_done(peer, skb, &endpoint); + free = false; + +next: + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + if (unlikely(free)) + dev_kfree_skb(skb); + + if (++work_done >= budget) + break; + } + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; +} + +void wg_packet_decrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + simd_context_t simd_context; + struct sk_buff *skb; + + simd_get(&simd_context); + while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = likely(decrypt_packet(skb, + &PACKET_CB(skb)->keypair->receiving, + &simd_context)) ? + PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; + wg_queue_enqueue_per_peer_napi(skb, state); + simd_relax(&simd_context); + } + + simd_put(&simd_context); +} + +static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) +{ + __le32 idx = ((struct message_data *)skb->data)->key_idx; + struct wg_peer *peer = NULL; + int ret; + + rcu_read_lock_bh(); + PACKET_CB(skb)->keypair = + (struct noise_keypair *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, + &peer); + if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) + goto err_keypair; + + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, + &peer->rx_queue, skb, + wg->packet_crypt_wq, + &wg->decrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + if (likely(!ret || ret == -EPIPE)) { + rcu_read_unlock_bh(); + return; + } +err: + wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); +err_keypair: + rcu_read_unlock_bh(); + wg_peer_put(peer); + dev_kfree_skb(skb); +} + +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) +{ + if (unlikely(prepare_skb_header(skb, wg) < 0)) + goto err; + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): + case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { + int cpu; + + if (skb_queue_len(&wg->incoming_handshakes) > + MAX_QUEUED_INCOMING_HANDSHAKES || + unlikely(!rng_is_initialized())) { + net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + skb_queue_tail(&wg->incoming_handshakes, skb); + /* Queues up a call to packet_process_queued_handshake_ + * packets(skb): + */ + cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); + queue_work_on(cpu, wg->handshake_receive_wq, + &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); + break; + } + case cpu_to_le32(MESSAGE_DATA): + PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); + wg_packet_consume_data(wg, skb); + break; + default: + net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + return; + +err: + dev_kfree_skb(skb); +} diff --git a/net/wireguard/selftest/allowedips.c b/net/wireguard/selftest/allowedips.c new file mode 100644 index 0000000000000..846db14cb046b --- /dev/null +++ b/net/wireguard/selftest/allowedips.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This contains some basic static unit tests for the allowedips data structure. + * It also has two additional modes that are disabled and meant to be used by + * folks directly playing with this file. If you define the macro + * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in + * memory, it will be printed out as KERN_DEBUG in a format that can be passed + * to graphviz (the dot command) to visualize it. If you define the macro + * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of + * randomized tests done against a trivial implementation, which may take + * upwards of a half-hour to complete. There's no set of users who should be + * enabling these, and the only developers that should go anywhere near these + * nobs are the ones who are reading this comment. + */ + +#ifdef DEBUG + +#include + +static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits, + u8 cidr) +{ + swap_endian(dst, src, bits); + memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8); + if (cidr) + dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8); +} + +static __init void print_node(struct allowedips_node *node, u8 bits) +{ + char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n"; + char *fmt_declaration = KERN_DEBUG + "\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n"; + char *style = "dotted"; + u8 ip1[16], ip2[16]; + u32 color = 0; + + if (bits == 32) { + fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n"; + } else if (bits == 128) { + fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n"; + } + if (node->peer) { + hsiphash_key_t key = { { 0 } }; + + memcpy(&key, &node->peer, sizeof(node->peer)); + color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 | + hsiphash_1u32(0xbabecafe, &key) % 200 << 8 | + hsiphash_1u32(0xabad1dea, &key) % 200; + style = "bold"; + } + swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr); + printk(fmt_declaration, ip1, node->cidr, style, color); + if (node->bit[0]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[0])->bits, bits, + node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[0])->cidr); + print_node(rcu_dereference_raw(node->bit[0]), bits); + } + if (node->bit[1]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[1])->bits, + bits, node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[1])->cidr); + print_node(rcu_dereference_raw(node->bit[1]), bits); + } +} + +static __init void print_tree(struct allowedips_node __rcu *top, u8 bits) +{ + printk(KERN_DEBUG "digraph trie {\n"); + print_node(rcu_dereference_raw(top), bits); + printk(KERN_DEBUG "}\n"); +} + +enum { + NUM_PEERS = 2000, + NUM_RAND_ROUTES = 400, + NUM_MUTATED_ROUTES = 100, + NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30 +}; + +struct horrible_allowedips { + struct hlist_head head; +}; + +struct horrible_allowedips_node { + struct hlist_node table; + union nf_inet_addr ip; + union nf_inet_addr mask; + u8 ip_version; + void *value; +}; + +static __init void horrible_allowedips_init(struct horrible_allowedips *table) +{ + INIT_HLIST_HEAD(&table->head); +} + +static __init void horrible_allowedips_free(struct horrible_allowedips *table) +{ + struct horrible_allowedips_node *node; + struct hlist_node *h; + + hlist_for_each_entry_safe(node, h, &table->head, table) { + hlist_del(&node->table); + kfree(node); + } +} + +static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr) +{ + union nf_inet_addr mask; + + memset(&mask, 0x00, 128 / 8); + memset(&mask, 0xff, cidr / 8); + if (cidr % 32) + mask.all[cidr / 32] = (__force u32)htonl( + (0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL); + return mask; +} + +static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet) +{ + return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) + + hweight32(subnet.all[2]) + hweight32(subnet.all[3]); +} + +static __init inline void +horrible_mask_self(struct horrible_allowedips_node *node) +{ + if (node->ip_version == 4) { + node->ip.ip &= node->mask.ip; + } else if (node->ip_version == 6) { + node->ip.ip6[0] &= node->mask.ip6[0]; + node->ip.ip6[1] &= node->mask.ip6[1]; + node->ip.ip6[2] &= node->mask.ip6[2]; + node->ip.ip6[3] &= node->mask.ip6[3]; + } +} + +static __init inline bool +horrible_match_v4(const struct horrible_allowedips_node *node, + struct in_addr *ip) +{ + return (ip->s_addr & node->mask.ip) == node->ip.ip; +} + +static __init inline bool +horrible_match_v6(const struct horrible_allowedips_node *node, + struct in6_addr *ip) +{ + return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) == + node->ip.ip6[0] && + (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) == + node->ip.ip6[1] && + (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) == + node->ip.ip6[2] && + (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3]; +} + +static __init void +horrible_insert_ordered(struct horrible_allowedips *table, + struct horrible_allowedips_node *node) +{ + struct horrible_allowedips_node *other = NULL, *where = NULL; + u8 my_cidr = horrible_mask_to_cidr(node->mask); + + hlist_for_each_entry(other, &table->head, table) { + if (!memcmp(&other->mask, &node->mask, + sizeof(union nf_inet_addr)) && + !memcmp(&other->ip, &node->ip, + sizeof(union nf_inet_addr)) && + other->ip_version == node->ip_version) { + other->value = node->value; + kfree(node); + return; + } + where = other; + if (horrible_mask_to_cidr(other->mask) <= my_cidr) + break; + } + if (!other && !where) + hlist_add_head(&node->table, &table->head); + else if (!other) + hlist_add_behind(&node->table, &where->table); + else + hlist_add_before(&node->table, &where->table); +} + +static __init int +horrible_allowedips_insert_v4(struct horrible_allowedips *table, + struct in_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 4; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init int +horrible_allowedips_insert_v6(struct horrible_allowedips *table, + struct in6_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in6 = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 6; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init void * +horrible_allowedips_lookup_v4(struct horrible_allowedips *table, + struct in_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 4) + continue; + if (horrible_match_v4(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init void * +horrible_allowedips_lookup_v6(struct horrible_allowedips *table, + struct in6_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 6) + continue; + if (horrible_match_v6(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init bool randomized_test(void) +{ + unsigned int i, j, k, mutate_amount, cidr; + u8 ip[16], mutate_mask[16], mutated[16]; + struct wg_peer **peers, *peer; + struct horrible_allowedips h; + DEFINE_MUTEX(mutex); + struct allowedips t; + bool ret = false; + + mutex_init(&mutex); + + wg_allowedips_init(&t); + horrible_allowedips_init(&h); + + peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL); + if (unlikely(!peers)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + for (i = 0; i < NUM_PEERS; ++i) { + peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL); + if (unlikely(!peers[i])) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + kref_init(&peers[i]->refcount); + } + + mutex_lock(&mutex); + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 4); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 4); + prandom_bytes(mutate_mask, 4); + mutate_amount = prandom_u32_max(32); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, + (struct in_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, + (struct in_addr *)mutated, cidr, peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 16); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 16); + prandom_bytes(mutate_mask, 16); + mutate_amount = prandom_u32_max(128); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, + (struct in6_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6( + &h, (struct in6_addr *)mutated, cidr, + peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + mutex_unlock(&mutex); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 4); + if (lookup(t.root4, 32, ip) != + horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 16); + if (lookup(t.root6, 128, ip) != + horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + ret = true; + +free: + mutex_lock(&mutex); +free_locked: + wg_allowedips_free(&t, &mutex); + mutex_unlock(&mutex); + horrible_allowedips_free(&h); + if (peers) { + for (i = 0; i < NUM_PEERS; ++i) + kfree(peers[i]); + } + kfree(peers); + return ret; +} + +static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d) +{ + static struct in_addr ip; + u8 *split = (u8 *)&ip; + + split[0] = a; + split[1] = b; + split[2] = c; + split[3] = d; + return &ip; +} + +static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d) +{ + static struct in6_addr ip; + __be32 *split = (__be32 *)&ip; + + split[0] = cpu_to_be32(a); + split[1] = cpu_to_be32(b); + split[2] = cpu_to_be32(c); + split[3] = cpu_to_be32(d); + return &ip; +} + +static __init struct wg_peer *init_peer(void) +{ + struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL); + + if (!peer) + return NULL; + kref_init(&peer->refcount); + INIT_LIST_HEAD(&peer->allowedips_list); + return peer; +} + +#define insert(version, mem, ipa, ipb, ipc, ipd, cidr) \ + wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ + cidr, mem, &mutex) + +#define maybe_fail() do { \ + ++i; \ + if (!_s) { \ + pr_info("allowedips self-test %zu: FAIL\n", i); \ + success = false; \ + } \ + } while (0) + +#define test(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) == (mem); \ + maybe_fail(); \ + } while (0) + +#define test_negative(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) != (mem); \ + maybe_fail(); \ + } while (0) + +#define test_boolean(cond) do { \ + bool _s = (cond); \ + maybe_fail(); \ + } while (0) + +bool __init wg_allowedips_selftest(void) +{ + bool found_a = false, found_b = false, found_c = false, found_d = false, + found_e = false, found_other = false; + struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(), + *d = init_peer(), *e = init_peer(), *f = init_peer(), + *g = init_peer(), *h = init_peer(); + struct allowedips_node *iter_node; + bool success = false; + struct allowedips t; + DEFINE_MUTEX(mutex); + struct in6_addr ip; + size_t i = 0, count = 0; + __be64 part; + + mutex_init(&mutex); + mutex_lock(&mutex); + wg_allowedips_init(&t); + + if (!a || !b || !c || !d || !e || !f || !g || !h) { + pr_err("allowedips self-test malloc: FAIL\n"); + goto free; + } + + insert(4, a, 192, 168, 4, 0, 24); + insert(4, b, 192, 168, 4, 4, 32); + insert(4, c, 192, 168, 0, 0, 16); + insert(4, d, 192, 95, 5, 64, 27); + /* replaces previous entry, and maskself is required */ + insert(4, c, 192, 95, 5, 65, 27); + insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64); + insert(4, e, 0, 0, 0, 0, 0); + insert(6, e, 0, 0, 0, 0, 0); + /* replaces previous entry */ + insert(6, f, 0, 0, 0, 0, 0); + insert(6, g, 0x24046800, 0, 0, 0, 32); + /* maskself is required */ + insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); + insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); + insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + insert(4, g, 64, 15, 112, 0, 20); + /* maskself is required */ + insert(4, h, 64, 15, 123, 211, 25); + insert(4, a, 10, 0, 0, 0, 25); + insert(4, b, 10, 0, 0, 128, 25); + insert(4, a, 10, 1, 0, 0, 30); + insert(4, b, 10, 1, 0, 4, 30); + insert(4, c, 10, 1, 0, 8, 29); + insert(4, d, 10, 1, 0, 16, 29); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + success = true; + + test(4, a, 192, 168, 4, 20); + test(4, a, 192, 168, 4, 0); + test(4, b, 192, 168, 4, 4); + test(4, c, 192, 168, 200, 182); + test(4, c, 192, 95, 5, 68); + test(4, e, 192, 95, 5, 96); + test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543); + test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee); + test(6, f, 0x26075300, 0x60006b01, 0, 0); + test(6, g, 0x24046800, 0x40040806, 0, 0x1006); + test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0, 0); + test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010); + test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); + test(4, g, 64, 15, 116, 26); + test(4, g, 64, 15, 127, 3); + test(4, g, 64, 15, 123, 1); + test(4, h, 64, 15, 123, 128); + test(4, h, 64, 15, 123, 129); + test(4, a, 10, 0, 0, 52); + test(4, b, 10, 0, 0, 220); + test(4, a, 10, 1, 0, 2); + test(4, b, 10, 1, 0, 6); + test(4, c, 10, 1, 0, 10); + test(4, d, 10, 1, 0, 20); + + insert(4, a, 1, 0, 0, 0, 32); + insert(4, a, 64, 0, 0, 0, 32); + insert(4, a, 128, 0, 0, 0, 32); + insert(4, a, 192, 0, 0, 0, 32); + insert(4, a, 255, 0, 0, 0, 32); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 1, 0, 0, 0); + test_negative(4, a, 64, 0, 0, 0); + test_negative(4, a, 128, 0, 0, 0); + test_negative(4, a, 192, 0, 0, 0); + test_negative(4, a, 255, 0, 0, 0); + + wg_allowedips_free(&t, &mutex); + wg_allowedips_init(&t); + insert(4, a, 192, 168, 0, 0, 16); + insert(4, a, 192, 168, 0, 0, 24); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 192, 168, 0, 1); + + /* These will hit the WARN_ON(len >= 128) in free_node if something + * goes wrong. + */ + for (i = 0; i < 128; ++i) { + part = cpu_to_be64(~(1LLU << (i % 64))); + memset(&ip, 0xff, 16); + memcpy((u8 *)&ip + (i < 64) * 8, &part, 8); + wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex); + } + + wg_allowedips_free(&t, &mutex); + + wg_allowedips_init(&t); + insert(4, a, 192, 95, 5, 93, 27); + insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(4, a, 10, 1, 0, 20, 29); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); + list_for_each_entry(iter_node, &a->allowedips_list, peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family = wg_allowedips_read_node(iter_node, ip, &cidr); + + count++; + + if (cidr == 27 && family == AF_INET && + !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr))) + found_a = true; + else if (cidr == 128 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), + sizeof(struct in6_addr))) + found_b = true; + else if (cidr == 29 && family == AF_INET && + !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr))) + found_c = true; + else if (cidr == 83 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), + sizeof(struct in6_addr))) + found_d = true; + else if (cidr == 21 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075000, 0, 0, 0), + sizeof(struct in6_addr))) + found_e = true; + else + found_other = true; + } + test_boolean(count == 5); + test_boolean(found_a); + test_boolean(found_b); + test_boolean(found_c); + test_boolean(found_d); + test_boolean(found_e); + test_boolean(!found_other); + + if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success) + success = randomized_test(); + + if (success) + pr_info("allowedips self-tests: pass\n"); + +free: + wg_allowedips_free(&t, &mutex); + kfree(a); + kfree(b); + kfree(c); + kfree(d); + kfree(e); + kfree(f); + kfree(g); + kfree(h); + mutex_unlock(&mutex); + + return success; +} + +#undef test_negative +#undef test +#undef remove +#undef insert +#undef init_peer + +#endif diff --git a/net/wireguard/selftest/counter.c b/net/wireguard/selftest/counter.c new file mode 100644 index 0000000000000..f4fbb9072ed73 --- /dev/null +++ b/net/wireguard/selftest/counter.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG +bool __init wg_packet_counter_selftest(void) +{ + unsigned int test_num = 0, i; + union noise_counter counter; + bool success = true; + +#define T_INIT do { \ + memset(&counter, 0, sizeof(union noise_counter)); \ + spin_lock_init(&counter.receive.lock); \ + } while (0) +#define T_LIM (COUNTER_WINDOW_SIZE + 1) +#define T(n, v) do { \ + ++test_num; \ + if (counter_validate(&counter, n) != (v)) { \ + pr_err("nonce counter self-test %u: FAIL\n", \ + test_num); \ + success = false; \ + } \ + } while (0) + + T_INIT; + /* 1 */ T(0, true); + /* 2 */ T(1, true); + /* 3 */ T(1, false); + /* 4 */ T(9, true); + /* 5 */ T(8, true); + /* 6 */ T(7, true); + /* 7 */ T(7, false); + /* 8 */ T(T_LIM, true); + /* 9 */ T(T_LIM - 1, true); + /* 10 */ T(T_LIM - 1, false); + /* 11 */ T(T_LIM - 2, true); + /* 12 */ T(2, true); + /* 13 */ T(2, false); + /* 14 */ T(T_LIM + 16, true); + /* 15 */ T(3, false); + /* 16 */ T(T_LIM + 16, false); + /* 17 */ T(T_LIM * 4, true); + /* 18 */ T(T_LIM * 4 - (T_LIM - 1), true); + /* 19 */ T(10, false); + /* 20 */ T(T_LIM * 4 - T_LIM, false); + /* 21 */ T(T_LIM * 4 - (T_LIM + 1), false); + /* 22 */ T(T_LIM * 4 - (T_LIM - 2), true); + /* 23 */ T(T_LIM * 4 + 1 - T_LIM, false); + /* 24 */ T(0, false); + /* 25 */ T(REJECT_AFTER_MESSAGES, false); + /* 26 */ T(REJECT_AFTER_MESSAGES - 1, true); + /* 27 */ T(REJECT_AFTER_MESSAGES, false); + /* 28 */ T(REJECT_AFTER_MESSAGES - 1, false); + /* 29 */ T(REJECT_AFTER_MESSAGES - 2, true); + /* 30 */ T(REJECT_AFTER_MESSAGES + 1, false); + /* 31 */ T(REJECT_AFTER_MESSAGES + 2, false); + /* 32 */ T(REJECT_AFTER_MESSAGES - 2, false); + /* 33 */ T(REJECT_AFTER_MESSAGES - 3, true); + /* 34 */ T(0, false); + + T_INIT; + for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) + T(i, true); + T(0, true); + T(0, false); + + T_INIT; + for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) + T(i, true); + T(1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) + T(i, true); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) + T(i, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(COUNTER_WINDOW_SIZE + 1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(0, true); + T(COUNTER_WINDOW_SIZE + 1, true); + +#undef T +#undef T_LIM +#undef T_INIT + + if (success) + pr_info("nonce counter self-tests: pass\n"); + return success; +} +#endif diff --git a/net/wireguard/selftest/ratelimiter.c b/net/wireguard/selftest/ratelimiter.c new file mode 100644 index 0000000000000..bcd6462e45401 --- /dev/null +++ b/net/wireguard/selftest/ratelimiter.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG + +#include + +static const struct { + bool result; + unsigned int msec_to_sleep_before; +} expected_results[] __initconst = { + [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, + [PACKETS_BURSTABLE] = { false, 0 }, + [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 2] = { false, 0 }, + [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, + [PACKETS_BURSTABLE + 4] = { true, 0 }, + [PACKETS_BURSTABLE + 5] = { false, 0 } +}; + +static __init unsigned int maximum_jiffies_at_index(int index) +{ + unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; + int i; + + for (i = 0; i <= index; ++i) + total_msecs += expected_results[i].msec_to_sleep_before; + return msecs_to_jiffies(total_msecs); +} + +static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, + struct sk_buff *skb6, struct ipv6hdr *hdr6, + int *test) +{ + unsigned long loop_start_time; + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + loop_start_time = jiffies; + + for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { + if (expected_results[i].msec_to_sleep_before) + msleep(expected_results[i].msec_to_sleep_before); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb4, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb4, &init_net)) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1); + +#if IS_ENABLED(CONFIG_IPV6) + hdr6->saddr.in6_u.u6_addr32[2] = htonl(i); + hdr6->saddr.in6_u.u6_addr32[3] = htonl(i); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb6, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb6, &init_net)) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; +#endif + } + return 0; +} + +static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4, + int *test) +{ + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + + if (atomic_read(&total_entries)) + return -EXFULL; + ++(*test); + + for (i = 0; i <= max_entries; ++i) { + hdr4->saddr = htonl(i); + if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries)) + return -EXFULL; + ++(*test); + } + return 0; +} + +bool __init wg_ratelimiter_selftest(void) +{ + enum { TRIALS_BEFORE_GIVING_UP = 5000 }; + bool success = false; + int test = 0, trials; + struct sk_buff *skb4, *skb6; + struct iphdr *hdr4; + struct ipv6hdr *hdr6; + + if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) + return true; + + BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); + + if (wg_ratelimiter_init()) + goto out; + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + goto out; + } + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + goto out; + } + ++test; + + skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL); + if (unlikely(!skb4)) + goto err_nofree; + skb4->protocol = htons(ETH_P_IP); + hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4)); + hdr4->saddr = htonl(8182); + skb_reset_network_header(skb4); + ++test; + +#if IS_ENABLED(CONFIG_IPV6) + skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL); + if (unlikely(!skb6)) { + kfree_skb(skb4); + goto err_nofree; + } + skb6->protocol = htons(ETH_P_IPV6); + hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6)); + hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212); + hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188); + skb_reset_network_header(skb6); + ++test; +#endif + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0, ret; + + ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); + if (ret == -ETIMEDOUT) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(500); + continue; + } else if (ret < 0) { + test += test_count; + goto err; + } else { + test += test_count; + break; + } + } + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0; + + if (capacity_test(skb4, hdr4, &test_count) < 0) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(50); + continue; + } + test += test_count; + break; + } + + success = true; + +err: + kfree_skb(skb4); +#if IS_ENABLED(CONFIG_IPV6) + kfree_skb(skb6); +#endif +err_nofree: + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + /* Uninit one extra time to check underflow detection. */ + wg_ratelimiter_uninit(); +out: + if (success) + pr_info("ratelimiter self-tests: pass\n"); + else + pr_err("ratelimiter self-test %d: FAIL\n", test); + + return success; +} +#endif diff --git a/net/wireguard/send.c b/net/wireguard/send.c new file mode 100644 index 0000000000000..85b83fafbe4f6 --- /dev/null +++ b/net/wireguard/send.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include +#include + +static void wg_packet_send_handshake_initiation(struct wg_peer *peer) +{ + struct message_handshake_initiation packet; + + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT)) + return; /* This function is rate limited. */ + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), + HANDSHAKE_DSCP); + wg_timers_handshake_initiated(peer); + } +} + +void wg_packet_handshake_send_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + transmit_handshake_work); + + wg_packet_send_handshake_initiation(peer); + wg_peer_put(peer); +} + +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry) +{ + if (!is_retry) + peer->timer_handshake_attempts = 0; + + rcu_read_lock_bh(); + /* We check last_sent_handshake here in addition to the actual function + * we're queueing up, so that we don't queue things if not strictly + * necessary: + */ + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT) || + unlikely(READ_ONCE(peer->is_dead))) + goto out; + + wg_peer_get(peer); + /* Queues up calling packet_send_queued_handshakes(peer), where we do a + * peer_put(peer) after: + */ + if (!queue_work(peer->device->handshake_send_wq, + &peer->transmit_handshake_work)) + /* If the work was already queued, we want to drop the + * extra reference: + */ + wg_peer_put(peer); +out: + rcu_read_unlock_bh(); +} + +void wg_packet_send_handshake_response(struct wg_peer *peer) +{ + struct message_handshake_response packet; + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, + sizeof(packet), + HANDSHAKE_DSCP); + } + } +} + +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index) +{ + struct message_handshake_cookie packet; + + net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", + wg->dev->name, initiating_skb); + wg_cookie_message_create(&packet, initiating_skb, sender_index, + &wg->cookie_checker); + wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, + sizeof(packet)); +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + (unlikely(atomic64_read(&keypair->sending.counter.counter) > + REKEY_AFTER_MESSAGES) || + (keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REKEY_AFTER_TIME))))) + send = true; + rcu_read_unlock_bh(); + + if (send) + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static unsigned int calculate_skb_padding(struct sk_buff *skb) +{ + /* We do this modulo business with the MTU, just in case the networking + * layer gives us a packet that's bigger than the MTU. In that case, we + * wouldn't want the final subtraction to overflow in the case of the + * padded_size being clamped. + */ + unsigned int last_unit = skb->len % PACKET_CB(skb)->mtu; + unsigned int padded_size = ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE); + + if (padded_size > PACKET_CB(skb)->mtu) + padded_size = PACKET_CB(skb)->mtu; + return padded_size - last_unit; +} + +static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair, + simd_context_t *simd_context) +{ + unsigned int padding_len, plaintext_len, trailer_len; + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct message_data *header; + struct sk_buff *trailer; + int num_frags; + + /* Calculate lengths. */ + padding_len = calculate_skb_padding(skb); + trailer_len = padding_len + noise_encrypted_len(0); + plaintext_len = skb->len + padding_len; + + /* Expand data section to have room for padding and auth tag. */ + num_frags = skb_cow_data(skb, trailer_len, &trailer); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + /* Set the padding to zeros, and make sure it and the auth tag are part + * of the skb. + */ + memset(skb_tail_pointer(trailer), 0, padding_len); + + /* Expand head section to have room for our header and the network + * stack's headers. + */ + if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) + return false; + + /* Finalize checksum calculation for the inner packet, if required. */ + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_help(skb))) + return false; + + /* Only after checksumming can we safely add on the padding at the end + * and the header. + */ + skb_set_inner_network_header(skb, 0); + header = (struct message_data *)skb_push(skb, sizeof(*header)); + header->header.type = cpu_to_le32(MESSAGE_DATA); + header->key_idx = keypair->remote_index; + header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); + pskb_put(skb, trailer, trailer_len); + + /* Now we can encrypt the scattergather segments */ + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, sizeof(struct message_data), + noise_encrypted_len(plaintext_len)) <= 0) + return false; + return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + keypair->sending.key, + simd_context); +} + +void wg_packet_send_keepalive(struct wg_peer *peer) +{ + struct sk_buff *skb; + + if (skb_queue_empty(&peer->staged_packet_queue)) { + skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, + GFP_ATOMIC); + if (unlikely(!skb)) + return; + skb_reserve(skb, DATA_PACKET_HEAD_ROOM); + skb->dev = peer->device->dev; + PACKET_CB(skb)->mtu = skb->dev->mtu; + skb_queue_tail(&peer->staged_packet_queue, skb); + net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + } + + wg_packet_send_staged_packets(peer); +} + +static void wg_packet_create_data_done(struct sk_buff *first, + struct wg_peer *peer) +{ + struct sk_buff *skb, *next; + bool is_keepalive, data_sent = false; + + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + skb_list_walk_safe(first, skb, next) { + is_keepalive = skb->len == message_data_len(0); + if (likely(!wg_socket_send_skb_to_peer(peer, skb, + PACKET_CB(skb)->ds) && !is_keepalive)) + data_sent = true; + } + + if (likely(data_sent)) + wg_timers_data_sent(peer); + + keep_key_fresh(peer); +} + +void wg_packet_tx_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct crypt_queue, + work); + struct noise_keypair *keypair; + enum packet_state state; + struct sk_buff *first; + struct wg_peer *peer; + + while ((first = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(first)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(first); + keypair = PACKET_CB(first)->keypair; + + if (likely(state == PACKET_STATE_CRYPTED)) + wg_packet_create_data_done(first, peer); + else + kfree_skb_list(first); + + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + } +} + +void wg_packet_encrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *first, *skb, *next; + simd_context_t simd_context; + + simd_get(&simd_context); + while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = PACKET_STATE_CRYPTED; + + skb_list_walk_safe(first, skb, next) { + if (likely(encrypt_packet(skb, + PACKET_CB(first)->keypair, + &simd_context))) { + wg_reset_packet(skb); + } else { + state = PACKET_STATE_DEAD; + break; + } + } + wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, + state); + + simd_relax(&simd_context); + } + simd_put(&simd_context); +} + +static void wg_packet_create_data(struct sk_buff *first) +{ + struct wg_peer *peer = PACKET_PEER(first); + struct wg_device *wg = peer->device; + int ret = -EINVAL; + + rcu_read_lock_bh(); + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, + &peer->tx_queue, first, + wg->packet_crypt_wq, + &wg->encrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer(&peer->tx_queue, first, + PACKET_STATE_DEAD); +err: + rcu_read_unlock_bh(); + if (likely(!ret || ret == -EPIPE)) + return; + wg_noise_keypair_put(PACKET_CB(first)->keypair, false); + wg_peer_put(peer); + kfree_skb_list(first); +} + +void wg_packet_purge_staged_packets(struct wg_peer *peer) +{ + spin_lock_bh(&peer->staged_packet_queue.lock); + peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; + __skb_queue_purge(&peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); +} + +void wg_packet_send_staged_packets(struct wg_peer *peer) +{ + struct noise_symmetric_key *key; + struct noise_keypair *keypair; + struct sk_buff_head packets; + struct sk_buff *skb; + + /* Steal the current queue into our local one. */ + __skb_queue_head_init(&packets); + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice_init(&peer->staged_packet_queue, &packets); + spin_unlock_bh(&peer->staged_packet_queue.lock); + if (unlikely(skb_queue_empty(&packets))) + return; + + /* First we make sure we have a valid reference to a valid key. */ + rcu_read_lock_bh(); + keypair = wg_noise_keypair_get( + rcu_dereference_bh(peer->keypairs.current_keypair)); + rcu_read_unlock_bh(); + if (unlikely(!keypair)) + goto out_nokey; + key = &keypair->sending; + if (unlikely(!READ_ONCE(key->is_valid))) + goto out_nokey; + if (unlikely(wg_birthdate_has_expired(key->birthdate, + REJECT_AFTER_TIME))) + goto out_invalid; + + /* After we know we have a somewhat valid key, we now try to assign + * nonces to all of the packets in the queue. If we can't assign nonces + * for all of them, we just consider it a failure and wait for the next + * handshake. + */ + skb_queue_walk(&packets, skb) { + /* 0 for no outer TOS: no leak. TODO: at some later point, we + * might consider using flowi->tos as outer instead. + */ + PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); + PACKET_CB(skb)->nonce = + atomic64_inc_return(&key->counter.counter) - 1; + if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) + goto out_invalid; + } + + packets.prev->next = NULL; + wg_peer_get(keypair->entry.peer); + PACKET_CB(packets.next)->keypair = keypair; + wg_packet_create_data(packets.next); + return; + +out_invalid: + WRITE_ONCE(key->is_valid, false); +out_nokey: + wg_noise_keypair_put(keypair, false); + + /* We orphan the packets if we're waiting on a handshake, so that they + * don't block a socket's pool. + */ + skb_queue_walk(&packets, skb) + skb_orphan(skb); + /* Then we put them back on the top of the queue. We're not too + * concerned about accidentally getting things a little out of order if + * packets are being added really fast, because this queue is for before + * packets can even be sent and it's small anyway. + */ + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + /* If we're exiting because there's something wrong with the key, it + * means we should initiate a new handshake. + */ + wg_packet_send_queued_handshake_initiation(peer, false); +} diff --git a/net/wireguard/socket.c b/net/wireguard/socket.c new file mode 100644 index 0000000000000..c46256d0d81c1 --- /dev/null +++ b/net/wireguard/socket.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include + +static int send4(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ + struct flowi4 fl = { + .saddr = endpoint->src4.s_addr, + .daddr = endpoint->addr4.sin_addr.s_addr, + .fl4_dport = endpoint->addr4.sin_port, + .flowi4_mark = wg->fwmark, + .flowi4_proto = IPPROTO_UDP + }; + struct rtable *rt = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock4); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl4_sport = inet_sk(sock)->inet_sport; + + if (cache) + rt = dst_cache_get_ip4(cache, &fl.saddr); + + if (!rt) { + security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); + if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + } + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && + PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && + rt->dst.dev->ifindex != endpoint->src_if4)))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + if (!IS_ERR(rt)) + ip_rt_put(rt); + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + } + if (unlikely(IS_ERR(rt))) { + ret = PTR_ERR(rt); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(rt->dst.dev == skb->dev)) { + ip_rt_put(rt); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +} + +static int send6(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct flowi6 fl = { + .saddr = endpoint->src6, + .daddr = endpoint->addr6.sin6_addr, + .fl6_dport = endpoint->addr6.sin6_port, + .flowi6_mark = wg->fwmark, + .flowi6_oif = endpoint->addr6.sin6_scope_id, + .flowi6_proto = IPPROTO_UDP + /* TODO: addr->sin6_flowinfo */ + }; + struct dst_entry *dst = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock6); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl6_sport = inet_sk(sock)->inet_sport; + + if (cache) + dst = dst_cache_get_ip6(cache, &fl.saddr); + + if (!dst) { + security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); + if (unlikely(!ipv6_addr_any(&fl.saddr) && + !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { + endpoint->src6 = fl.saddr = in6addr_any; + if (cache) + dst_cache_reset(cache); + } + dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, + NULL); + if (unlikely(IS_ERR(dst))) { + ret = PTR_ERR(dst); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(dst->dev == skb->dev)) { + dst_release(dst); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip6(cache, dst, &fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +#else + return -EAFNOSUPPORT; +#endif +} + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +{ + size_t skb_len = skb->len; + int ret = -EAFNOSUPPORT; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + ret = send4(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else if (peer->endpoint.addr.sa_family == AF_INET6) + ret = send6(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else + dev_kfree_skb(skb); + if (likely(!ret)) + peer->tx_bytes += skb_len; + read_unlock_bh(&peer->endpoint_lock); + + return ret; +} + +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, + size_t len, u8 ds) +{ + struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + + if (unlikely(!skb)) + return -ENOMEM; + + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + return wg_socket_send_skb_to_peer(peer, skb, ds); +} + +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, void *buffer, + size_t len) +{ + int ret = 0; + struct sk_buff *skb; + struct endpoint endpoint; + + if (unlikely(!in_skb)) + return -EINVAL; + ret = wg_socket_endpoint_from_skb(&endpoint, in_skb); + if (unlikely(ret < 0)) + return ret; + + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + if (unlikely(!skb)) + return -ENOMEM; + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + + if (endpoint.addr.sa_family == AF_INET) + ret = send4(wg, skb, &endpoint, 0, NULL); + else if (endpoint.addr.sa_family == AF_INET6) + ret = send6(wg, skb, &endpoint, 0, NULL); + /* No other possibilities if the endpoint is valid, which it is, + * as we checked above. + */ + + return ret; +} + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb) +{ + memset(endpoint, 0, sizeof(*endpoint)); + if (skb->protocol == htons(ETH_P_IP)) { + endpoint->addr4.sin_family = AF_INET; + endpoint->addr4.sin_port = udp_hdr(skb)->source; + endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; + endpoint->src4.s_addr = ip_hdr(skb)->daddr; + endpoint->src_if4 = skb->skb_iif; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + endpoint->addr6.sin6_family = AF_INET6; + endpoint->addr6.sin6_port = udp_hdr(skb)->source; + endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; + endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( + &ipv6_hdr(skb)->saddr, skb->skb_iif); + endpoint->src6 = ipv6_hdr(skb)->daddr; + } else { + return -EINVAL; + } + return 0; +} + +static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) +{ + return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && + a->addr4.sin_port == b->addr4.sin_port && + a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && + a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || + (a->addr.sa_family == AF_INET6 && + b->addr.sa_family == AF_INET6 && + a->addr6.sin6_port == b->addr6.sin6_port && + ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) && + a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && + ipv6_addr_equal(&a->src6, &b->src6)) || + unlikely(!a->addr.sa_family && !b->addr.sa_family); +} + +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (endpoint_eq(endpoint, &peer->endpoint)) + return; + write_lock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) { + peer->endpoint.addr4 = endpoint->addr4; + peer->endpoint.src4 = endpoint->src4; + peer->endpoint.src_if4 = endpoint->src_if4; + } else if (endpoint->addr.sa_family == AF_INET6) { + peer->endpoint.addr6 = endpoint->addr6; + peer->endpoint.src6 = endpoint->src6; + } else { + goto out; + } + dst_cache_reset(&peer->endpoint_cache); +out: + write_unlock_bh(&peer->endpoint_lock); +} + +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb) +{ + struct endpoint endpoint; + + if (!wg_socket_endpoint_from_skb(&endpoint, skb)) + wg_socket_set_peer_endpoint(peer, &endpoint); +} + +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +{ + write_lock_bh(&peer->endpoint_lock); + memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); + dst_cache_reset(&peer->endpoint_cache); + write_unlock_bh(&peer->endpoint_lock); +} + +static int wg_receive(struct sock *sk, struct sk_buff *skb) +{ + struct wg_device *wg; + + if (unlikely(!sk)) + goto err; + wg = sk->sk_user_data; + if (unlikely(!wg)) + goto err; + wg_packet_receive(wg, skb); + return 0; + +err: + kfree_skb(skb); + return 0; +} + +static void sock_free(struct sock *sock) +{ + if (unlikely(!sock)) + return; + sk_clear_memalloc(sock); + udp_tunnel_sock_release(sock->sk_socket); +} + +static void set_sock_opts(struct socket *sock) +{ + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_sndbuf = INT_MAX; + sk_set_memalloc(sock->sk); +} + +int wg_socket_init(struct wg_device *wg, u16 port) +{ + int ret; + struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = wg, + .encap_type = 1, + .encap_rcv = wg_receive + }; + struct socket *new4 = NULL, *new6 = NULL; + struct udp_port_cfg port4 = { + .family = AF_INET, + .local_ip.s_addr = htonl(INADDR_ANY), + .local_udp_port = htons(port), + .use_udp_checksums = true + }; +#if IS_ENABLED(CONFIG_IPV6) + int retries = 0; + struct udp_port_cfg port6 = { + .family = AF_INET6, + .local_ip6 = IN6ADDR_ANY_INIT, + .use_udp6_tx_checksums = true, + .use_udp6_rx_checksums = true, + .ipv6_v6only = true + }; +#endif + +#if IS_ENABLED(CONFIG_IPV6) +retry: +#endif + + ret = udp_sock_create(wg->creating_net, &port4, &new4); + if (ret < 0) { + pr_err("%s: Could not create IPv4 socket\n", wg->dev->name); + return ret; + } + set_sock_opts(new4); + setup_udp_tunnel_sock(wg->creating_net, new4, &cfg); + +#if IS_ENABLED(CONFIG_IPV6) + if (ipv6_mod_enabled()) { + port6.local_udp_port = inet_sk(new4->sk)->inet_sport; + ret = udp_sock_create(wg->creating_net, &port6, &new6); + if (ret < 0) { + udp_tunnel_sock_release(new4); + if (ret == -EADDRINUSE && !port && retries++ < 100) + goto retry; + pr_err("%s: Could not create IPv6 socket\n", + wg->dev->name); + return ret; + } + set_sock_opts(new6); + setup_udp_tunnel_sock(wg->creating_net, new6, &cfg); + } +#endif + + wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL); + return 0; +} + +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6) +{ + struct sock *old4, *old6; + + mutex_lock(&wg->socket_update_lock); + old4 = rcu_dereference_protected(wg->sock4, + lockdep_is_held(&wg->socket_update_lock)); + old6 = rcu_dereference_protected(wg->sock6, + lockdep_is_held(&wg->socket_update_lock)); + rcu_assign_pointer(wg->sock4, new4); + rcu_assign_pointer(wg->sock6, new6); + if (new4) + wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); + mutex_unlock(&wg->socket_update_lock); + synchronize_rcu(); + synchronize_net(); + sock_free(old4); + sock_free(old6); +} diff --git a/net/wireguard/socket.h b/net/wireguard/socket.h new file mode 100644 index 0000000000000..bab5848efbcdf --- /dev/null +++ b/net/wireguard/socket.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SOCKET_H +#define _WG_SOCKET_H + +#include +#include +#include +#include + +int wg_socket_init(struct wg_device *wg, u16 port); +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6); +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, + size_t len, u8 ds); +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, + u8 ds); +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, + void *out_buffer, size_t len); + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb); +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb); +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); + +#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) +#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ + struct endpoint __endpoint; \ + wg_socket_endpoint_from_skb(&__endpoint, skb); \ + net_dbg_ratelimited(fmt, dev, &__endpoint.addr, \ + ##__VA_ARGS__); \ + } while (0) +#else +#define net_dbg_skb_ratelimited(fmt, skb, ...) +#endif + +#endif /* _WG_SOCKET_H */ diff --git a/net/wireguard/timers.c b/net/wireguard/timers.c new file mode 100644 index 0000000000000..d54d32ac9bc41 --- /dev/null +++ b/net/wireguard/timers.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "queueing.h" +#include "socket.h" + +/* + * - Timer for retransmitting the handshake if we don't hear back after + * `REKEY_TIMEOUT + jitter` ms. + * + * - Timer for sending empty packet if we have received a packet but after have + * not sent one for `KEEPALIVE_TIMEOUT` ms. + * + * - Timer for initiating new handshake if we have sent a packet but after have + * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + * jitter` ms. + * + * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms + * if no new keys have been received. + * + * - Timer for, if enabled, sending an empty authenticated packet every user- + * specified seconds. + */ + +static inline void mod_peer_timer(struct wg_peer *peer, + struct timer_list *timer, + unsigned long expires) +{ + rcu_read_lock_bh(); + if (likely(netif_running(peer->device->dev) && + !READ_ONCE(peer->is_dead))) + mod_timer(timer, expires); + rcu_read_unlock_bh(); +} + +static void wg_expired_retransmit_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_retransmit_handshake); + + if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); + + del_timer(&peer->timer_send_keepalive); + /* We drop all packets without a keypair and don't try again, + * if we try unsuccessfully for too long to make a handshake. + */ + wg_packet_purge_staged_packets(peer); + + /* We set a timer for destroying any residue that might be left + * of a partial exchange. + */ + if (!timer_pending(&peer->timer_zero_key_material)) + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); + } else { + ++peer->timer_handshake_attempts; + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REKEY_TIMEOUT, + peer->timer_handshake_attempts + 1); + + /* We clear the endpoint address src address, in case this is + * the cause of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + + wg_packet_send_queued_handshake_initiation(peer, true); + } +} + +static void wg_expired_send_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); + + wg_packet_send_keepalive(peer); + if (peer->timer_need_another_keepalive) { + peer->timer_need_another_keepalive = false; + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + } +} + +static void wg_expired_new_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); + + pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + /* We clear the endpoint address src address, in case this is the cause + * of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static void wg_expired_zero_key_material(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); + + rcu_read_lock_bh(); + if (!READ_ONCE(peer->is_dead)) { + wg_peer_get(peer); + if (!queue_work(peer->device->handshake_send_wq, + &peer->clear_peer_work)) + /* If the work was already on the queue, we want to drop + * the extra reference. + */ + wg_peer_put(peer); + } + rcu_read_unlock_bh(); +} + +static void wg_queued_expired_zero_key_material(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + clear_peer_work); + + pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REJECT_AFTER_TIME * 3); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_peer_put(peer); +} + +static void wg_expired_send_persistent_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_persistent_keepalive); + + if (likely(peer->persistent_keepalive_interval)) + wg_packet_send_keepalive(peer); +} + +/* Should be called after an authenticated data packet is sent. */ +void wg_timers_data_sent(struct wg_peer *peer) +{ + if (!timer_pending(&peer->timer_new_handshake)) + mod_peer_timer(peer, &peer->timer_new_handshake, + jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after an authenticated data packet is received. */ +void wg_timers_data_received(struct wg_peer *peer) +{ + if (likely(netif_running(peer->device->dev))) { + if (!timer_pending(&peer->timer_send_keepalive)) + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + else + peer->timer_need_another_keepalive = true; + } +} + +/* Should be called after any type of authenticated packet is sent, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) +{ + del_timer(&peer->timer_send_keepalive); +} + +/* Should be called after any type of authenticated packet is received, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) +{ + del_timer(&peer->timer_new_handshake); +} + +/* Should be called after a handshake initiation message is sent. */ +void wg_timers_handshake_initiated(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_retransmit_handshake, + jiffies + REKEY_TIMEOUT * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after a handshake response message is received and processed + * or when getting key confirmation via the first data message. + */ +void wg_timers_handshake_complete(struct wg_peer *peer) +{ + del_timer(&peer->timer_retransmit_handshake); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + ktime_get_real_ts64(&peer->walltime_last_handshake); +} + +/* Should be called after an ephemeral key is created, which is before sending a + * handshake response or after receiving a handshake response. + */ +void wg_timers_session_derived(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); +} + +/* Should be called before a packet with authentication, whether + * keepalive, data, or handshakem is sent, or after one is received. + */ +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer) +{ + if (peer->persistent_keepalive_interval) + mod_peer_timer(peer, &peer->timer_persistent_keepalive, + jiffies + peer->persistent_keepalive_interval * HZ); +} + +void wg_timers_init(struct wg_peer *peer) +{ + timer_setup(&peer->timer_retransmit_handshake, + wg_expired_retransmit_handshake, 0); + timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0); + timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0); + timer_setup(&peer->timer_zero_key_material, + wg_expired_zero_key_material, 0); + timer_setup(&peer->timer_persistent_keepalive, + wg_expired_send_persistent_keepalive, 0); + INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + peer->timer_need_another_keepalive = false; +} + +void wg_timers_stop(struct wg_peer *peer) +{ + del_timer_sync(&peer->timer_retransmit_handshake); + del_timer_sync(&peer->timer_send_keepalive); + del_timer_sync(&peer->timer_new_handshake); + del_timer_sync(&peer->timer_zero_key_material); + del_timer_sync(&peer->timer_persistent_keepalive); + flush_work(&peer->clear_peer_work); +} diff --git a/net/wireguard/timers.h b/net/wireguard/timers.h new file mode 100644 index 0000000000000..f0653dcb13266 --- /dev/null +++ b/net/wireguard/timers.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_TIMERS_H +#define _WG_TIMERS_H + +#include + +struct wg_peer; + +void wg_timers_init(struct wg_peer *peer); +void wg_timers_stop(struct wg_peer *peer); +void wg_timers_data_sent(struct wg_peer *peer); +void wg_timers_data_received(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer); +void wg_timers_handshake_initiated(struct wg_peer *peer); +void wg_timers_handshake_complete(struct wg_peer *peer); +void wg_timers_session_derived(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer); + +static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds, + u64 expiration_seconds) +{ + return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC) + <= (s64)ktime_get_coarse_boottime_ns(); +} + +#endif /* _WG_TIMERS_H */ diff --git a/net/wireguard/uapi/wireguard.h b/net/wireguard/uapi/wireguard.h new file mode 100644 index 0000000000000..ae88be14c9478 --- /dev/null +++ b/net/wireguard/uapi/wireguard.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Documentation + * ============= + * + * The below enums and macros are for interfacing with WireGuard, using generic + * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two + * methods: get and set. Note that while they share many common attributes, + * these two functions actually accept a slightly different set of inputs and + * outputs. + * + * WG_CMD_GET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain + * one but not both of: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * + * The kernel will then return several messages (NLM_F_MULTI) containing the + * following tree of nested items: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_LISTEN_PORT: NLA_U16 + * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16 + * WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec + * WGPEER_A_RX_BYTES: NLA_U64 + * WGPEER_A_TX_BYTES: NLA_U64 + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32 + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that all of the allowed IPs of a single peer will not + * fit within a single netlink message. In that case, the same peer will + * be written in the following message, except it will only contain + * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several + * times in a row for the same peer. It is then up to the receiver to + * coalesce adjacent peers. Likewise, it is possible that all peers will + * not fit within a single message. So, subsequent peers will be sent + * in following messages, except those will only contain WGDEVICE_A_IFNAME + * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these + * messages to form the complete list of peers. + * + * Since this is an NLA_F_DUMP command, the final message will always be + * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message + * contains an integer error code. It is either zero or a negative error + * code corresponding to the errno. + * + * WG_CMD_SET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST. The command should contain the + * following tree of nested items, containing one but not both of + * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current + * peers should be removed prior to adding the list below. + * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove + * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly + * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN + * WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the + * specified peer should not exist at the end of the + * operation, rather than added/updated and/or + * WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed + * IPs of this peer should be removed prior to adding + * the list below and/or WGPEER_F_UPDATE_ONLY if the + * peer should only be set if it already exists. + * WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove + * WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at + * all by most users of this API, as the + * most recent protocol will be used when + * this is unset. Otherwise, must be set + * to 1. + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that the amount of configuration data exceeds that of + * the maximum message length accepted by the kernel. In that case, several + * messages should be sent one after another, with each successive one + * filling in information not contained in the prior. Note that if + * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably + * should not be specified in fragments that come after, so that the list + * of peers is only cleared the first time but appended after. Likewise for + * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message + * of a peer, it likely should not be specified in subsequent fragments. + * + * If an error occurs, NLMSG_ERROR will reply containing an errno. + */ + +#ifndef _WG_UAPI_WIREGUARD_H +#define _WG_UAPI_WIREGUARD_H + +#define WG_GENL_NAME "wireguard" +#define WG_GENL_VERSION 1 + +#define WG_KEY_LEN 32 + +enum wg_cmd { + WG_CMD_GET_DEVICE, + WG_CMD_SET_DEVICE, + __WG_CMD_MAX +}; +#define WG_CMD_MAX (__WG_CMD_MAX - 1) + +enum wgdevice_flag { + WGDEVICE_F_REPLACE_PEERS = 1U << 0, + __WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS +}; +enum wgdevice_attribute { + WGDEVICE_A_UNSPEC, + WGDEVICE_A_IFINDEX, + WGDEVICE_A_IFNAME, + WGDEVICE_A_PRIVATE_KEY, + WGDEVICE_A_PUBLIC_KEY, + WGDEVICE_A_FLAGS, + WGDEVICE_A_LISTEN_PORT, + WGDEVICE_A_FWMARK, + WGDEVICE_A_PEERS, + __WGDEVICE_A_LAST +}; +#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) + +enum wgpeer_flag { + WGPEER_F_REMOVE_ME = 1U << 0, + WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1, + WGPEER_F_UPDATE_ONLY = 1U << 2, + __WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS | + WGPEER_F_UPDATE_ONLY +}; +enum wgpeer_attribute { + WGPEER_A_UNSPEC, + WGPEER_A_PUBLIC_KEY, + WGPEER_A_PRESHARED_KEY, + WGPEER_A_FLAGS, + WGPEER_A_ENDPOINT, + WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + WGPEER_A_LAST_HANDSHAKE_TIME, + WGPEER_A_RX_BYTES, + WGPEER_A_TX_BYTES, + WGPEER_A_ALLOWEDIPS, + WGPEER_A_PROTOCOL_VERSION, + __WGPEER_A_LAST +}; +#define WGPEER_A_MAX (__WGPEER_A_LAST - 1) + +enum wgallowedip_attribute { + WGALLOWEDIP_A_UNSPEC, + WGALLOWEDIP_A_FAMILY, + WGALLOWEDIP_A_IPADDR, + WGALLOWEDIP_A_CIDR_MASK, + __WGALLOWEDIP_A_LAST +}; +#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) + +#endif /* _WG_UAPI_WIREGUARD_H */ diff --git a/net/wireguard/version.h b/net/wireguard/version.h new file mode 100644 index 0000000000000..ddba10344448b --- /dev/null +++ b/net/wireguard/version.h @@ -0,0 +1 @@ +#define WIREGUARD_VERSION "0.0.20191219" diff --git a/net/wireless/core.h b/net/wireless/core.h index 90f90c7d8bf9b..eaff636169c22 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -217,6 +217,7 @@ enum cfg80211_event_type { EVENT_DISCONNECTED, EVENT_IBSS_JOINED, EVENT_STOPPED, + EVENT_PORT_AUTHORIZED, }; struct cfg80211_event { @@ -236,6 +237,9 @@ struct cfg80211_event { u8 bssid[ETH_ALEN]; struct ieee80211_channel *channel; } ij; + struct { + u8 bssid[ETH_ALEN]; + } pa; }; }; @@ -386,6 +390,7 @@ int cfg80211_disconnect(struct cfg80211_registered_device *rdev, bool wextev); void __cfg80211_roamed(struct wireless_dev *wdev, struct cfg80211_roam_info *info); +void __cfg80211_port_authorized(struct wireless_dev *wdev, const u8 *bssid); int cfg80211_mgd_wext_connect(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev); void cfg80211_autodisconnect_wk(struct work_struct *work); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 46e9812d13c02..8e5324591a8c5 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -421,6 +421,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_FILS_CACHE_ID] = { .len = 2 }, [NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN }, [NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG }, + [NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY, + .len = SAE_PASSWORD_MAX_LEN }, }; /* policy for the key attributes */ @@ -3864,9 +3866,12 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return false; return true; case NL80211_CMD_CONNECT: - /* SAE not supported yet */ - if (auth_type == NL80211_AUTHTYPE_SAE) + if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && + auth_type == NL80211_AUTHTYPE_SAE) return false; + /* FILS with SK PFS or PK not supported yet */ if (auth_type == NL80211_AUTHTYPE_FILS_SK_PFS || auth_type == NL80211_AUTHTYPE_FILS_PK) @@ -3878,8 +3883,9 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return false; return true; case NL80211_CMD_START_AP: - /* SAE not supported yet */ - if (auth_type == NL80211_AUTHTYPE_SAE) + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && + auth_type == NL80211_AUTHTYPE_SAE) return false; /* FILS not supported yet */ if (auth_type == NL80211_AUTHTYPE_FILS_SK || @@ -7967,7 +7973,8 @@ static int nl80211_dump_survey(struct sk_buff *skb, struct netlink_callback *cb) static bool nl80211_valid_wpa_versions(u32 wpa_versions) { return !(wpa_versions & ~(NL80211_WPA_VERSION_1 | - NL80211_WPA_VERSION_2)); + NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)); } static int nl80211_authenticate(struct sk_buff *skb, struct genl_info *info) @@ -8176,11 +8183,23 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, if (nla_len(info->attrs[NL80211_ATTR_PMK]) != WLAN_PMK_LEN) return -EINVAL; if (!wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK)) + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK)) return -EINVAL; settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } + if (info->attrs[NL80211_ATTR_SAE_PASSWORD]) { + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD)) + return -EINVAL; + settings->sae_pwd = + nla_data(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + settings->sae_pwd_len = + nla_len(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + } + return 0; } @@ -13766,6 +13785,8 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev, (nla_put_flag(msg, NL80211_ATTR_TIMED_OUT) || nla_put_u32(msg, NL80211_ATTR_TIMEOUT_REASON, cr->timeout_reason))) || + (cr->authorized && + nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) || (cr->req_ie && nla_put(msg, NL80211_ATTR_REQ_IE, cr->req_ie_len, cr->req_ie)) || (cr->resp_ie && @@ -13837,6 +13858,37 @@ void nl80211_send_roamed(struct cfg80211_registered_device *rdev, nlmsg_free(msg); } +void nl80211_send_port_authorized(struct cfg80211_registered_device *rdev, + struct net_device *netdev, const u8 *bssid) +{ + struct sk_buff *msg; + void *hdr; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return; + + hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_PORT_AUTHORIZED); + if (!hdr) { + nlmsg_free(msg); + return; + } + + if (nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex) || + nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid)) + goto nla_put_failure; + + genlmsg_end(msg, hdr); + + genlmsg_multicast_netns(&nl80211_fam, wiphy_net(&rdev->wiphy), msg, 0, + NL80211_MCGRP_MLME, GFP_KERNEL); + return; + + nla_put_failure: + genlmsg_cancel(msg, hdr); + nlmsg_free(msg); +} + void nl80211_send_disconnected(struct cfg80211_registered_device *rdev, struct net_device *netdev, u16 reason, const u8 *ie, size_t ie_len, bool from_ap) diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h index fc415c8f7aac4..79e47fe60c352 100644 --- a/net/wireless/nl80211.h +++ b/net/wireless/nl80211.h @@ -59,6 +59,8 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev, void nl80211_send_roamed(struct cfg80211_registered_device *rdev, struct net_device *netdev, struct cfg80211_roam_info *info, gfp_t gfp); +void nl80211_send_port_authorized(struct cfg80211_registered_device *rdev, + struct net_device *netdev, const u8 *bssid); void nl80211_send_disconnected(struct cfg80211_registered_device *rdev, struct net_device *netdev, u16 reason, const u8 *ie, size_t ie_len, bool from_ap); diff --git a/net/wireless/sme.c b/net/wireless/sme.c index d014aea07160c..e4cd1e5d16ed5 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -857,6 +857,7 @@ void cfg80211_connect_done(struct net_device *dev, ev->cr.bss = params->bss; ev->cr.status = params->status; ev->cr.timeout_reason = params->timeout_reason; + ev->cr.authorized = params->authorized; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); @@ -965,6 +966,50 @@ void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info, } EXPORT_SYMBOL(cfg80211_roamed); +void __cfg80211_port_authorized(struct wireless_dev *wdev, const u8 *bssid) +{ + ASSERT_WDEV_LOCK(wdev); + + if (WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION)) + return; + + if (WARN_ON(!wdev->current_bss) || + WARN_ON(!ether_addr_equal(wdev->current_bss->pub.bssid, bssid))) + return; + + nl80211_send_port_authorized(wiphy_to_rdev(wdev->wiphy), wdev->netdev, + bssid); +} + +void cfg80211_port_authorized(struct net_device *dev, const u8 *bssid, + gfp_t gfp) +{ + struct wireless_dev *wdev = dev->ieee80211_ptr; + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy); + struct cfg80211_event *ev; + unsigned long flags; + + if (WARN_ON(!bssid)) + return; + + ev = kzalloc(sizeof(*ev), gfp); + if (!ev) + return; + + ev->type = EVENT_PORT_AUTHORIZED; + memcpy(ev->pa.bssid, bssid, ETH_ALEN); + + /* + * Use the wdev event list so that if there are pending + * connected/roamed events, they will be reported first. + */ + spin_lock_irqsave(&wdev->event_lock, flags); + list_add_tail(&ev->list, &wdev->event_list); + spin_unlock_irqrestore(&wdev->event_lock, flags); + queue_work(cfg80211_wq, &rdev->event_work); +} +EXPORT_SYMBOL(cfg80211_port_authorized); + void __cfg80211_disconnected(struct net_device *dev, const u8 *ie, size_t ie_len, u16 reason, bool from_ap) { diff --git a/net/wireless/util.c b/net/wireless/util.c index ca3361a3e750e..790eaa603a22d 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -964,6 +964,9 @@ void cfg80211_process_wdev_events(struct wireless_dev *wdev) case EVENT_STOPPED: __cfg80211_leave(wiphy_to_rdev(wdev->wiphy), wdev); break; + case EVENT_PORT_AUTHORIZED: + __cfg80211_port_authorized(wdev, ev->pa.bssid); + break; } wdev_unlock(wdev); diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile index 19a870eed82b3..368987fd8eec7 100644 --- a/samples/seccomp/Makefile +++ b/samples/seccomp/Makefile @@ -21,6 +21,7 @@ bpf-direct-objs := bpf-direct.o # Try to match the kernel target. ifndef CROSS_COMPILE ifndef CONFIG_64BIT +ifndef CONFIG_ARM # s390 has -m31 flag to build 31 bit binaries ifndef CONFIG_S390 @@ -47,3 +48,4 @@ ifndef CONFIG_MIPS always := $(hostprogs-m) endif endif +endif diff --git a/scripts/package/Makefile b/scripts/package/Makefile index 348af5b20618e..ca7f46b562a4f 100644 --- a/scripts/package/Makefile +++ b/scripts/package/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # Makefile for the different targets used to generate full packages of a kernel # It uses the generic clean infrastructure of kbuild @@ -24,6 +25,7 @@ # Remove hyphens since they have special meaning in RPM filenames KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE)) KDEB_SOURCENAME ?= linux-$(KERNELRELEASE) +KBUILD_PKG_ROOTCMD ?="fakeroot -u" export KDEB_SOURCENAME # Include only those top-level files that are needed by make, plus the GPL copy TAR_CONTENT := $(KBUILD_ALLDIRS) .config .scmversion Makefile \ @@ -33,9 +35,12 @@ MKSPEC := $(srctree)/scripts/package/mkspec quiet_cmd_src_tar = TAR $(2).tar.gz cmd_src_tar = \ if test "$(objtree)" != "$(srctree)"; then \ - echo "Building source tarball is not possible outside the"; \ - echo "kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \ - echo "binrpm-pkg or bindeb-pkg target instead."; \ + echo >&2; \ + echo >&2 " ERROR:"; \ + echo >&2 " Building source tarball is not possible outside the"; \ + echo >&2 " kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \ + echo >&2 " binrpm-pkg or bindeb-pkg target instead."; \ + echo >&2; \ false; \ fi ; \ $(srctree)/scripts/setlocalversion --save-scmversion; \ @@ -45,60 +50,61 @@ rm -f $(objtree)/.scmversion # rpm-pkg # --------------------------------------------------------------------------- -rpm-pkg rpm: FORCE +rpm-pkg: FORCE $(MAKE) clean $(CONFIG_SHELL) $(MKSPEC) >$(objtree)/kernel.spec $(call cmd,src_tar,$(KERNELPATH),kernel.spec) - +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz + +rpmbuild $(RPMOPTS) --target $(UTS_MACHINE) -ta $(KERNELPATH).tar.gz \ + --define='_smp_mflags %{nil}' # binrpm-pkg # --------------------------------------------------------------------------- binrpm-pkg: FORCE - $(MAKE) KBUILD_SRC= + $(MAKE) -f $(srctree)/Makefile $(CONFIG_SHELL) $(MKSPEC) prebuilt > $(objtree)/binkernel.spec +rpmbuild $(RPMOPTS) --define "_builddir $(objtree)" --target \ $(UTS_MACHINE) -bb $(objtree)/binkernel.spec clean-files += $(objtree)/*.spec -# Deb target -# --------------------------------------------------------------------------- -quiet_cmd_builddeb = BUILDDEB - cmd_builddeb = set -e; \ - test `id -u` = 0 || \ - test -n "$(KBUILD_PKG_ROOTCMD)" || { \ - which fakeroot >/dev/null 2>&1 && \ - KBUILD_PKG_ROOTCMD="fakeroot -u"; \ - } || { \ - echo; \ - echo "builddeb must be run as root (or using fakeroot)."; \ - echo "KBUILD_PKG_ROOTCMD is unset and fakeroot not found."; \ - echo "Try setting KBUILD_PKG_ROOTCMD to a command to acquire"; \ - echo "root privileges (e.g., 'fakeroot -u' or 'sudo')."; \ - false; \ - } && \ - \ - $$KBUILD_PKG_ROOTCMD $(CONFIG_SHELL) \ - $(srctree)/scripts/package/builddeb $@ - deb-pkg: FORCE $(MAKE) clean + $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian $(call cmd,src_tar,$(KDEB_SOURCENAME)) - $(MAKE) KBUILD_SRC= - +$(call cmd,builddeb) + origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\ + mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -i.git -us -uc bindeb-pkg: FORCE - $(MAKE) KBUILD_SRC= - +$(call cmd,builddeb) + $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -b -nc -uc + +intdeb-pkg: FORCE + +$(CONFIG_SHELL) $(srctree)/scripts/package/builddeb clean-dirs += $(objtree)/debian/ +# snap-pkg +# --------------------------------------------------------------------------- +snap-pkg: FORCE + rm -rf $(objtree)/snap + mkdir $(objtree)/snap + $(MAKE) clean + $(call cmd,src_tar,$(KERNELPATH)) + sed "s@KERNELRELEASE@$(KERNELRELEASE)@; \ + s@SRCTREE@$(shell realpath $(KERNELPATH).tar.gz)@" \ + $(srctree)/scripts/package/snapcraft.template > \ + $(objtree)/snap/snapcraft.yaml + cd $(objtree)/snap && \ + snapcraft --target-arch=$(UTS_MACHINE) + +clean-dirs += $(objtree)/snap/ # tarball targets # --------------------------------------------------------------------------- tar%pkg: FORCE - $(MAKE) KBUILD_SRC= - $(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ + $(MAKE) -f $(srctree)/Makefile + +$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ clean-dirs += $(objtree)/tar-install/ @@ -137,6 +143,7 @@ help: FORCE @echo ' binrpm-pkg - Build only the binary kernel RPM package' @echo ' deb-pkg - Build both source and binary deb kernel packages' @echo ' bindeb-pkg - Build only the binary kernel deb package' + @echo ' snap-pkg - Build only the binary kernel snap package (will connect to external hosts)' @echo ' tar-pkg - Build the kernel as an uncompressed tarball' @echo ' targz-pkg - Build the kernel as a gzip compressed tarball' @echo ' tarbz2-pkg - Build the kernel as a bzip2 compressed tarball' diff --git a/scripts/package/builddeb b/scripts/package/builddeb index e15159d0a884e..96321fe491322 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -12,6 +12,18 @@ set -e +is_enabled() { + grep -q "^$1=y" include/config/auto.conf +} + +if_enabled_echo() { + if is_enabled "$1"; then + echo -n "$2" + elif [ $# -ge 3 ]; then + echo -n "$3" + fi +} + create_package() { local pname="$1" pdir="$2" @@ -30,67 +42,11 @@ create_package() { chmod -R a+rX "$pdir" # Create the package - dpkg-gencontrol $forcearch -Vkernel:debarch="${debarch}" -p$pname -P"$pdir" + dpkg-gencontrol -p$pname -P"$pdir" dpkg --build "$pdir" .. } -set_debarch() { - # Attempt to find the correct Debian architecture - case "$UTS_MACHINE" in - i386|ia64|alpha) - debarch="$UTS_MACHINE" ;; - x86_64) - debarch=amd64 ;; - sparc*) - debarch=sparc ;; - s390*) - debarch=s390$(grep -q CONFIG_64BIT=y $KCONFIG_CONFIG && echo x || true) ;; - ppc*) - debarch=$(grep -q CPU_LITTLE_ENDIAN=y $KCONFIG_CONFIG && echo ppc64el || echo powerpc) ;; - parisc*) - debarch=hppa ;; - mips*) - debarch=mips$(grep -q CPU_LITTLE_ENDIAN=y $KCONFIG_CONFIG && echo el || true) ;; - aarch64|arm64) - debarch=arm64 ;; - arm*) - if grep -q CONFIG_AEABI=y $KCONFIG_CONFIG; then - if grep -q CONFIG_VFP=y $KCONFIG_CONFIG; then - debarch=armhf - else - debarch=armel - fi - else - debarch=arm - fi - ;; - *) - debarch=$(dpkg --print-architecture) - echo "" >&2 - echo "** ** ** WARNING ** ** **" >&2 - echo "" >&2 - echo "Your architecture doesn't have its equivalent" >&2 - echo "Debian userspace architecture defined!" >&2 - echo "Falling back to using your current userspace instead!" >&2 - echo "Please add support for $UTS_MACHINE to ${0} ..." >&2 - echo "" >&2 - esac - if [ -n "$KBUILD_DEBARCH" ] ; then - debarch="$KBUILD_DEBARCH" - fi - forcearch="-DArchitecture=$debarch" - -} - -# Some variables and settings used throughout the script version=$KERNELRELEASE -revision=$(cat .version) -if [ -n "$KDEB_PKGVERSION" ]; then - packageversion=$KDEB_PKGVERSION -else - packageversion=$version-$revision -fi -sourcename=$KDEB_SOURCENAME tmpdir="$objtree/debian/tmp" kernel_headers_dir="$objtree/debian/hdrtmp" libc_headers_dir="$objtree/debian/headertmp" @@ -99,9 +55,6 @@ packagename=linux-image-$version kernel_headers_packagename=linux-headers-$version libc_headers_packagename=linux-libc-dev dbg_packagename=$packagename-dbg -debarch= -forcearch= -set_debarch if [ "$ARCH" = "um" ] ; then packagename=user-mode-linux-$version @@ -121,7 +74,7 @@ parisc|mips|powerpc) installed_image_path="boot/vmlinuz-$version" esac -BUILD_DEBUG="$(grep -s '^CONFIG_DEBUG_INFO=y' $KCONFIG_CONFIG || true)" +BUILD_DEBUG=$(if_enabled_echo CONFIG_DEBUG_INFO Yes) # Setup the directory structure rm -rf "$tmpdir" "$kernel_headers_dir" "$libc_headers_dir" "$dbg_dir" $objtree/debian/files @@ -140,17 +93,18 @@ else cp System.map "$tmpdir/boot/System.map-$version" cp $KCONFIG_CONFIG "$tmpdir/boot/config-$version" fi -cp "$($MAKE -s image_name)" "$tmpdir/$installed_image_path" +cp "$($MAKE -s -f $srctree/Makefile image_name)" "$tmpdir/$installed_image_path" -if grep -q "^CONFIG_OF=y" $KCONFIG_CONFIG ; then +if is_enabled CONFIG_OF_EARLY_FLATTREE; then # Only some architectures with OF support have this target - if grep -q dtbs_install "${srctree}/arch/$SRCARCH/Makefile"; then - $MAKE KBUILD_SRC= INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install + if [ -d "${srctree}/arch/$SRCARCH/boot/dts" ]; then + $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install + $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/boot/dtbs/$version" dtbs_install fi fi -if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then - INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_install +if is_enabled CONFIG_MODULES; then + INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_install rm -f "$tmpdir/lib/modules/$version/build" rm -f "$tmpdir/lib/modules/$version/source" if [ "$ARCH" = "um" ] ; then @@ -170,16 +124,15 @@ if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then done # resign stripped modules - MODULE_SIG_ALL="$(grep -s '^CONFIG_MODULE_SIG_ALL=y' $KCONFIG_CONFIG || true)" - if [ -n "$MODULE_SIG_ALL" ]; then - INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_sign + if is_enabled CONFIG_MODULE_SIG_ALL; then + INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_sign fi fi fi if [ "$ARCH" != "um" ]; then - $MAKE headers_check KBUILD_SRC= - $MAKE headers_install KBUILD_SRC= INSTALL_HDR_PATH="$libc_headers_dir/usr" + $MAKE -f $srctree/Makefile headers_check + $MAKE -f $srctree/Makefile headers_install INSTALL_HDR_PATH="$libc_headers_dir/usr" fi # Install the maintainer scripts @@ -188,11 +141,6 @@ fi # make-kpkg sets $INITRD to indicate whether an initramfs is wanted, and # so do we; recent versions of dracut and initramfs-tools will obey this. debhookdir=${KDEB_HOOKDIR:-/etc/kernel} -if grep -q '^CONFIG_BLK_DEV_INITRD=y' $KCONFIG_CONFIG; then - want_initrd=Yes -else - want_initrd=No -fi for script in postinst postrm preinst prerm ; do mkdir -p "$tmpdir$debhookdir/$script.d" cat < "$tmpdir/DEBIAN/$script" @@ -204,7 +152,7 @@ set -e export DEB_MAINT_PARAMS="\$*" # Tell initramfs builder whether it's wanted -export INITRD=$want_initrd +export INITRD=$(if_enabled_echo CONFIG_BLK_DEV_INITRD Yes No) test -d $debhookdir/$script.d && run-parts --arg="$version" --arg="/$installed_image_path" $debhookdir/$script.d exit 0 @@ -212,115 +160,16 @@ EOF chmod 755 "$tmpdir/DEBIAN/$script" done -# Try to determine maintainer and email values -if [ -n "$DEBEMAIL" ]; then - email=$DEBEMAIL -elif [ -n "$EMAIL" ]; then - email=$EMAIL -else - email=$(id -nu)@$(hostname -f 2>/dev/null || hostname) -fi -if [ -n "$DEBFULLNAME" ]; then - name=$DEBFULLNAME -elif [ -n "$NAME" ]; then - name=$NAME -else - name="Anonymous" -fi -maintainer="$name <$email>" - -# Try to determine distribution -if [ -n "$KDEB_CHANGELOG_DIST" ]; then - distribution=$KDEB_CHANGELOG_DIST -# In some cases lsb_release returns the codename as n/a, which breaks dpkg-parsechangelog -elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ] && [ "$distribution" != "n/a" ]; then - : # nothing to do in this case -else - distribution="unstable" - echo >&2 "Using default distribution of 'unstable' in the changelog" - echo >&2 "Install lsb-release or set \$KDEB_CHANGELOG_DIST explicitly" -fi - -# Generate a simple changelog template -cat < debian/changelog -$sourcename ($packageversion) $distribution; urgency=low - - * Custom built Linux kernel. - - -- $maintainer $(date -R) -EOF - -# Generate copyright file -cat < debian/copyright -This is a packacked upstream version of the Linux kernel. - -The sources may be found at most Linux archive sites, including: -https://www.kernel.org/pub/linux/kernel - -Copyright: 1991 - 2017 Linus Torvalds and others. - -The git repository for mainline kernel development is at: -git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 dated June, 1991. - -On Debian GNU/Linux systems, the complete text of the GNU General Public -License version 2 can be found in \`/usr/share/common-licenses/GPL-2'. -EOF - - -build_depends="bc, kmod, cpio " - -# Generate a control file -cat < debian/control -Source: $sourcename -Section: kernel -Priority: optional -Maintainer: $maintainer -Build-Depends: $build_depends -Homepage: http://www.kernel.org/ -EOF - -if [ "$ARCH" = "um" ]; then - cat <> debian/control - -Package: $packagename -Architecture: any -Description: User Mode Linux kernel, version $version - User-mode Linux is a port of the Linux kernel to its own system call - interface. It provides a kind of virtual machine, which runs Linux - as a user process under another Linux kernel. This is useful for - kernel development, sandboxes, jails, experimentation, and - many other things. - . - This package contains the Linux kernel, modules and corresponding other - files, version: $version. -EOF - -else - cat <> debian/control - -Package: $packagename -Architecture: any -Description: Linux kernel, version $version - This package contains the Linux kernel, modules and corresponding other - files, version: $version. -EOF - -fi - # Build kernel header package (cd $srctree; find . -name Makefile\* -o -name Kconfig\* -o -name \*.pl) > "$objtree/debian/hdrsrcfiles" (cd $srctree; find arch/*/include include scripts -type f -o -type l) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles" -if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then +if is_enabled CONFIG_STACK_VALIDATION; then (cd $objtree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrobjfiles" fi (cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles" -if grep -q '^CONFIG_GCC_PLUGINS=y' $KCONFIG_CONFIG ; then +if is_enabled CONFIG_GCC_PLUGINS; then (cd $objtree; find scripts/gcc-plugins -name \*.so -o -name gcc-common.h) >> "$objtree/debian/hdrobjfiles" fi destdir=$kernel_headers_dir/usr/src/linux-headers-$version @@ -331,27 +180,6 @@ mkdir -p "$destdir" ln -sf "/usr/src/linux-headers-$version" "$kernel_headers_dir/lib/modules/$version/build" rm -f "$objtree/debian/hdrsrcfiles" "$objtree/debian/hdrobjfiles" -cat <> debian/control - -Package: $kernel_headers_packagename -Architecture: any -Description: Linux kernel headers for $KERNELRELEASE on \${kernel:debarch} - This package provides kernel header files for $KERNELRELEASE on \${kernel:debarch} - . - This is useful for people who need to build external modules -EOF - -cat <> debian/control - -Package: $libc_headers_packagename -Section: devel -Provides: linux-kernel-headers -Architecture: any -Description: Linux support headers for userspace development - This package provides userspaces headers from the Linux kernel. These headers - are used by the installed headers for GNU glibc and other system libraries. -EOF - if [ "$ARCH" != "um" ]; then create_package "$kernel_headers_packagename" "$kernel_headers_dir" create_package "$libc_headers_packagename" "$libc_headers_dir" @@ -370,47 +198,7 @@ if [ -n "$BUILD_DEBUG" ] ; then ln -s ../lib/modules/$version/vmlinux $dbg_dir/usr/lib/debug/boot/vmlinux-$version # kdump-tools ln -s lib/modules/$version/vmlinux $dbg_dir/usr/lib/debug/vmlinux-$version - - cat <> debian/control - -Package: $dbg_packagename -Section: debug -Architecture: any -Description: Linux kernel debugging symbols for $version - This package will come in handy if you need to debug the kernel. It provides - all the necessary debug symbols for the kernel and its modules. -EOF - create_package "$dbg_packagename" "$dbg_dir" fi -if [ "x$1" = "xdeb-pkg" ] -then - cat < debian/rules -#!/usr/bin/make -f - -build: - \$(MAKE) - -binary-arch: - \$(MAKE) KDEB_SOURCENAME=${sourcename} KDEB_PKGVERSION=${packageversion} bindeb-pkg - -clean: - rm -rf debian/*tmp debian/files - mv debian/ debian.backup # debian/ might be cleaned away - \$(MAKE) clean - mv debian.backup debian - -binary: binary-arch -EOF - mv ${sourcename}.tar.gz ../${sourcename}_${version}.orig.tar.gz - tar caf ../${sourcename}_${packageversion}.debian.tar.gz debian/{copyright,rules,changelog,control} - dpkg-source -cdebian/control -ldebian/changelog --format="3.0 (custom)" --target-format="3.0 (quilt)" \ - -b / ../${sourcename}_${version}.orig.tar.gz ../${sourcename}_${packageversion}.debian.tar.gz - mv ${sourcename}_${packageversion}*dsc .. - dpkg-genchanges > ../${sourcename}_${packageversion}_${debarch}.changes -else - dpkg-genchanges -b > ../${sourcename}_${packageversion}_${debarch}.changes -fi - exit 0 diff --git a/scripts/package/buildtar b/scripts/package/buildtar index e8cc72a51b32e..2f66c81e4021b 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -56,8 +56,8 @@ dirs=boot # # Try to install modules # -if grep -q '^CONFIG_MODULES=y' "${KCONFIG_CONFIG}"; then - make ARCH="${ARCH}" O="${objtree}" KBUILD_SRC= INSTALL_MOD_PATH="${tmpdir}" modules_install +if grep -q '^CONFIG_MODULES=y' include/config/auto.conf; then + make ARCH="${ARCH}" -f ${srctree}/Makefile INSTALL_MOD_PATH="${tmpdir}" modules_install dirs="$dirs lib" fi @@ -84,10 +84,6 @@ case "${ARCH}" in [ -f "${KBUILD_IMAGE}" ] && cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" [ -f "${objtree}/lifimage" ] && cp -v -- "${objtree}/lifimage" "${tmpdir}/boot/lifimage-${KERNELRELEASE}" ;; - vax) - [ -f "${objtree}/vmlinux.SYS" ] && cp -v -- "${objtree}/vmlinux.SYS" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}.SYS" - [ -f "${objtree}/vmlinux.dsk" ] && cp -v -- "${objtree}/vmlinux.dsk" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}.dsk" - ;; mips) if [ -f "${objtree}/arch/mips/boot/compressed/vmlinux.bin" ]; then cp -v -- "${objtree}/arch/mips/boot/compressed/vmlinux.bin" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" @@ -109,6 +105,14 @@ case "${ARCH}" in cp -v -- "${objtree}/vmlinux" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" fi ;; + arm64) + for i in Image.bz2 Image.gz Image.lz4 Image.lzma Image.lzo ; do + if [ -f "${objtree}/arch/arm64/boot/${i}" ] ; then + cp -v -- "${objtree}/arch/arm64/boot/${i}" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" + break + fi + done + ;; *) [ -f "${KBUILD_IMAGE}" ] && cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-kbuild-${KERNELRELEASE}" echo "" >&2 diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian new file mode 100755 index 0000000000000..8351584cb24e6 --- /dev/null +++ b/scripts/package/mkdebian @@ -0,0 +1,230 @@ +#!/bin/sh +# +# Copyright 2003 Wichert Akkerman +# +# Simple script to generate a debian/ directory for a Linux kernel. + +set -e + +is_enabled() { + grep -q "^$1=y" include/config/auto.conf +} + +if_enabled_echo() { + if is_enabled "$1"; then + echo -n "$2" + elif [ $# -ge 3 ]; then + echo -n "$3" + fi +} + +set_debarch() { + if [ -n "$KBUILD_DEBARCH" ] ; then + debarch="$KBUILD_DEBARCH" + return + fi + + # Attempt to find the correct Debian architecture + case "$UTS_MACHINE" in + i386|ia64|alpha|m68k|riscv*) + debarch="$UTS_MACHINE" ;; + x86_64) + debarch=amd64 ;; + sparc*) + debarch=sparc$(if_enabled_echo CONFIG_64BIT 64) ;; + s390*) + debarch=s390x ;; + ppc*) + if is_enabled CONFIG_64BIT; then + debarch=ppc64$(if_enabled_echo CONFIG_CPU_LITTLE_ENDIAN el) + else + debarch=powerpc$(if_enabled_echo CONFIG_SPE spe) + fi + ;; + parisc*) + debarch=hppa ;; + mips*) + if is_enabled CONFIG_CPU_LITTLE_ENDIAN; then + debarch=mips$(if_enabled_echo CONFIG_64BIT 64)$(if_enabled_echo CONFIG_CPU_MIPSR6 r6)el + elif is_enabled CONFIG_CPU_MIPSR6; then + debarch=mips$(if_enabled_echo CONFIG_64BIT 64)r6 + else + debarch=mips + fi + ;; + aarch64|arm64) + debarch=arm64 ;; + arm*) + if is_enabled CONFIG_AEABI; then + debarch=arm$(if_enabled_echo CONFIG_VFP hf el) + else + debarch=arm + fi + ;; + openrisc) + debarch=or1k ;; + sh) + if is_enabled CONFIG_CPU_SH3; then + debarch=sh3$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb) + elif is_enabled CONFIG_CPU_SH4; then + debarch=sh4$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb) + fi + ;; + esac + if [ -z "$debarch" ]; then + debarch=$(dpkg-architecture -qDEB_HOST_ARCH) + echo "" >&2 + echo "** ** ** WARNING ** ** **" >&2 + echo "" >&2 + echo "Your architecture doesn't have its equivalent" >&2 + echo "Debian userspace architecture defined!" >&2 + echo "Falling back to the current host architecture ($debarch)." >&2 + echo "Please add support for $UTS_MACHINE to ${0} ..." >&2 + echo "" >&2 + fi +} + +# Some variables and settings used throughout the script +version=$KERNELRELEASE +if [ -n "$KDEB_PKGVERSION" ]; then + packageversion=$KDEB_PKGVERSION + revision=${packageversion##*-} +else + revision=$(cat .version 2>/dev/null||echo 1) + packageversion=$version-$revision +fi +sourcename=$KDEB_SOURCENAME +packagename=linux-image-$version +kernel_headers_packagename=linux-headers-$version +dbg_packagename=$packagename-dbg +debarch= +set_debarch + +if [ "$ARCH" = "um" ] ; then + packagename=user-mode-linux-$version +fi + +email=${DEBEMAIL-$EMAIL} + +# use email string directly if it contains +if echo $email | grep -q '<.*>'; then + maintainer=$email +else + # or construct the maintainer string + user=${KBUILD_BUILD_USER-$(id -nu)} + name=${DEBFULLNAME-$user} + if [ -z "$email" ]; then + buildhost=${KBUILD_BUILD_HOST-$(hostname -f 2>/dev/null || hostname)} + email="$user@$buildhost" + fi + maintainer="$name <$email>" +fi + +# Try to determine distribution +if [ -n "$KDEB_CHANGELOG_DIST" ]; then + distribution=$KDEB_CHANGELOG_DIST +# In some cases lsb_release returns the codename as n/a, which breaks dpkg-parsechangelog +elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ] && [ "$distribution" != "n/a" ]; then + : # nothing to do in this case +else + distribution="unstable" + echo >&2 "Using default distribution of 'unstable' in the changelog" + echo >&2 "Install lsb-release or set \$KDEB_CHANGELOG_DIST explicitly" +fi + +mkdir -p debian/source/ +echo "1.0" > debian/source/format + +echo $debarch > debian/arch +extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev)" +extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)" + +# Generate a simple changelog template +cat < debian/changelog +$sourcename ($packageversion) $distribution; urgency=low + + * Custom built Linux kernel. + + -- $maintainer $(date -R) +EOF + +# Generate copyright file +cat < debian/copyright +This is a packacked upstream version of the Linux kernel. + +The sources may be found at most Linux archive sites, including: +https://www.kernel.org/pub/linux/kernel + +Copyright: 1991 - 2018 Linus Torvalds and others. + +The git repository for mainline kernel development is at: +git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 dated June, 1991. + +On Debian GNU/Linux systems, the complete text of the GNU General Public +License version 2 can be found in \`/usr/share/common-licenses/GPL-2'. +EOF + +# Generate a control file +cat < debian/control +Source: $sourcename +Section: kernel +Priority: optional +Maintainer: $maintainer +Build-Depends: bc, kmod, cpio, bison, flex | flex:native $extra_build_depends +Homepage: http://www.kernel.org/ + +Package: $packagename +Architecture: $debarch +Description: Linux kernel, version $version + This package contains the Linux kernel, modules and corresponding other + files, version: $version. + +Package: $kernel_headers_packagename +Architecture: $debarch +Description: Linux kernel headers for $version on $debarch + This package provides kernel header files for $version on $debarch + . + This is useful for people who need to build external modules + +Package: linux-libc-dev +Section: devel +Provides: linux-kernel-headers +Architecture: $debarch +Description: Linux support headers for userspace development + This package provides userspaces headers from the Linux kernel. These headers + are used by the installed headers for GNU glibc and other system libraries. + +Package: $dbg_packagename +Section: debug +Architecture: $debarch +Description: Linux kernel debugging symbols for $version + This package will come in handy if you need to debug the kernel. It provides + all the necessary debug symbols for the kernel and its modules. +EOF + +cat < debian/rules +#!$(command -v $MAKE) -f + +srctree ?= . + +build: + \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ + KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile + +binary-arch: + \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ + KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile intdeb-pkg + +clean: + rm -rf debian/*tmp debian/files + \$(MAKE) clean + +binary: binary-arch +EOF +chmod +x debian/rules + +exit 0 diff --git a/scripts/package/mkspec b/scripts/package/mkspec index f47f17aae1351..009147d4718ee 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -10,156 +10,138 @@ # # how we were called determines which rpms we build and how we build them -if [ "$1" = "prebuilt" ]; then - PREBUILT=true +if [ "$1" = prebuilt ]; then + S=DEL + MAKE="$MAKE -f $srctree/Makefile" else - PREBUILT=false + S= fi -# starting to output the spec -if [ "`grep CONFIG_DRM=y .config | cut -f2 -d\=`" = "y" ]; then - PROVIDES=kernel-drm -fi - -PROVIDES="$PROVIDES kernel-$KERNELRELEASE" -__KERNELRELEASE=`echo $KERNELRELEASE | sed -e "s/-/_/g"` - -echo "Name: kernel" -echo "Summary: The Linux Kernel" -echo "Version: $__KERNELRELEASE" -echo "Release: $(cat .version 2>/dev/null || echo 1)" -echo "License: GPL" -echo "Group: System Environment/Kernel" -echo "Vendor: The Linux Community" -echo "URL: http://www.kernel.org" - -if ! $PREBUILT; then -echo "Source: kernel-$__KERNELRELEASE.tar.gz" -fi - -echo "BuildRoot: %{_tmppath}/%{name}-%{PACKAGE_VERSION}-root" -echo "Provides: $PROVIDES" -echo "%define __spec_install_post /usr/lib/rpm/brp-compress || :" -echo "%define debug_package %{nil}" -echo "" -echo "%description" -echo "The Linux Kernel, the operating system core itself" -echo "" -echo "%package headers" -echo "Summary: Header files for the Linux kernel for use by glibc" -echo "Group: Development/System" -echo "Obsoletes: kernel-headers" -echo "Provides: kernel-headers = %{version}" -echo "%description headers" -echo "Kernel-headers includes the C header files that specify the interface" -echo "between the Linux kernel and userspace libraries and programs. The" -echo "header files define structures and constants that are needed for" -echo "building most standard programs and are also needed for rebuilding the" -echo "glibc package." -echo "" -echo "%package devel" -echo "Summary: Development package for building kernel modules to match the $__KERNELRELEASE kernel" -echo "Group: System Environment/Kernel" -echo "AutoReqProv: no" -echo "%description -n kernel-devel" -echo "This package provides kernel headers and makefiles sufficient to build modules" -echo "against the $__KERNELRELEASE kernel package." -echo "" - -if ! $PREBUILT; then -echo "%prep" -echo "%setup -q" -echo "" +if grep -q CONFIG_MODULES=y .config; then + M= +else + M=DEL fi -echo "%build" - -if ! $PREBUILT; then -echo "make clean && make %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release}" -echo "" +if grep -q CONFIG_DRM=y .config; then + PROVIDES=kernel-drm fi -echo "%install" -echo 'KBUILD_IMAGE=$(make image_name)' -echo "%ifarch ia64" -echo 'mkdir -p $RPM_BUILD_ROOT/boot/efi $RPM_BUILD_ROOT/lib/modules' -echo "%else" -echo 'mkdir -p $RPM_BUILD_ROOT/boot $RPM_BUILD_ROOT/lib/modules' -echo "%endif" - -echo 'INSTALL_MOD_PATH=$RPM_BUILD_ROOT make %{?_smp_mflags} KBUILD_SRC= modules_install' -echo "%ifarch ia64" -echo 'cp $KBUILD_IMAGE $RPM_BUILD_ROOT'"/boot/efi/vmlinuz-$KERNELRELEASE" -echo 'ln -s '"efi/vmlinuz-$KERNELRELEASE" '$RPM_BUILD_ROOT'"/boot/" -echo "%else" -echo "%ifarch ppc64" -echo "cp vmlinux arch/powerpc/boot" -echo "cp arch/powerpc/boot/"'$KBUILD_IMAGE $RPM_BUILD_ROOT'"/boot/vmlinuz-$KERNELRELEASE" -echo "%else" -echo 'cp $KBUILD_IMAGE $RPM_BUILD_ROOT'"/boot/vmlinuz-$KERNELRELEASE" -echo "%endif" -echo "%endif" - -echo 'make %{?_smp_mflags} INSTALL_HDR_PATH=$RPM_BUILD_ROOT/usr KBUILD_SRC= headers_install' -echo 'cp System.map $RPM_BUILD_ROOT'"/boot/System.map-$KERNELRELEASE" - -echo 'cp .config $RPM_BUILD_ROOT'"/boot/config-$KERNELRELEASE" - -echo "%ifnarch ppc64" -echo 'bzip2 -9 --keep vmlinux' -echo 'mv vmlinux.bz2 $RPM_BUILD_ROOT'"/boot/vmlinux-$KERNELRELEASE.bz2" -echo "%endif" - -if ! $PREBUILT; then -echo 'rm -f $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE/build" -echo 'rm -f $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE/source" -echo "mkdir -p "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE" -echo "EXCLUDES=\"$RCS_TAR_IGNORE --exclude .tmp_versions --exclude=*vmlinux* --exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation --exclude .config.old --exclude .missing-syscalls.d\"" -echo "tar "'$EXCLUDES'" -cf- . | (cd "'$RPM_BUILD_ROOT'"/usr/src/kernels/$KERNELRELEASE;tar xvf -)" -echo 'cd $RPM_BUILD_ROOT'"/lib/modules/$KERNELRELEASE" -echo "ln -sf /usr/src/kernels/$KERNELRELEASE build" -echo "ln -sf /usr/src/kernels/$KERNELRELEASE source" -fi +PROVIDES="$PROVIDES kernel-$KERNELRELEASE" +__KERNELRELEASE=$(echo $KERNELRELEASE | sed -e "s/-/_/g") +EXCLUDES="$RCS_TAR_IGNORE --exclude=.tmp_versions --exclude=*vmlinux* \ +--exclude=*.o --exclude=*.ko --exclude=*.cmd --exclude=Documentation \ +--exclude=.config.old --exclude=.missing-syscalls.d" -echo "" -echo "%clean" -echo 'rm -rf $RPM_BUILD_ROOT' -echo "" -echo "%post" -echo "if [ -x /sbin/installkernel -a -r /boot/vmlinuz-$KERNELRELEASE -a -r /boot/System.map-$KERNELRELEASE ]; then" -echo "cp /boot/vmlinuz-$KERNELRELEASE /boot/.vmlinuz-$KERNELRELEASE-rpm" -echo "cp /boot/System.map-$KERNELRELEASE /boot/.System.map-$KERNELRELEASE-rpm" -echo "rm -f /boot/vmlinuz-$KERNELRELEASE /boot/System.map-$KERNELRELEASE" -echo "/sbin/installkernel $KERNELRELEASE /boot/.vmlinuz-$KERNELRELEASE-rpm /boot/.System.map-$KERNELRELEASE-rpm" -echo "rm -f /boot/.vmlinuz-$KERNELRELEASE-rpm /boot/.System.map-$KERNELRELEASE-rpm" -echo "fi" -echo "" -echo "%preun" -echo "if [ -x /sbin/new-kernel-pkg ]; then" -echo "new-kernel-pkg --remove $KERNELRELEASE --rminitrd --initrdfile=/boot/initramfs-$KERNELRELEASE.img" -echo "fi" -echo "" -echo "%postun" -echo "if [ -x /sbin/update-bootloader ]; then" -echo "/sbin/update-bootloader --remove $KERNELRELEASE" -echo "fi" -echo "" -echo "%files" -echo '%defattr (-, root, root)' -echo "/lib/modules/$KERNELRELEASE" -echo "%exclude /lib/modules/$KERNELRELEASE/build" -echo "%exclude /lib/modules/$KERNELRELEASE/source" -echo "/boot/*" -echo "" -echo "%files headers" -echo '%defattr (-, root, root)' -echo "/usr/include" -echo "" -if ! $PREBUILT; then -echo "%files devel" -echo '%defattr (-, root, root)' -echo "/usr/src/kernels/$KERNELRELEASE" -echo "/lib/modules/$KERNELRELEASE/build" -echo "/lib/modules/$KERNELRELEASE/source" -echo "" -fi +# We can label the here-doc lines for conditional output to the spec file +# +# Labels: +# $S: this line is enabled only when building source package +# $M: this line is enabled only when CONFIG_MODULES is enabled +sed -e '/^DEL/d' -e 's/^\t*//' </dev/null || echo 1) + License: GPL + Group: System Environment/Kernel + Vendor: The Linux Community + URL: http://www.kernel.org +$S Source: kernel-$__KERNELRELEASE.tar.gz + Provides: $PROVIDES + %define __spec_install_post /usr/lib/rpm/brp-compress || : + %define debug_package %{nil} + + %description + The Linux Kernel, the operating system core itself + + %package headers + Summary: Header files for the Linux kernel for use by glibc + Group: Development/System + Obsoletes: kernel-headers + Provides: kernel-headers = %{version} + %description headers + Kernel-headers includes the C header files that specify the interface + between the Linux kernel and userspace libraries and programs. The + header files define structures and constants that are needed for + building most standard programs and are also needed for rebuilding the + glibc package. + +$S$M %package devel +$S$M Summary: Development package for building kernel modules to match the $__KERNELRELEASE kernel +$S$M Group: System Environment/Kernel +$S$M AutoReqProv: no +$S$M %description -n kernel-devel +$S$M This package provides kernel headers and makefiles sufficient to build modules +$S$M against the $__KERNELRELEASE kernel package. +$S$M +$S %prep +$S %setup -q +$S +$S %build +$S $MAKE %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release} +$S + %install + mkdir -p %{buildroot}/boot + %ifarch ia64 + mkdir -p %{buildroot}/boot/efi + cp \$($MAKE image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE + ln -s efi/vmlinuz-$KERNELRELEASE %{buildroot}/boot/ + %else + cp \$($MAKE image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE + %endif +$M $MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install + $MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install + cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE + cp .config %{buildroot}/boot/config-$KERNELRELEASE + bzip2 -9 --keep vmlinux + mv vmlinux.bz2 %{buildroot}/boot/vmlinux-$KERNELRELEASE.bz2 +$S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/build +$S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/source +$S$M mkdir -p %{buildroot}/usr/src/kernels/$KERNELRELEASE +$S$M tar cf - $EXCLUDES . | tar xf - -C %{buildroot}/usr/src/kernels/$KERNELRELEASE +$S$M cd %{buildroot}/lib/modules/$KERNELRELEASE +$S$M ln -sf /usr/src/kernels/$KERNELRELEASE build +$S$M ln -sf /usr/src/kernels/$KERNELRELEASE source + + %clean + rm -rf %{buildroot} + + %post + if [ -x /sbin/installkernel -a -r /boot/vmlinuz-$KERNELRELEASE -a -r /boot/System.map-$KERNELRELEASE ]; then + cp /boot/vmlinuz-$KERNELRELEASE /boot/.vmlinuz-$KERNELRELEASE-rpm + cp /boot/System.map-$KERNELRELEASE /boot/.System.map-$KERNELRELEASE-rpm + rm -f /boot/vmlinuz-$KERNELRELEASE /boot/System.map-$KERNELRELEASE + /sbin/installkernel $KERNELRELEASE /boot/.vmlinuz-$KERNELRELEASE-rpm /boot/.System.map-$KERNELRELEASE-rpm + rm -f /boot/.vmlinuz-$KERNELRELEASE-rpm /boot/.System.map-$KERNELRELEASE-rpm + fi + + %preun + if [ -x /sbin/new-kernel-pkg ]; then + new-kernel-pkg --remove $KERNELRELEASE --rminitrd --initrdfile=/boot/initramfs-$KERNELRELEASE.img + elif [ -x /usr/bin/kernel-install ]; then + kernel-install remove $KERNELRELEASE + fi + + %postun + if [ -x /sbin/update-bootloader ]; then + /sbin/update-bootloader --remove $KERNELRELEASE + fi + + %files + %defattr (-, root, root) +$M /lib/modules/$KERNELRELEASE +$M %exclude /lib/modules/$KERNELRELEASE/build +$M %exclude /lib/modules/$KERNELRELEASE/source + /boot/* + + %files headers + %defattr (-, root, root) + /usr/include +$S$M +$S$M %files devel +$S$M %defattr (-, root, root) +$S$M /usr/src/kernels/$KERNELRELEASE +$S$M /lib/modules/$KERNELRELEASE/build +$S$M /lib/modules/$KERNELRELEASE/source +EOF diff --git a/scripts/package/snapcraft.template b/scripts/package/snapcraft.template new file mode 100644 index 0000000000000..626d278e4a5a7 --- /dev/null +++ b/scripts/package/snapcraft.template @@ -0,0 +1,14 @@ +name: kernel +version: KERNELRELEASE +summary: Linux kernel +description: The upstream Linux kernel +grade: stable +confinement: strict +type: kernel + +parts: + kernel: + plugin: kernel + source: SRCTREE + source-type: tar + kernel-with-firmware: false diff --git a/security/device_cgroup.c b/security/device_cgroup.c index 5ef7e52405632..5b8f88987b31d 100644 --- a/security/device_cgroup.c +++ b/security/device_cgroup.c @@ -850,6 +850,7 @@ int __devcgroup_inode_permission(struct inode *inode, int mask) return __devcgroup_check_permission(type, imajor(inode), iminor(inode), access); } +EXPORT_SYMBOL_GPL(__devcgroup_inode_permission); int devcgroup_inode_mknod(int mode, dev_t dev) { diff --git a/security/security.c b/security/security.c index 4fbe4e495c021..640687ed12bda 100644 --- a/security/security.c +++ b/security/security.c @@ -532,6 +532,7 @@ int security_path_rmdir(const struct path *dir, struct dentry *dentry) return 0; return call_int_hook(path_rmdir, 0, dir, dentry); } +EXPORT_SYMBOL_GPL(security_path_rmdir); int security_path_unlink(const struct path *dir, struct dentry *dentry) { @@ -548,6 +549,7 @@ int security_path_symlink(const struct path *dir, struct dentry *dentry, return 0; return call_int_hook(path_symlink, 0, dir, dentry, old_name); } +EXPORT_SYMBOL_GPL(security_path_symlink); int security_path_link(struct dentry *old_dentry, const struct path *new_dir, struct dentry *new_dentry) @@ -556,6 +558,7 @@ int security_path_link(struct dentry *old_dentry, const struct path *new_dir, return 0; return call_int_hook(path_link, 0, old_dentry, new_dir, new_dentry); } +EXPORT_SYMBOL_GPL(security_path_link); int security_path_rename(const struct path *old_dir, struct dentry *old_dentry, const struct path *new_dir, struct dentry *new_dentry, @@ -583,6 +586,7 @@ int security_path_truncate(const struct path *path) return 0; return call_int_hook(path_truncate, 0, path); } +EXPORT_SYMBOL_GPL(security_path_truncate); int security_path_chmod(const struct path *path, umode_t mode) { @@ -590,6 +594,7 @@ int security_path_chmod(const struct path *path, umode_t mode) return 0; return call_int_hook(path_chmod, 0, path, mode); } +EXPORT_SYMBOL_GPL(security_path_chmod); int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) { @@ -597,6 +602,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) return 0; return call_int_hook(path_chown, 0, path, uid, gid); } +EXPORT_SYMBOL_GPL(security_path_chown); int security_path_chroot(const struct path *path) { @@ -697,6 +703,7 @@ int security_inode_permission(struct inode *inode, int mask) return 0; return call_int_hook(inode_permission, 0, inode, mask); } +EXPORT_SYMBOL_GPL(security_inode_permission); int security_inode_setattr(struct dentry *dentry, struct iattr *attr) { @@ -868,6 +875,7 @@ int security_file_permission(struct file *file, int mask) return fsnotify_perm(file, mask); } +EXPORT_SYMBOL_GPL(security_file_permission); int security_file_alloc(struct file *file) { @@ -927,6 +935,7 @@ int security_mmap_file(struct file *file, unsigned long prot, return ret; return ima_file_mmap(file, prot); } +EXPORT_SYMBOL_GPL(security_mmap_file); int security_mmap_addr(unsigned long addr) { diff --git a/sound/drivers/Kconfig b/sound/drivers/Kconfig index 7144cc36e8ae1..f1876924a6427 100644 --- a/sound/drivers/Kconfig +++ b/sound/drivers/Kconfig @@ -184,6 +184,15 @@ config SND_PORTMAN2X4 To compile this driver as a module, choose M here: the module will be called snd-portman2x4. +config SND_PWMSP + tristate "PWM Speaker driver (PCM)" + depends on HIGH_RES_TIMERS + select SND_PCM + select PWM + help + To compile this driver as a module, choose M here: the module + will be called snd-pwm. + config SND_ML403_AC97CR tristate "Xilinx ML403 AC97 Controller Reference" depends on XILINX_VIRTEX diff --git a/sound/drivers/Makefile b/sound/drivers/Makefile index 615558a281c8f..a3127cfdc3f1b 100644 --- a/sound/drivers/Makefile +++ b/sound/drivers/Makefile @@ -23,4 +23,4 @@ obj-$(CONFIG_SND_MTS64) += snd-mts64.o obj-$(CONFIG_SND_PORTMAN2X4) += snd-portman2x4.o obj-$(CONFIG_SND_ML403_AC97CR) += snd-ml403-ac97cr.o -obj-$(CONFIG_SND) += opl3/ opl4/ mpu401/ vx/ pcsp/ +obj-$(CONFIG_SND) += opl3/ opl4/ mpu401/ vx/ pcsp/ pwmsp/ diff --git a/sound/drivers/pwmsp/Makefile b/sound/drivers/pwmsp/Makefile new file mode 100644 index 0000000000000..14372e679cf6a --- /dev/null +++ b/sound/drivers/pwmsp/Makefile @@ -0,0 +1,2 @@ +snd-pwmsp-objs := pwmsp.o pwmsp_lib.o +obj-$(CONFIG_SND_PWMSP) += snd-pwmsp.o diff --git a/sound/drivers/pwmsp/pwmsp.c b/sound/drivers/pwmsp/pwmsp.c new file mode 100644 index 0000000000000..a90203b3fc86b --- /dev/null +++ b/sound/drivers/pwmsp/pwmsp.c @@ -0,0 +1,174 @@ +/* + * Based on PC-Speaker driver for Linux + * + * Copyright (C) 1997-2001 David Woodhouse + * Copyright (C) 2001-2008 Stas Sergeev + * + * sndpwm { + * compatible = "snd-pwm"; + * pwms = <&ehrpwm2 1 0 0>; + * status = "okay"; + * }; + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pwmsp.h" + + +static int index = SNDRV_DEFAULT_IDX1; /* Index 0-MAX */ +static char *id = SNDRV_DEFAULT_STR1; /* ID for this card */ +static bool enable = SNDRV_DEFAULT_ENABLE1; /* Enable this card */ + +module_param(index, int, 0444); +MODULE_PARM_DESC(index, "Index value for pwmsp soundcard."); +module_param(id, charp, 0444); +MODULE_PARM_DESC(id, "ID string for pwmsp soundcard."); +module_param(enable, bool, 0444); +MODULE_PARM_DESC(enable, "Enable PC-Speaker sound."); + +struct snd_pwmsp pwmsp_chip; + +static int snd_pwmsp_create(struct snd_card *card) +{ + static struct snd_device_ops ops = { }; + int err; + + pr_info("PCSP: Timer resolution is (%linS)\n", hrtimer_resolution); + + spin_lock_init(&pwmsp_chip.substream_lock); + atomic_set(&pwmsp_chip.timer_active, 0); + pwmsp_chip.playback_ptr = 0; + pwmsp_chip.period_ptr = 0; + pwmsp_chip.enable = 1; + pwmsp_chip.card = card; + + /* Register device */ + err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, &pwmsp_chip, &ops); + if (err < 0) + return err; + + return 0; +} + +static int snd_card_pwmsp_probe(int devnum, struct device *dev) +{ + struct snd_card *card; + int err; + + if (devnum != 0) + return -EINVAL; + + hrtimer_init(&pwmsp_chip.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + pwmsp_chip.timer.function = pwmsp_do_timer; + + err = snd_card_new(dev, index, id, THIS_MODULE, 0, &card); + if (err < 0) { + printk(KERN_ERR "PWM PC-Speaker snd_card_new() call failed.\n"); + return err; + } + + err = snd_pwmsp_create(card); + if (err < 0) { + printk(KERN_ERR "PWM PC-Speaker snd_pwmsp_create() call failed.\n"); + snd_card_free(card); + return err; + } + err = snd_pwmsp_new_pcm(&pwmsp_chip); + if (err < 0) { + printk(KERN_ERR "PWM PC-Speaker snd_pwmsp_new_pcm() call failed.\n"); + snd_card_free(card); + return err; + } + + strcpy(card->driver, "PC-Speaker"); + strcpy(card->shortname, "pwmsp"); + sprintf(card->longname, "PWM PC-Speaker"); + + err = snd_card_register(card); + if (err < 0) { + printk(KERN_ERR "PWM PC-Speaker snd_card_register() call failed.\n"); + snd_card_free(card); + return err; + } + + return 0; +} + +static int alsa_card_pwmsp_init(struct device *dev) +{ + int err; + + err = snd_card_pwmsp_probe(0, dev); + if (err) { + printk(KERN_ERR "PC-Speaker initialization failed.\n"); + return err; + } + +#ifdef CONFIG_DEBUG_PAGEALLOC + /* Well, CONFIG_DEBUG_PAGEALLOC makes the sound horrible. Lets alert */ + printk(KERN_WARNING "PCSP: CONFIG_DEBUG_PAGEALLOC is enabled, " + "which may make the sound noisy.\n"); +#endif + + return 0; +} + +static void alsa_card_pwmsp_exit(struct snd_pwmsp *chip) +{ + snd_card_free(chip->card); +} + +static int pwmsp_probe(struct platform_device *dev) +{ + int err; + + pwmsp_chip.pwm = devm_pwm_get(&dev->dev, NULL); + if (IS_ERR(pwmsp_chip.pwm)) { + printk(KERN_ERR "PWM PC-Speaker devm_pwm_get() call failed.\n"); + dev_err(&dev->dev, "unable to request PWM\n"); + err = PTR_ERR(pwmsp_chip.pwm); + return err; + } + + err = alsa_card_pwmsp_init(&dev->dev); + if (err < 0) { + printk(KERN_ERR "PWM PC-Speaker alsa_card_pwmsp_init() call failed.\n"); + return err; + } + + platform_set_drvdata(dev, &pwmsp_chip); + return 0; +} + +static int pwmsp_remove(struct platform_device *dev) +{ + struct snd_pwmsp *chip = platform_get_drvdata(dev); + alsa_card_pwmsp_exit(chip); + return 0; +} + +static struct of_device_id pwmsp_of_match[] = { + { .compatible = "snd-pwmsp" }, + { }, +}; +static struct platform_driver pwmsp_platform_driver = { + .driver = { + .name = "pwmspkr", + .owner = THIS_MODULE, + .of_match_table = pwmsp_of_match, + }, + .probe = pwmsp_probe, + .remove = pwmsp_remove, +}; +module_platform_driver(pwmsp_platform_driver); + +MODULE_LICENSE("GPL"); diff --git a/sound/drivers/pwmsp/pwmsp.h b/sound/drivers/pwmsp/pwmsp.h new file mode 100644 index 0000000000000..a02a0c931a9a5 --- /dev/null +++ b/sound/drivers/pwmsp/pwmsp.h @@ -0,0 +1,58 @@ +/* + * PC-Speaker driver for Linux + * + * Copyright (C) 1993-1997 Michael Beck + * Copyright (C) 1997-2001 David Woodhouse + * Copyright (C) 2001-2008 Stas Sergeev + */ + +#ifndef __PCSP_H__ +#define __PCSP_H__ + +#include +#include + +#define MAX_DUTY_NS 998 +#define MIN_DUTY_NS 10 +#define DELTA_DUTY (MAX_DUTY_NS - MIN_DUTY_NS) + +#define NSECS_PER_SEC 1000000000UL +#define HZ_TO_NANOSECONDS(x) (NSECS_PER_SEC / (x)) + +#define PWM_FREQ 1000000UL /* Period is 1us */ +#define SAMPLING_FREQ 44100 + +#define PWM_PERIOD_NS HZ_TO_NANOSECONDS(PWM_FREQ) +#define SAMPLING_PERIOD_NS HZ_TO_NANOSECONDS(SAMPLING_FREQ) + +#define PWMSP_MAX_PERIOD_SIZE (64*1024) +#define PWMSP_MAX_PERIODS 512 +#define PWMSP_BUFFER_SIZE (128*1024) + +struct snd_pwmsp { + struct snd_card *card; + struct snd_pcm *pcm; + struct hrtimer timer; + spinlock_t substream_lock; + struct snd_pcm_substream *playback_substream; + unsigned int fmt_size; + unsigned int is_signed; + size_t playback_ptr; + size_t period_ptr; + atomic_t timer_active; + int enable; + + struct pwm_device *pwm; +}; + +extern struct snd_pwmsp pwmsp_chip; + +extern enum hrtimer_restart pwmsp_do_timer(struct hrtimer *handle); +extern void pwmsp_sync_stop(struct snd_pwmsp *chip); + +extern int snd_pwmsp_new_pcm(struct snd_pwmsp *chip); +extern int snd_pwmsp_new_mixer(struct snd_pwmsp *chip, int nopcm); + +void pwmsp_stop_sound(void); + +#endif diff --git a/sound/drivers/pwmsp/pwmsp_lib.c b/sound/drivers/pwmsp/pwmsp_lib.c new file mode 100644 index 0000000000000..d625369718481 --- /dev/null +++ b/sound/drivers/pwmsp/pwmsp_lib.c @@ -0,0 +1,346 @@ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* + * Based on PC-Speaker driver for Linux + * + * Copyright (C) 1993-1997 Michael Beck + * Copyright (C) 1997-2001 David Woodhouse + * Copyright (C) 2001-2008 Stas Sergeev + */ + +#include +#include +#include +#include +#include +#include +#include "pwmsp.h" + +//#define PWMSP_DEBUG +#define DEBUG_TIME + +static __maybe_unused ktime_t calltime, delta, rettime; +static __maybe_unused unsigned long duration; +static __maybe_unused unsigned long pwm_config_calls; + +/* + * Call snd_pcm_period_elapsed in a tasklet + * This avoids spinlock messes and long-running irq contexts + */ +static void pwmsp_call_pcm_elapsed(unsigned long priv) +{ + if (atomic_read(&pwmsp_chip.timer_active)) { + struct snd_pcm_substream *substream; + substream = pwmsp_chip.playback_substream; + if (substream) + snd_pcm_period_elapsed(substream); +#ifdef DEBUG_TIME + pr_debug("%lu calls, %lu ns, %lu ns per call\n", + pwm_config_calls, duration, + duration / pwm_config_calls); + pwm_config_calls = 0; + duration = 0; +#endif + } +} + +static DECLARE_TASKLET(pwmsp_pcm_tasklet, pwmsp_call_pcm_elapsed, 0); + +/* write the port and returns the next expire time in ns; + * called at the trigger-start and in hrtimer callback + */ +static u64 pwmsp_timer_update(struct snd_pwmsp *chip) +{ + struct snd_pcm_substream *substream; + struct snd_pcm_runtime *runtime; + int duty_ns; + s16 val; + + substream = chip->playback_substream; + if (!substream) + return 0; + + runtime = substream->runtime; + + /* TODO: Revisit here */ + val = runtime->dma_area[chip->playback_ptr] | + runtime->dma_area[chip->playback_ptr + 1] << 8; + duty_ns = (val * DELTA_DUTY) / 65535 + DELTA_DUTY / 2 + MIN_DUTY_NS; + +#ifdef DEBUG_TIME + calltime = ktime_get(); +#endif + + if (duty_ns && chip->enable) + pwm_config(chip->pwm, duty_ns, PWM_PERIOD_NS); + +#ifdef DEBUG_TIME + rettime = ktime_get(); + delta = ktime_sub(rettime, calltime); + duration += (unsigned long long) ktime_to_ns(delta) >> 10; + pwm_config_calls++; +#endif + + return SAMPLING_PERIOD_NS; +} + +static void pwmsp_pointer_update(struct snd_pwmsp *chip) +{ + struct snd_pcm_substream *substream; + size_t period_bytes, buffer_bytes; + int periods_elapsed; + unsigned long flags; + + /* update the playback position */ + substream = chip->playback_substream; + if (!substream) + return; + + period_bytes = snd_pcm_lib_period_bytes(substream); + buffer_bytes = snd_pcm_lib_buffer_bytes(substream); + + spin_lock_irqsave(&chip->substream_lock, flags); + chip->playback_ptr += chip->fmt_size; + periods_elapsed = chip->playback_ptr - chip->period_ptr; + if (periods_elapsed < 0) { +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: buffer_bytes mod period_bytes != 0 ? " + "(%zi %zi %zi)\n", + chip->playback_ptr, period_bytes, buffer_bytes); +#endif + periods_elapsed += buffer_bytes; + } + periods_elapsed /= period_bytes; + /* wrap the pointer _before_ calling snd_pcm_period_elapsed(), + * or ALSA will BUG on us. */ + chip->playback_ptr %= buffer_bytes; + + if (periods_elapsed) { + chip->period_ptr += periods_elapsed * period_bytes; + chip->period_ptr %= buffer_bytes; + } + spin_unlock_irqrestore(&chip->substream_lock, flags); + + if (periods_elapsed) + tasklet_schedule(&pwmsp_pcm_tasklet); +} + +enum hrtimer_restart pwmsp_do_timer(struct hrtimer *handle) +{ + struct snd_pwmsp *chip = container_of(handle, struct snd_pwmsp, timer); + u64 ns; + + if (!atomic_read(&chip->timer_active) || !chip->playback_substream) + return HRTIMER_NORESTART; + + ns = pwmsp_timer_update(chip); + if (!ns) { + printk(KERN_WARNING "PWMSP: unexpected stop\n"); + return HRTIMER_NORESTART; + } + + hrtimer_forward(handle, hrtimer_get_expires(handle), ns_to_ktime(ns)); + + pwmsp_pointer_update(chip); + + return HRTIMER_RESTART; +} + +static int pwmsp_start_playing(struct snd_pwmsp *chip) +{ +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: start_playing called\n"); +#endif + if (atomic_read(&chip->timer_active)) { + printk(KERN_ERR "PWMSP: Timer already active\n"); + return -EIO; + } + + /* Start the PWM */ + pwm_enable(chip->pwm); + pwm_config(chip->pwm, 0, PWM_PERIOD_NS); + atomic_set(&chip->timer_active, 1); + + hrtimer_start(&pwmsp_chip.timer, ktime_set(0, 0), HRTIMER_MODE_REL); + return 0; +} + +static void pwmsp_stop_playing(struct snd_pwmsp *chip) +{ +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: stop_playing called\n"); +#endif + if (!atomic_read(&chip->timer_active)) + return; + + atomic_set(&chip->timer_active, 0); + pwm_disable(chip->pwm); +} + +/* + * Force to stop and sync the stream + */ +void pwmsp_sync_stop(struct snd_pwmsp *chip) +{ + local_irq_disable(); + pwmsp_stop_playing(chip); + local_irq_enable(); + hrtimer_cancel(&chip->timer); + tasklet_kill(&pwmsp_pcm_tasklet); +} + +static int snd_pwmsp_playback_close(struct snd_pcm_substream *substream) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: close called\n"); +#endif + pwmsp_sync_stop(chip); + chip->playback_substream = NULL; + return 0; +} + +static int snd_pwmsp_playback_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *hw_params) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); + int err; + pwmsp_sync_stop(chip); + err = snd_pcm_lib_malloc_pages(substream, + params_buffer_bytes(hw_params)); + if (err < 0) + return err; + return 0; +} + +static int snd_pwmsp_playback_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: hw_free called\n"); +#endif + pwmsp_sync_stop(chip); + return snd_pcm_lib_free_pages(substream); +} + +static int snd_pwmsp_playback_prepare(struct snd_pcm_substream *substream) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); + pwmsp_sync_stop(chip); + chip->playback_ptr = 0; + chip->period_ptr = 0; + chip->fmt_size = + snd_pcm_format_physical_width(substream->runtime->format) >> 3; + chip->is_signed = snd_pcm_format_signed(substream->runtime->format); +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: prepare called, " + "size=%zi psize=%zi f=%zi f1=%i fsize=%i\n", + snd_pcm_lib_buffer_bytes(substream), + snd_pcm_lib_period_bytes(substream), + snd_pcm_lib_buffer_bytes(substream) / + snd_pcm_lib_period_bytes(substream), + substream->runtime->periods, + chip->fmt_size); +#endif + return 0; +} + +static int snd_pwmsp_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: trigger called\n"); +#endif + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + return pwmsp_start_playing(chip); + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + pwmsp_stop_playing(chip); + break; + default: + return -EINVAL; + } + return 0; +} + +static snd_pcm_uframes_t snd_pwmsp_playback_pointer(struct snd_pcm_substream + *substream) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); + unsigned int pos; + spin_lock(&chip->substream_lock); + pos = chip->playback_ptr; + spin_unlock(&chip->substream_lock); + return bytes_to_frames(substream->runtime, pos); +} + +static struct snd_pcm_hardware snd_pwmsp_playback = { + .info = (SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_HALF_DUPLEX | + SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID), + .formats = SNDRV_PCM_FMTBIT_S16_LE, + .rates = SNDRV_PCM_RATE_KNOT, + .rate_min = SAMPLING_FREQ, + .rate_max = SAMPLING_FREQ, + .channels_min = 1, + .channels_max = 1, + .buffer_bytes_max = PWMSP_BUFFER_SIZE, + .period_bytes_min = 64, + .period_bytes_max = PWMSP_MAX_PERIOD_SIZE, + .periods_min = 2, + .periods_max = PWMSP_MAX_PERIODS, + .fifo_size = 0, +}; + +static int snd_pwmsp_playback_open(struct snd_pcm_substream *substream) +{ + struct snd_pwmsp *chip = snd_pcm_substream_chip(substream); + struct snd_pcm_runtime *runtime = substream->runtime; +#ifdef PWMSP_DEBUG + printk(KERN_INFO "PWMSP: open called\n"); +#endif + if (atomic_read(&chip->timer_active)) { + printk(KERN_ERR "PWMSP: still active!!\n"); + return -EBUSY; + } + runtime->hw = snd_pwmsp_playback; + chip->playback_substream = substream; + return 0; +} + +static struct snd_pcm_ops snd_pwmsp_playback_ops = { + .open = snd_pwmsp_playback_open, + .close = snd_pwmsp_playback_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = snd_pwmsp_playback_hw_params, + .hw_free = snd_pwmsp_playback_hw_free, + .prepare = snd_pwmsp_playback_prepare, + .trigger = snd_pwmsp_trigger, + .pointer = snd_pwmsp_playback_pointer, +}; + +int snd_pwmsp_new_pcm(struct snd_pwmsp *chip) +{ + int err; + + err = snd_pcm_new(chip->card, "pwmspeaker", 0, 1, 0, &chip->pcm); + if (err < 0) + return err; + + snd_pcm_set_ops(chip->pcm, SNDRV_PCM_STREAM_PLAYBACK, + &snd_pwmsp_playback_ops); + + chip->pcm->private_data = chip; + chip->pcm->info_flags = SNDRV_PCM_INFO_HALF_DUPLEX; + strcpy(chip->pcm->name, "pwmsp"); + + snd_pcm_lib_preallocate_pages_for_all(chip->pcm, + SNDRV_DMA_TYPE_CONTINUOUS, + snd_dma_continuous_data + (GFP_KERNEL), PWMSP_BUFFER_SIZE, + PWMSP_BUFFER_SIZE); + + return 0; +} diff --git a/sound/soc/davinci/Kconfig b/sound/soc/davinci/Kconfig index cf686dbf986b1..3451d6ea528ad 100644 --- a/sound/soc/davinci/Kconfig +++ b/sound/soc/davinci/Kconfig @@ -44,6 +44,7 @@ config SND_AM33XX_SOC_EVM tristate "SoC Audio for the AM33XX chip based boards" depends on SND_EDMA_SOC && SOC_AM33XX && I2C select SND_DAVINCI_SOC_GENERIC_EVM + select SND_SOC_PCM5102A help Say Y or M if you want to add support for SoC audio on AM33XX boards using McASP and TLV320AIC3X codec. For example AM335X-EVM,