From 8e463e34f3c81ab31ca94e4d598d8dd0e243529e Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 14:59:02 -0600 Subject: [PATCH 001/135] merge: can-isotp: https://github.com/hartkopp/can-isotp https://github.com/hartkopp/can-isotp/commit/748f8efe84ab7c27496d8c8165f28892ba0e8868 Signed-off-by: Robert Nelson --- include/uapi/linux/can/isotp.h | 136 +++ net/can/isotp.c | 1554 ++++++++++++++++++++++++++++++++ 2 files changed, 1690 insertions(+) create mode 100644 include/uapi/linux/can/isotp.h create mode 100644 net/can/isotp.c diff --git a/include/uapi/linux/can/isotp.h b/include/uapi/linux/can/isotp.h new file mode 100644 index 0000000000000..88831772f9ee9 --- /dev/null +++ b/include/uapi/linux/can/isotp.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * socketcan/can/isotp.h + * + * Definitions for isotp CAN sockets + * + * Author: Oliver Hartkopp + * Copyright (c) 2008 Volkswagen Group Electronic Research + * All rights reserved. + * + * Send feedback to + * + */ + +#ifndef CAN_ISOTP_H +#define CAN_ISOTP_H + +#include + +#define SOL_CAN_ISOTP (SOL_CAN_BASE + CAN_ISOTP) + +/* for socket options affecting the socket (not the global system) */ + +#define CAN_ISOTP_OPTS 1 /* pass struct can_isotp_options */ + +#define CAN_ISOTP_RECV_FC 2 /* pass struct can_isotp_fc_options */ + +/* sockopts to force stmin timer values for protocol regression tests */ + +#define CAN_ISOTP_TX_STMIN 3 /* pass __u32 value in nano secs */ + /* use this time instead of value */ + /* provided in FC from the receiver */ + +#define CAN_ISOTP_RX_STMIN 4 /* pass __u32 value in nano secs */ + /* ignore received CF frames which */ + /* timestamps differ less than val */ + +#define CAN_ISOTP_LL_OPTS 5 /* pass struct can_isotp_ll_options */ + +struct can_isotp_options { + + __u32 flags; /* set flags for isotp behaviour. */ + /* __u32 value : flags see below */ + + __u32 frame_txtime; /* frame transmission time (N_As/N_Ar) */ + /* __u32 value : time in nano secs */ + + __u8 ext_address; /* set address for extended addressing */ + /* __u8 value : extended address */ + + __u8 txpad_content; /* set content of padding byte (tx) */ + /* __u8 value : content on tx path */ + + __u8 rxpad_content; /* set content of padding byte (rx) */ + /* __u8 value : content on rx path */ + + __u8 rx_ext_address; /* set address for extended addressing */ + /* __u8 value : extended address (rx) */ +}; + +struct can_isotp_fc_options { + + __u8 bs; /* blocksize provided in FC frame */ + /* __u8 value : blocksize. 0 = off */ + + __u8 stmin; /* separation time provided in FC frame */ + /* __u8 value : */ + /* 0x00 - 0x7F : 0 - 127 ms */ + /* 0x80 - 0xF0 : reserved */ + /* 0xF1 - 0xF9 : 100 us - 900 us */ + /* 0xFA - 0xFF : reserved */ + + __u8 wftmax; /* max. number of wait frame transmiss. */ + /* __u8 value : 0 = omit FC N_PDU WT */ +}; + +struct can_isotp_ll_options { + + __u8 mtu; /* generated & accepted CAN frame type */ + /* __u8 value : */ + /* CAN_MTU (16) -> standard CAN 2.0 */ + /* CANFD_MTU (72) -> CAN FD frame */ + + __u8 tx_dl; /* tx link layer data length in bytes */ + /* (configured maximum payload length) */ + /* __u8 value : 8,12,16,20,24,32,48,64 */ + /* => rx path supports all LL_DL values */ + + __u8 tx_flags; /* set into struct canfd_frame.flags */ + /* at frame creation: e.g. CANFD_BRS */ + /* Obsolete when the BRS flag is fixed */ + /* by the CAN netdriver configuration */ +}; + +/* flags for isotp behaviour */ + +#define CAN_ISOTP_LISTEN_MODE 0x001 /* listen only (do not send FC) */ +#define CAN_ISOTP_EXTEND_ADDR 0x002 /* enable extended addressing */ +#define CAN_ISOTP_TX_PADDING 0x004 /* enable CAN frame padding tx path */ +#define CAN_ISOTP_RX_PADDING 0x008 /* enable CAN frame padding rx path */ +#define CAN_ISOTP_CHK_PAD_LEN 0x010 /* check received CAN frame padding */ +#define CAN_ISOTP_CHK_PAD_DATA 0x020 /* check received CAN frame padding */ +#define CAN_ISOTP_HALF_DUPLEX 0x040 /* half duplex error state handling */ +#define CAN_ISOTP_FORCE_TXSTMIN 0x080 /* ignore stmin from received FC */ +#define CAN_ISOTP_FORCE_RXSTMIN 0x100 /* ignore CFs depending on rx stmin */ +#define CAN_ISOTP_RX_EXT_ADDR 0x200 /* different rx extended addressing */ +#define CAN_ISOTP_WAIT_TX_DONE 0x400 /* wait for tx completion */ +#define CAN_ISOTP_SF_BROADCAST 0x800 /* 1-to-N functional addressing */ + +/* default values */ + +#define CAN_ISOTP_DEFAULT_FLAGS 0 +#define CAN_ISOTP_DEFAULT_EXT_ADDRESS 0x00 +#define CAN_ISOTP_DEFAULT_PAD_CONTENT 0xCC /* prevent bit-stuffing */ +#define CAN_ISOTP_DEFAULT_FRAME_TXTIME 0 +#define CAN_ISOTP_DEFAULT_RECV_BS 0 +#define CAN_ISOTP_DEFAULT_RECV_STMIN 0x00 +#define CAN_ISOTP_DEFAULT_RECV_WFTMAX 0 + +#define CAN_ISOTP_DEFAULT_LL_MTU CAN_MTU +#define CAN_ISOTP_DEFAULT_LL_TX_DL CAN_MAX_DLEN +#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS 0 + +/* + * Remark on CAN_ISOTP_DEFAULT_RECV_* values: + * + * We can strongly assume, that the Linux Kernel implementation of + * CAN_ISOTP is capable to run with BS=0, STmin=0 and WFTmax=0. + * But as we like to be able to behave as a commonly available ECU, + * these default settings can be changed via sockopts. + * For that reason the STmin value is intentionally _not_ checked for + * consistency and copied directly into the flow control (FC) frame. + * + */ + +#endif diff --git a/net/can/isotp.c b/net/can/isotp.c new file mode 100644 index 0000000000000..ae939bf4b4561 --- /dev/null +++ b/net/can/isotp.c @@ -0,0 +1,1554 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* isotp.c - ISO 15765-2 CAN transport protocol for protocol family CAN + * + * WARNING: This is ALPHA code for discussions and first tests that should + * not be used in production environments. + * + * In the discussion the Socket-API to the userspace or the ISO-TP socket + * options or the return values we may change! Current behaviour: + * + * - no ISO-TP specific return values are provided to the userspace + * - when a transfer (tx) is on the run the next write() blocks until it's done + * - no support for sending wait frames to the data source in the rx path + * + * Copyright (c) 2020 Volkswagen Group Electronic Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Volkswagen nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General + * Public License ("GPL") version 2, in which case the provisions of the + * GPL apply INSTEAD OF those given above. + * + * The provided data structures and external interfaces from this code + * are not restricted to be used by modules with a GPL compatible license. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) +#include +#define CAN_SKBRES sizeof(struct can_skb_priv) +#else +#define CAN_SKBRES 0 +#endif +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25) +#include "compat.h" +#endif + +#define CAN_ISOTP_VERSION "20201014 - out-of-tree" + +MODULE_DESCRIPTION("PF_CAN isotp 15765-2:2016 protocol"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Oliver Hartkopp "); +MODULE_ALIAS("can-proto-6"); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#error This modules needs hrtimers (available since Kernel 2.6.22) +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,10,0) +#error No need to compile this out-of-tree driver! ISO-TP is part of Linux Mainline kernel since Linux 5.10. +#endif + +#define DBG(fmt, args...) (printk( KERN_DEBUG "can-isotp: %s: " fmt, \ + __func__, ##args)) +#undef DBG +#define DBG(fmt, args...) + +#define SINGLE_MASK(id) (((id) & CAN_EFF_FLAG) ? \ + (CAN_EFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG) : \ + (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG)) + +/* ISO 15765-2:2016 supports more than 4095 byte per ISO PDU as the FF_DL can + * take full 32 bit values (4 Gbyte). We would need some good concept to handle + * this between user space and kernel space. For now increase the static buffer + * to something about 8 kbyte to be able to test this new functionality. + */ +#define MAX_MSG_LENGTH 8200 + +/* N_PCI type values in bits 7-4 of N_PCI bytes */ +#define N_PCI_SF 0x00 /* single frame */ +#define N_PCI_FF 0x10 /* first frame */ +#define N_PCI_CF 0x20 /* consecutive frame */ +#define N_PCI_FC 0x30 /* flow control */ + +#define N_PCI_SZ 1 /* size of the PCI byte #1 */ +#define SF_PCI_SZ4 1 /* size of SingleFrame PCI including 4 bit SF_DL */ +#define SF_PCI_SZ8 2 /* size of SingleFrame PCI including 8 bit SF_DL */ +#define FF_PCI_SZ12 2 /* size of FirstFrame PCI including 12 bit FF_DL */ +#define FF_PCI_SZ32 6 /* size of FirstFrame PCI including 32 bit FF_DL */ +#define FC_CONTENT_SZ 3 /* flow control content size in byte (FS/BS/STmin) */ + +#define ISOTP_CHECK_PADDING (CAN_ISOTP_CHK_PAD_LEN | CAN_ISOTP_CHK_PAD_DATA) + +/* Flow Status given in FC frame */ +#define ISOTP_FC_CTS 0 /* clear to send */ +#define ISOTP_FC_WT 1 /* wait */ +#define ISOTP_FC_OVFLW 2 /* overflow */ + +enum { + ISOTP_IDLE = 0, + ISOTP_WAIT_FIRST_FC, + ISOTP_WAIT_FC, + ISOTP_WAIT_DATA, + ISOTP_SENDING +}; + +struct tpcon { + int idx; + int len; + u8 state; + u8 bs; + u8 sn; + u8 ll_dl; + u8 buf[MAX_MSG_LENGTH + 1]; +}; + +struct isotp_sock { + struct sock sk; + int bound; + int ifindex; + canid_t txid; + canid_t rxid; + ktime_t tx_gap; + ktime_t lastrxcf_tstamp; + struct hrtimer rxtimer, txtimer; + struct tasklet_struct txtsklet; + struct can_isotp_options opt; + struct can_isotp_fc_options rxfc, txfc; + struct can_isotp_ll_options ll; + u32 force_tx_stmin; + u32 force_rx_stmin; + struct tpcon rx, tx; + struct notifier_block notifier; + wait_queue_head_t wait; +}; + +static inline struct isotp_sock *isotp_sk(const struct sock *sk) +{ + return (struct isotp_sock *)sk; +} + +static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + rxtimer); + if (so->rx.state == ISOTP_WAIT_DATA) { + DBG("we did not get new data frames in time.\n"); + + /* reset tx state */ + so->rx.state = ISOTP_IDLE; + } + + return HRTIMER_NORESTART; +} + +static void isotp_skb_reserve(struct sk_buff *skb, struct net_device *dev) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) + can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,5) + can_skb_prv(skb)->skbcnt = 0; +#endif +} + +static void isotp_skb_destructor(struct sk_buff *skb) +{ + sock_put(skb->sk); +} + +static inline void isotp_skb_set_owner(struct sk_buff *skb, struct sock *sk) +{ + if (sk) { + sock_hold(sk); + skb->destructor = isotp_skb_destructor; + skb->sk = sk; + } +} + +static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus) +{ + struct net_device *dev; + struct sk_buff *nskb; + struct canfd_frame *ncf; + struct isotp_sock *so = isotp_sk(sk); + + nskb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any()); + if (!nskb) + return 1; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) { + kfree_skb(nskb); + return 1; + } + isotp_skb_reserve(nskb, dev); + nskb->dev = dev; + isotp_skb_set_owner(nskb, sk); + ncf = (struct canfd_frame *) nskb->data; + skb_put(nskb, so->ll.mtu); + + /* create & send flow control reply */ + ncf->can_id = so->txid; + + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + memset(ncf->data, so->opt.txpad_content, CAN_MAX_DLEN); + ncf->len = CAN_MAX_DLEN; + } else { + ncf->len = ae + FC_CONTENT_SZ; + } + + ncf->data[ae] = N_PCI_FC | flowstatus; + ncf->data[ae + 1] = so->rxfc.bs; + ncf->data[ae + 2] = so->rxfc.stmin; + + if (ae) + ncf->data[0] = so->opt.ext_address; + + if (so->ll.mtu == CANFD_MTU) + ncf->flags = so->ll.tx_flags; + + can_send(nskb, 1); + dev_put(dev); + + /* reset blocksize counter */ + so->rx.bs = 0; + + /* reset last CF frame rx timestamp for rx stmin enforcement */ + so->lastrxcf_tstamp = ktime_set(0, 0); + + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1, 0), HRTIMER_MODE_REL); + return 0; +} + +static void isotp_rcv_skb(struct sk_buff *skb, struct sock *sk) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)skb->cb; + + BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct sockaddr_can)); + + memset(addr, 0, sizeof(*addr)); + addr->can_family = AF_CAN; + addr->can_ifindex = skb->dev->ifindex; + + if (sock_queue_rcv_skb(sk, skb) < 0) + kfree_skb(skb); +} + +static u8 padlen(u8 datalen) +{ + const u8 plen[] = {8, 8, 8, 8, 8, 8, 8, 8, 8, /* 0 - 8 */ + 12, 12, 12, 12, /* 9 - 12 */ + 16, 16, 16, 16, /* 13 - 16 */ + 20, 20, 20, 20, /* 17 - 20 */ + 24, 24, 24, 24, /* 21 - 24 */ + 32, 32, 32, 32, 32, 32, 32, 32, /* 25 - 32 */ + 48, 48, 48, 48, 48, 48, 48, 48, /* 33 - 40 */ + 48, 48, 48, 48, 48, 48, 48, 48}; /* 41 - 48 */ + + if (datalen > 48) + return 64; + + return plen[datalen]; +} + +/* check for length optimization and return 1/true when the check fails */ +static int check_optimized(struct canfd_frame *cf, int start_index) +{ + /* for CAN_DL <= 8 the start_index is equal to the CAN_DL as the + * padding would start at this point. E.g. if the padding would + * start at cf.data[7] cf->len has to be 7 to be optimal. + * Note: The data[] index starts with zero. + */ + if (cf->len <= CAN_MAX_DLEN) + return (cf->len != start_index); + + /* This relation is also valid in the non-linear DLC range, where + * we need to take care of the minimal next possible CAN_DL. + * The correct check would be (padlen(cf->len) != padlen(start_index)). + * But as cf->len can only take discrete values from 12, .., 64 at this + * point the padlen(cf->len) is always equal to cf->len. + */ + return (cf->len != padlen(start_index)); +} + +/* check padding and return 1/true when the check fails */ +static int check_pad(struct isotp_sock *so, struct canfd_frame *cf, + int start_index, u8 content) +{ + int i; + + /* no RX_PADDING value => check length of optimized frame length */ + if (!(so->opt.flags & CAN_ISOTP_RX_PADDING)) { + if (so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) + return check_optimized(cf, start_index); + + /* no valid test against empty value => ignore frame */ + return 1; + } + + /* check datalength of correctly padded CAN frame */ + if ((so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) && + cf->len != padlen(cf->len)) + return 1; + + /* check padding content */ + if (so->opt.flags & CAN_ISOTP_CHK_PAD_DATA) { + for (i = start_index; i < cf->len; i++) + if (cf->data[i] != content) + return 1; + } + return 0; +} + +static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) +{ + if (so->tx.state != ISOTP_WAIT_FC && + so->tx.state != ISOTP_WAIT_FIRST_FC) + return 0; + + hrtimer_cancel(&so->txtimer); + + if ((cf->len < ae + FC_CONTENT_SZ) || + ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) { + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + return 1; + } + + /* get communication parameters only from the first FC frame */ + if (so->tx.state == ISOTP_WAIT_FIRST_FC) { + so->txfc.bs = cf->data[ae + 1]; + so->txfc.stmin = cf->data[ae + 2]; + + /* fix wrong STmin values according spec */ + if (so->txfc.stmin > 0x7F && + (so->txfc.stmin < 0xF1 || so->txfc.stmin > 0xF9)) + so->txfc.stmin = 0x7F; + + so->tx_gap = ktime_set(0, 0); + /* add transmission time for CAN frame N_As */ + so->tx_gap = ktime_add_ns(so->tx_gap, so->opt.frame_txtime); + /* add waiting time for consecutive frames N_Cs */ + if (so->opt.flags & CAN_ISOTP_FORCE_TXSTMIN) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->force_tx_stmin); + else if (so->txfc.stmin < 0x80) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->txfc.stmin * 1000000); + else + so->tx_gap = ktime_add_ns(so->tx_gap, + (so->txfc.stmin - 0xF0) + * 100000); + so->tx.state = ISOTP_WAIT_FC; + } + + DBG("FC frame: FS %d, BS %d, STmin 0x%02X, tx_gap %lld\n", + cf->data[ae] & 0x0F & 0x0F, so->txfc.bs, so->txfc.stmin, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + (long long)so->tx_gap); +#else + (long long)so->tx_gap.tv64); +#endif + + switch (cf->data[ae] & 0x0F) { + case ISOTP_FC_CTS: + so->tx.bs = 0; + so->tx.state = ISOTP_SENDING; + DBG("starting txtimer for sending\n"); + /* start cyclic timer for sending CF frame */ + hrtimer_start(&so->txtimer, so->tx_gap, + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_WT: + DBG("starting waiting for next FC\n"); + /* start timer to wait for next FC frame */ + hrtimer_start(&so->txtimer, ktime_set(1,0), + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_OVFLW: + DBG("overflow in receiver side\n"); + + default: + /* stop this tx job */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + } + return 0; +} + +static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen, + struct sk_buff *skb, int len) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + if (!len || len > cf->len - pcilen) + return 1; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, pcilen + len, so->opt.rxpad_content)) { + return 1; + } + + nskb = alloc_skb(len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, len), &cf->data[pcilen], len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; +} + +static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) +{ + struct isotp_sock *so = isotp_sk(sk); + int i; + int off; + int ff_pci_sz; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + /* get the used sender LL_DL from the (first) CAN frame data length */ + so->rx.ll_dl = padlen(cf->len); + + /* the first frame has to use the entire frame up to LL_DL length */ + if (cf->len != so->rx.ll_dl) + return 1; + + /* get the FF_DL */ + so->rx.len = (cf->data[ae] & 0x0F) << 8; + so->rx.len += cf->data[ae + 1]; + + /* Check for FF_DL escape sequence supporting 32 bit PDU length */ + if (so->rx.len) { + ff_pci_sz = FF_PCI_SZ12; + } else { + /* FF_DL = 0 => get real length from next 4 bytes */ + so->rx.len = cf->data[ae + 2] << 24; + so->rx.len += cf->data[ae + 3] << 16; + so->rx.len += cf->data[ae + 4] << 8; + so->rx.len += cf->data[ae + 5]; + ff_pci_sz = FF_PCI_SZ32; + } + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->rx.ll_dl > CAN_MAX_DLEN) ? 1 : 0; + + if (so->rx.len + ae + off + ff_pci_sz < so->rx.ll_dl) + return 1; + + if (so->rx.len > MAX_MSG_LENGTH) { + /* send FC frame with overflow status */ + isotp_send_fc(sk, ae, ISOTP_FC_OVFLW); + return 1; + } + + /* copy the first received data bytes */ + so->rx.idx = 0; + for (i = ae + ff_pci_sz; i < so->rx.ll_dl; i++) + so->rx.buf[so->rx.idx++] = cf->data[i]; + + /* initial setup for this pdu reception */ + so->rx.sn = 1; + so->rx.state = ISOTP_WAIT_DATA; + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* send our first FC frame */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, + struct sk_buff *skb) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + int i; + + if (so->rx.state != ISOTP_WAIT_DATA) + return 0; + + /* drop if timestamp gap is less than force_rx_stmin nano secs */ + if (so->opt.flags & CAN_ISOTP_FORCE_RXSTMIN) { + if (ktime_to_ns(ktime_sub(skb->tstamp, so->lastrxcf_tstamp)) < + so->force_rx_stmin) + return 0; + + so->lastrxcf_tstamp = skb->tstamp; + } + + hrtimer_cancel(&so->rxtimer); + + /* CFs are never longer than the FF */ + if (cf->len > so->rx.ll_dl) + return 1; + + /* CFs have usually the LL_DL length */ + if (cf->len < so->rx.ll_dl) { + /* this is only allowed for the last CF */ + if (so->rx.len - so->rx.idx > so->rx.ll_dl - ae - N_PCI_SZ) + return 1; + } + + if ((cf->data[ae] & 0x0F) != so->rx.sn) { + DBG("wrong sn %d. expected %d.\n", + cf->data[ae] & 0x0F, so->rx.sn); + /* some error reporting? */ + so->rx.state = ISOTP_IDLE; + return 1; + } + so->rx.sn++; + so->rx.sn %= 16; + + for (i = ae + N_PCI_SZ; i < cf->len; i++) { + so->rx.buf[so->rx.idx++] = cf->data[i]; + if (so->rx.idx >= so->rx.len) + break; + } + + if (so->rx.idx >= so->rx.len) { + /* we are done */ + so->rx.state = ISOTP_IDLE; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, i + 1, so->opt.rxpad_content)) { + return 1; + } + + nskb = alloc_skb(so->rx.len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, so->rx.len), so->rx.buf, + so->rx.len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; + } + + /* perform blocksize handling, if enabled */ + if (!so->rxfc.bs || ++so->rx.bs < so->rxfc.bs) { + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1, 0), + HRTIMER_MODE_REL); + return 0; + } + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* we reached the specified blocksize so->rxfc.bs */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static void isotp_rcv(struct sk_buff *skb, void *data) +{ + struct sock *sk = (struct sock *)data; + struct isotp_sock *so = isotp_sk(sk); + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; + u8 n_pci_type, sf_dl; + + /* Strictly receive only frames with the configured MTU size + * => clear separation of CAN2.0 / CAN FD transport channels + */ + if (skb->len != so->ll.mtu) + return; + + cf = (struct canfd_frame *)skb->data; + + /* if enabled: check reception of my configured extended address */ + if (ae && cf->data[0] != so->opt.rx_ext_address) + return; + + n_pci_type = cf->data[ae] & 0xF0; + + if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) { + /* check rx/tx path half duplex expectations */ + if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) || + (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC)) + return; + } + + switch (n_pci_type) { + case N_PCI_FC: + /* tx path: flow control frame containing the FC parameters */ + isotp_rcv_fc(so, cf, ae); + break; + + case N_PCI_SF: + /* rx path: single frame + * + * As we do not have a rx.ll_dl configuration, we can only test + * if the CAN frames payload length matches the LL_DL == 8 + * requirements - no matter if it's CAN 2.0 or CAN FD + */ + + /* get the SF_DL from the N_PCI byte */ + sf_dl = cf->data[ae] & 0x0F; + + if (cf->len <= CAN_MAX_DLEN) { + isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl); + } else { + if (skb->len == CANFD_MTU) { + /* We have a CAN FD frame and CAN_DL is greater than 8: + * Only frames with the SF_DL == 0 ESC value are valid. + * + * If so take care of the increased SF PCI size + * (SF_PCI_SZ8) to point to the message content behind + * the extended SF PCI info and get the real SF_DL + * length value from the formerly first data byte. + */ + if (sf_dl == 0) + isotp_rcv_sf(sk, cf, SF_PCI_SZ8 + ae, skb, + cf->data[SF_PCI_SZ4 + ae]); + } + } + break; + + case N_PCI_FF: + /* rx path: first frame */ + isotp_rcv_ff(sk, cf, ae); + break; + + case N_PCI_CF: + /* rx path: consecutive frame */ + isotp_rcv_cf(sk, cf, ae, skb); + break; + } +} + +static void isotp_fill_dataframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae, int off) +{ + int pcilen = N_PCI_SZ + ae + off; + int space = so->tx.ll_dl - pcilen; + int num = min_t(int, so->tx.len - so->tx.idx, space); + int i; + + cf->can_id = so->txid; + cf->len = num + pcilen; + + if (num < space) { + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + /* user requested padding */ + cf->len = padlen(cf->len); + memset(cf->data, so->opt.txpad_content, cf->len); + } else if (cf->len > CAN_MAX_DLEN) { + /* mandatory padding for CAN FD frames */ + cf->len = padlen(cf->len); + memset(cf->data, CAN_ISOTP_DEFAULT_PAD_CONTENT, + cf->len); + } + } + + for (i = 0; i < num; i++) + cf->data[pcilen + i] = so->tx.buf[so->tx.idx++]; + + if (ae) + cf->data[0] = so->opt.ext_address; +} + +static void isotp_create_fframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae) +{ + int i; + int ff_pci_sz; + + cf->can_id = so->txid; + cf->len = so->tx.ll_dl; + if (ae) + cf->data[0] = so->opt.ext_address; + + /* create N_PCI bytes with 12/32 bit FF_DL data length */ + if (so->tx.len > 4095) { + /* use 32 bit FF_DL notation */ + cf->data[ae] = N_PCI_FF; + cf->data[ae + 1] = 0; + cf->data[ae + 2] = (u8)(so->tx.len >> 24) & 0xFFU; + cf->data[ae + 3] = (u8)(so->tx.len >> 16) & 0xFFU; + cf->data[ae + 4] = (u8)(so->tx.len >> 8) & 0xFFU; + cf->data[ae + 5] = (u8)so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ32; + } else { + /* use 12 bit FF_DL notation */ + cf->data[ae] = (u8)(so->tx.len >> 8) | N_PCI_FF; + cf->data[ae + 1] = (u8)so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ12; + } + + /* add first data bytes depending on ae */ + for (i = ae + ff_pci_sz; i < so->tx.ll_dl; i++) + cf->data[i] = so->tx.buf[so->tx.idx++]; + + so->tx.sn = 1; + so->tx.state = ISOTP_WAIT_FIRST_FC; +} + +static void isotp_tx_timer_tsklet(unsigned long data) +{ + struct isotp_sock *so = (struct isotp_sock *)data; + struct sock *sk = &so->sk; + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; + + switch (so->tx.state) { + case ISOTP_WAIT_FC: + case ISOTP_WAIT_FIRST_FC: + + /* we did not get any flow control frame in time */ + + DBG("we did not get FC frame in time.\n"); + + /* reset tx state */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + break; + + case ISOTP_SENDING: + + /* push out the next segmented pdu */ + + DBG("next pdu to send.\n"); + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + break; + +isotp_tx_burst: + skb = alloc_skb(so->ll.mtu + CAN_SKBRES, GFP_ATOMIC); + if (!skb) { + dev_put(dev); + break; + } + + isotp_skb_reserve(skb, dev); + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* create consecutive frame */ + isotp_fill_dataframe(cf, so, ae, 0); + + /* place consecutive frame N_PCI in appropriate index */ + cf->data[ae] = N_PCI_CF | so->tx.sn++; + so->tx.sn %= 16; + so->tx.bs++; + + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + isotp_skb_set_owner(skb, sk); + can_send(skb, 1); + + if (so->tx.idx >= so->tx.len) { + /* we are done */ + DBG("we are done\n"); + so->tx.state = ISOTP_IDLE; + dev_put(dev); + wake_up_interruptible(&so->wait); + break; + } + + if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { + /* stop and wait for FC */ + DBG("BS stop and wait for FC\n"); + so->tx.state = ISOTP_WAIT_FC; + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), ktime_set(1,0)), + HRTIMER_MODE_ABS); + break; + } + + /* no gap between data frames needed => use burst mode */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (!so->tx_gap) + goto isotp_tx_burst; +#else + if (!so->tx_gap.tv64) + goto isotp_tx_burst; +#endif + + /* start timer to send next data frame with correct delay */ + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), so->tx_gap), + HRTIMER_MODE_ABS); + break; + + default: + WARN_ON_ONCE(1); + } +} + +static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + txtimer); + tasklet_schedule(&so->txtsklet); + + return HRTIMER_NORESTART; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) +#else +static int isotp_sendmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; + int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0; + int off; + int err; + + if (!so->bound) + return -EADDRNOTAVAIL; + + /* we do not support multiple buffers - for now */ + if (so->tx.state != ISOTP_IDLE) { + if (msg->msg_flags & MSG_DONTWAIT) + return -EAGAIN; + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + } + + if (!size || size > MAX_MSG_LENGTH) + return -EINVAL; + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->tx.ll_dl > CAN_MAX_DLEN) ? 1 : 0; + + /* does the given data fit into a single frame for SF_BROADCAST? */ + if ((so->opt.flags & CAN_ISOTP_SF_BROADCAST) && + (size > so->tx.ll_dl - SF_PCI_SZ4 - ae - off)) + return -EINVAL; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_from_msg(so->tx.buf, msg, size); +#else + err = memcpy_fromiovec(so->tx.buf, msg->msg_iov, size); +#endif + if (err < 0) + return err; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + return -ENXIO; + + skb = sock_alloc_send_skb(sk, so->ll.mtu + CAN_SKBRES, + msg->msg_flags & MSG_DONTWAIT, &err); + if (!skb) { + dev_put(dev); + return err; + } + + isotp_skb_reserve(skb, dev); + + so->tx.state = ISOTP_SENDING; + so->tx.len = size; + so->tx.idx = 0; + + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* check for single frame transmission depending on TX_DL */ + if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { + /* The message size generally fits into a SingleFrame - good. + * + * SF_DL ESC offset optimization: + * + * When TX_DL is greater 8 but the message would still fit + * into a 8 byte CAN frame, we can omit the offset. + * This prevents a protocol caused length extension from + * CAN_DL = 8 to CAN_DL = 12 due to the SF_SL ESC handling. + */ + if (size <= CAN_MAX_DLEN - SF_PCI_SZ4 - ae) + off = 0; + + isotp_fill_dataframe(cf, so, ae, off); + + /* place single frame N_PCI w/o length in appropriate index */ + cf->data[ae] = N_PCI_SF; + + /* place SF_DL size value depending on the SF_DL ESC offset */ + if (off) + cf->data[SF_PCI_SZ4 + ae] = size; + else + cf->data[ae] |= size; + + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + + /* don't enable wait queue for a single frame transmission */ + wait_tx_done = 0; + } else { + /* send first frame and wait for FC */ + + isotp_create_fframe(cf, so, ae); + + DBG("starting txtimer for fc\n"); + /* start timeout for FC */ + hrtimer_start(&so->txtimer, ktime_set(1,0), HRTIMER_MODE_REL); + } + + /* send the first or only CAN frame */ + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + skb->sk = sk; + err = can_send(skb, 1); + dev_put(dev); + if (err) { + printk_once(KERN_NOTICE "can-isotp: %s: can_send_ret %d\n", + __func__, err); + return err; + } + + if (wait_tx_done) { + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + } + + return size; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, + int flags) +#else +static int isotp_recvmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size, int flags) +#endif +{ + struct sock *sk = sock->sk; + struct sk_buff *skb; + int err = 0; + int noblock; + + noblock = flags & MSG_DONTWAIT; + flags &= ~MSG_DONTWAIT; + + skb = skb_recv_datagram(sk, flags, noblock, &err); + if (!skb) + return err; + + if (size < skb->len) + msg->msg_flags |= MSG_TRUNC; + else + size = skb->len; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_to_msg(msg, skb->data, size); +#else + err = memcpy_toiovec(msg->msg_iov, skb->data, size); +#endif + if (err < 0) { + skb_free_datagram(sk, skb); + return err; + } + + sock_recv_timestamp(msg, sk, skb); + + if (msg->msg_name) { + msg->msg_namelen = sizeof(struct sockaddr_can); + memcpy(msg->msg_name, skb->cb, msg->msg_namelen); + } + + skb_free_datagram(sk, skb); + + return size; +} + +static int isotp_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so; + struct net *net; + + if (!sk) + return 0; + + so = isotp_sk(sk); + net = sock_net(sk); + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + + unregister_netdevice_notifier(&so->notifier); + + lock_sock(sk); + + hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->rxtimer); + tasklet_kill(&so->txtsklet); + + /* remove current filters & unregister */ + if (so->bound && (!(so->opt.flags & CAN_ISOTP_SF_BROADCAST))) { + if (so->ifindex) { + struct net_device *dev; + + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + so->ifindex = 0; + so->bound = 0; + + sock_orphan(sk); + sock->sk = NULL; + + release_sock(sk); + sock_put(sk); + + return 0; +} + +static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct net *net = sock_net(sk); + int ifindex; + struct net_device *dev; + int err = 0; + int notify_enetdown = 0; + int do_rx_reg = 1; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) + if (len < CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.tp)) +#else + if (len < sizeof(*addr)) +#endif + return -EINVAL; + + /* do not register frame reception for functional addressing */ + if (so->opt.flags & CAN_ISOTP_SF_BROADCAST) + do_rx_reg = 0; + + /* do not validate rx address for functional addressing */ + if (do_rx_reg) { + if (addr->can_addr.tp.rx_id == addr->can_addr.tp.tx_id) + return -EADDRNOTAVAIL; + + if (addr->can_addr.tp.rx_id & (CAN_ERR_FLAG | CAN_RTR_FLAG)) + return -EADDRNOTAVAIL; + } + + if (addr->can_addr.tp.tx_id & (CAN_ERR_FLAG | CAN_RTR_FLAG)) + return -EADDRNOTAVAIL; + + if (!addr->can_ifindex) + return -ENODEV; + + lock_sock(sk); + + if (so->bound && addr->can_ifindex == so->ifindex && + addr->can_addr.tp.rx_id == so->rxid && + addr->can_addr.tp.tx_id == so->txid) + goto out; + + dev = dev_get_by_index(net, addr->can_ifindex); + if (!dev) { + err = -ENODEV; + goto out; + } + if (dev->type != ARPHRD_CAN) { + dev_put(dev); + err = -ENODEV; + goto out; + } + if (dev->mtu < so->ll.mtu) { + dev_put(dev); + err = -EINVAL; + goto out; + } + if (!(dev->flags & IFF_UP)) + notify_enetdown = 1; + + ifindex = dev->ifindex; + + if (do_rx_reg) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_register(net, dev, addr->can_addr.tp.rx_id, +#else + can_rx_register(dev, addr->can_addr.tp.rx_id, +#endif + SINGLE_MASK(addr->can_addr.tp.rx_id), isotp_rcv, sk, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,11)) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,50)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,49)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,42)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))) + "isotp", sk); +#else + "isotp"); +#endif + dev_put(dev); + + if (so->bound && do_rx_reg) { + /* unregister old filter */ + if (so->ifindex) { + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + /* switch to new settings */ + so->ifindex = ifindex; + so->rxid = addr->can_addr.tp.rx_id; + so->txid = addr->can_addr.tp.tx_id; + so->bound = 1; + +out: + release_sock(sk); + + if (notify_enetdown) { + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + } + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, int peer) +#else +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, + int *len, int peer) +#endif +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + + if (peer) + return -EOPNOTSUPP; + + addr->can_family = AF_CAN; + addr->can_ifindex = so->ifindex; + addr->can_addr.tp.rx_id = so->rxid; + addr->can_addr.tp.tx_id = so->txid; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) + return sizeof(*addr); +#else + *len = sizeof(*addr); + + return 0; +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,9,0) +#define copy_from_user copy_from_sockptr +static int isotp_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, unsigned int optlen) +#else +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, int optlen) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int ret = 0; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + + if (so->bound) + return -EISCONN; + + switch (optname) { + case CAN_ISOTP_OPTS: + if (optlen != sizeof(struct can_isotp_options)) + return -EINVAL; + + if (copy_from_user(&so->opt, optval, optlen)) + return -EFAULT; + + /* no separate rx_ext_address is given => use ext_address */ + if (!(so->opt.flags & CAN_ISOTP_RX_EXT_ADDR)) + so->opt.rx_ext_address = so->opt.ext_address; + break; + + case CAN_ISOTP_RECV_FC: + if (optlen != sizeof(struct can_isotp_fc_options)) + return -EINVAL; + + if (copy_from_user(&so->rxfc, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_TX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_tx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_RX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_rx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_LL_OPTS: + if (optlen == sizeof(struct can_isotp_ll_options)) { + struct can_isotp_ll_options ll; + + if (copy_from_user(&ll, optval, optlen)) + return -EFAULT; + + /* check for correct ISO 11898-1 DLC data length */ + if (ll.tx_dl != padlen(ll.tx_dl)) + return -EINVAL; + + if (ll.mtu != CAN_MTU && ll.mtu != CANFD_MTU) + return -EINVAL; + + if (ll.mtu == CAN_MTU && ll.tx_dl > CAN_MAX_DLEN) + return -EINVAL; + + memcpy(&so->ll, &ll, sizeof(ll)); + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = ll.tx_dl; + } else { + return -EINVAL; + } + break; + + default: + ret = -ENOPROTOOPT; + } + + return ret; +} + +static int isotp_getsockopt(struct socket *sock, int level, int optname, + char __user *optval, int __user *optlen) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int len; + void *val; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + switch (optname) { + case CAN_ISOTP_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_options)); + val = &so->opt; + break; + + case CAN_ISOTP_RECV_FC: + len = min_t(int, len, sizeof(struct can_isotp_fc_options)); + val = &so->rxfc; + break; + + case CAN_ISOTP_TX_STMIN: + len = min_t(int, len, sizeof(u32)); + val = &so->force_tx_stmin; + break; + + case CAN_ISOTP_RX_STMIN: + len = min_t(int, len, sizeof(u32)); + val = &so->force_rx_stmin; + break; + + case CAN_ISOTP_LL_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_ll_options)); + val = &so->ll; + break; + + default: + return -ENOPROTOOPT; + } + + if (put_user(len, optlen)) + return -EFAULT; + if (copy_to_user(optval, val, len)) + return -EFAULT; + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0) +static int isotp_notifier(struct notifier_block *nb, unsigned long msg, + void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); +#else +static int isotp_notifier(struct notifier_block *nb, + unsigned long msg, void *data) +{ + struct net_device *dev = (struct net_device *)data; +#endif + struct isotp_sock *so = container_of(nb, struct isotp_sock, notifier); + struct sock *sk = &so->sk; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + if (!net_eq(dev_net(dev), sock_net(sk))) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + if (dev_net(dev) != &init_net) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) + if (dev->nd_net != &init_net) + return NOTIFY_DONE; +#endif + + if (dev->type != ARPHRD_CAN) + return NOTIFY_DONE; + + if (so->ifindex != dev->ifindex) + return NOTIFY_DONE; + + switch (msg) { + + case NETDEV_UNREGISTER: + lock_sock(sk); + /* remove current filters & unregister */ + if (so->bound && (!(so->opt.flags & CAN_ISOTP_SF_BROADCAST))) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(dev_net(dev), dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + + so->ifindex = 0; + so->bound = 0; + release_sock(sk); + + sk->sk_err = ENODEV; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + + case NETDEV_DOWN: + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + } + + return NOTIFY_DONE; +} + +static int isotp_init(struct sock *sk) +{ + struct isotp_sock *so = isotp_sk(sk); + + so->ifindex = 0; + so->bound = 0; + + so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS; + so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rx_ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rxpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.txpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.frame_txtime = CAN_ISOTP_DEFAULT_FRAME_TXTIME; + so->rxfc.bs = CAN_ISOTP_DEFAULT_RECV_BS; + so->rxfc.stmin = CAN_ISOTP_DEFAULT_RECV_STMIN; + so->rxfc.wftmax = CAN_ISOTP_DEFAULT_RECV_WFTMAX; + so->ll.mtu = CAN_ISOTP_DEFAULT_LL_MTU; + so->ll.tx_dl = CAN_ISOTP_DEFAULT_LL_TX_DL; + so->ll.tx_flags = CAN_ISOTP_DEFAULT_LL_TX_FLAGS; + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = so->ll.tx_dl; + + so->rx.state = ISOTP_IDLE; + so->tx.state = ISOTP_IDLE; + + hrtimer_init(&so->rxtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->rxtimer.function = isotp_rx_timer_handler; + hrtimer_init(&so->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->txtimer.function = isotp_tx_timer_handler; + + tasklet_init(&so->txtsklet, isotp_tx_timer_tsklet, (unsigned long)so); + + init_waitqueue_head(&so->wait); + + so->notifier.notifier_call = isotp_notifier; + register_netdevice_notifier(&so->notifier); + + return 0; +} + +static int isotp_sock_no_ioctlcmd(struct socket *sock, unsigned int cmd, + unsigned long arg) +{ + /* no ioctls for socket layer -> hand it down to NIC layer */ + return -ENOIOCTLCMD; +} + +static const struct proto_ops isotp_ops = { + .family = PF_CAN, + .release = isotp_release, + .bind = isotp_bind, + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .accept = sock_no_accept, + .getname = isotp_getname, + .poll = datagram_poll, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,2,0) + .ioctl = isotp_sock_no_ioctlcmd, + .gettstamp = sock_gettstamp, +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,39) + .ioctl = can_ioctl, /* use can_ioctl() from af_can.c */ +#else + .ioctl = NULL, /* use can_ioctl() from af_can.c */ +#endif + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .setsockopt = isotp_setsockopt, + .getsockopt = isotp_getsockopt, + .sendmsg = isotp_sendmsg, + .recvmsg = isotp_recvmsg, + .mmap = sock_no_mmap, + .sendpage = sock_no_sendpage, +}; + +static struct proto isotp_proto __read_mostly = { + .name = "CAN_ISOTP", + .owner = THIS_MODULE, + .obj_size = sizeof(struct isotp_sock), + .init = isotp_init, +}; + +static const struct can_proto isotp_can_proto = { + .type = SOCK_DGRAM, + .protocol = CAN_ISOTP, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .capability = -1, +#endif + .ops = &isotp_ops, + .prot = &isotp_proto, +}; + +static __init int isotp_module_init(void) +{ + int err; + + printk(KERN_INFO "can: isotp protocol (rev " CAN_ISOTP_VERSION ")\n"); + + err = can_proto_register(&isotp_can_proto); + if (err < 0) + printk(KERN_ERR "can: registration of isotp protocol failed\n"); + + return err; +} + +static __exit void isotp_module_exit(void) +{ + can_proto_unregister(&isotp_can_proto); +} + +module_init(isotp_module_init); +module_exit(isotp_module_exit); From ae4b2ed070ce96d78b6c3f8eaea7ce6b18b629b4 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jul 2019 11:58:09 -0500 Subject: [PATCH 002/135] CAN_ISOTP: wire up to kconfig/makefile Signed-off-by: Robert Nelson --- net/can/Kconfig | 10 ++++++++++ net/can/Makefile | 3 +++ 2 files changed, 13 insertions(+) diff --git a/net/can/Kconfig b/net/can/Kconfig index d770427524577..9e2f196f0423e 100644 --- a/net/can/Kconfig +++ b/net/can/Kconfig @@ -55,6 +55,16 @@ config CAN_GW source "net/can/j1939/Kconfig" +config CAN_ISOTP + tristate "ISO 15765-2:2016 CAN transport protocol" + default y + ---help--- + ISO 15765-2 CAN transport protocol for protocol family CAN + + This implementation is already widely used in automotive use-cases, e.g. + for UDS based OBD diagnosis. Although some small adaptions may be applied + to make it ready for Linux Mainline. Feedback is welcome. + source "drivers/net/can/Kconfig" endif diff --git a/net/can/Makefile b/net/can/Makefile index 08bd217fc0510..cfa1024369cfb 100644 --- a/net/can/Makefile +++ b/net/can/Makefile @@ -16,4 +16,7 @@ can-bcm-y := bcm.o obj-$(CONFIG_CAN_GW) += can-gw.o can-gw-y := gw.o +obj-$(CONFIG_CAN_ISOTP) += can-isotp.o +can-isotp-y := isotp.o + obj-$(CONFIG_CAN_J1939) += j1939/ From a9d28a2145b0a5c9d5e4c4cd0e4b13ceffb264af Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:20:00 -0600 Subject: [PATCH 003/135] merge: WireGuard https://git.zx2c4.com/WireGuard/commit/edad0d6e99e5133b1e8e865d727a25fff6399cb4 Signed-off-by: Robert Nelson --- net/Kconfig | 1 + net/Makefile | 1 + net/wireguard/Kconfig | 33 + net/wireguard/Makefile | 15 + net/wireguard/allowedips.c | 381 + net/wireguard/allowedips.h | 59 + net/wireguard/compat/Makefile.include | 98 + .../compat/checksum/checksum_partial_compat.h | 208 + net/wireguard/compat/compat-asm.h | 48 + net/wireguard/compat/compat.h | 945 ++ net/wireguard/compat/dst_cache/dst_cache.c | 175 + .../compat/dst_cache/include/net/dst_cache.h | 97 + .../compat/fpu-x86/include/asm/fpu/api.h | 1 + .../include/asm/intel-family.h | 73 + net/wireguard/compat/memneq/include.h | 5 + net/wireguard/compat/memneq/memneq.c | 170 + .../compat/neon-arm/include/asm/neon.h | 7 + .../compat/ptr_ring/include/linux/ptr_ring.h | 640 ++ .../compat/simd-asm/include/asm/simd.h | 21 + .../compat/simd/include/linux/simd.h | 70 + .../compat/siphash/include/linux/siphash.h | 140 + net/wireguard/compat/siphash/siphash.c | 539 + .../udp_tunnel/include/net/udp_tunnel.h | 94 + net/wireguard/compat/udp_tunnel/udp_tunnel.c | 387 + .../udp_tunnel/udp_tunnel_partial_compat.h | 226 + net/wireguard/cookie.c | 236 + net/wireguard/cookie.h | 59 + net/wireguard/crypto/Makefile.include | 57 + net/wireguard/crypto/include/zinc/blake2s.h | 56 + net/wireguard/crypto/include/zinc/chacha20.h | 70 + .../crypto/include/zinc/chacha20poly1305.h | 50 + .../crypto/include/zinc/curve25519.h | 28 + net/wireguard/crypto/include/zinc/poly1305.h | 31 + net/wireguard/crypto/zinc.h | 15 + .../crypto/zinc/blake2s/blake2s-x86_64-glue.c | 72 + .../crypto/zinc/blake2s/blake2s-x86_64.S | 258 + net/wireguard/crypto/zinc/blake2s/blake2s.c | 276 + .../crypto/zinc/chacha20/chacha20-arm-glue.c | 98 + .../crypto/zinc/chacha20/chacha20-arm.pl | 1227 +++ .../crypto/zinc/chacha20/chacha20-arm64.pl | 1163 +++ .../crypto/zinc/chacha20/chacha20-mips-glue.c | 27 + .../crypto/zinc/chacha20/chacha20-mips.S | 424 + .../zinc/chacha20/chacha20-unrolled-arm.S | 461 + .../zinc/chacha20/chacha20-x86_64-glue.c | 105 + .../crypto/zinc/chacha20/chacha20-x86_64.pl | 4106 ++++++++ net/wireguard/crypto/zinc/chacha20/chacha20.c | 193 + net/wireguard/crypto/zinc/chacha20poly1305.c | 402 + .../zinc/curve25519/curve25519-arm-glue.c | 43 + .../crypto/zinc/curve25519/curve25519-arm.S | 2064 ++++ .../zinc/curve25519/curve25519-fiat32.c | 860 ++ .../zinc/curve25519/curve25519-hacl64.c | 784 ++ .../zinc/curve25519/curve25519-x86_64-glue.c | 50 + .../zinc/curve25519/curve25519-x86_64.c | 2366 +++++ .../crypto/zinc/curve25519/curve25519.c | 113 + .../crypto/zinc/poly1305/poly1305-arm-glue.c | 140 + .../crypto/zinc/poly1305/poly1305-arm.pl | 1276 +++ .../crypto/zinc/poly1305/poly1305-arm64.pl | 974 ++ .../crypto/zinc/poly1305/poly1305-donna32.c | 205 + .../crypto/zinc/poly1305/poly1305-donna64.c | 182 + .../crypto/zinc/poly1305/poly1305-mips-glue.c | 37 + .../crypto/zinc/poly1305/poly1305-mips.S | 407 + .../crypto/zinc/poly1305/poly1305-mips64.pl | 466 + .../zinc/poly1305/poly1305-x86_64-glue.c | 156 + .../crypto/zinc/poly1305/poly1305-x86_64.pl | 4266 ++++++++ net/wireguard/crypto/zinc/poly1305/poly1305.c | 168 + net/wireguard/crypto/zinc/selftest/blake2s.c | 2090 ++++ net/wireguard/crypto/zinc/selftest/chacha20.c | 2698 +++++ .../crypto/zinc/selftest/chacha20poly1305.c | 9076 +++++++++++++++++ .../crypto/zinc/selftest/curve25519.c | 1315 +++ net/wireguard/crypto/zinc/selftest/poly1305.c | 1107 ++ net/wireguard/crypto/zinc/selftest/run.h | 48 + net/wireguard/device.c | 470 + net/wireguard/device.h | 73 + net/wireguard/main.c | 68 + net/wireguard/messages.h | 128 + net/wireguard/netlink.c | 660 ++ net/wireguard/netlink.h | 12 + net/wireguard/noise.c | 830 ++ net/wireguard/noise.h | 137 + net/wireguard/peer.c | 240 + net/wireguard/peer.h | 83 + net/wireguard/peerlookup.c | 221 + net/wireguard/peerlookup.h | 64 + net/wireguard/queueing.c | 53 + net/wireguard/queueing.h | 197 + net/wireguard/ratelimiter.c | 235 + net/wireguard/ratelimiter.h | 19 + net/wireguard/receive.c | 605 ++ net/wireguard/selftest/allowedips.c | 683 ++ net/wireguard/selftest/counter.c | 104 + net/wireguard/selftest/ratelimiter.c | 226 + net/wireguard/send.c | 421 + net/wireguard/socket.c | 437 + net/wireguard/socket.h | 44 + net/wireguard/timers.c | 243 + net/wireguard/timers.h | 31 + net/wireguard/uapi/wireguard.h | 196 + net/wireguard/version.h | 1 + 98 files changed, 51220 insertions(+) create mode 100644 net/wireguard/Kconfig create mode 100644 net/wireguard/Makefile create mode 100644 net/wireguard/allowedips.c create mode 100644 net/wireguard/allowedips.h create mode 100644 net/wireguard/compat/Makefile.include create mode 100644 net/wireguard/compat/checksum/checksum_partial_compat.h create mode 100644 net/wireguard/compat/compat-asm.h create mode 100644 net/wireguard/compat/compat.h create mode 100644 net/wireguard/compat/dst_cache/dst_cache.c create mode 100644 net/wireguard/compat/dst_cache/include/net/dst_cache.h create mode 100644 net/wireguard/compat/fpu-x86/include/asm/fpu/api.h create mode 100644 net/wireguard/compat/intel-family-x86/include/asm/intel-family.h create mode 100644 net/wireguard/compat/memneq/include.h create mode 100644 net/wireguard/compat/memneq/memneq.c create mode 100644 net/wireguard/compat/neon-arm/include/asm/neon.h create mode 100644 net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h create mode 100644 net/wireguard/compat/simd-asm/include/asm/simd.h create mode 100644 net/wireguard/compat/simd/include/linux/simd.h create mode 100644 net/wireguard/compat/siphash/include/linux/siphash.h create mode 100644 net/wireguard/compat/siphash/siphash.c create mode 100644 net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h create mode 100644 net/wireguard/compat/udp_tunnel/udp_tunnel.c create mode 100644 net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h create mode 100644 net/wireguard/cookie.c create mode 100644 net/wireguard/cookie.h create mode 100644 net/wireguard/crypto/Makefile.include create mode 100644 net/wireguard/crypto/include/zinc/blake2s.h create mode 100644 net/wireguard/crypto/include/zinc/chacha20.h create mode 100644 net/wireguard/crypto/include/zinc/chacha20poly1305.h create mode 100644 net/wireguard/crypto/include/zinc/curve25519.h create mode 100644 net/wireguard/crypto/include/zinc/poly1305.h create mode 100644 net/wireguard/crypto/zinc.h create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-mips.S create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20.c create mode 100644 net/wireguard/crypto/zinc/chacha20poly1305.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-arm.S create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips.S create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/blake2s.c create mode 100644 net/wireguard/crypto/zinc/selftest/chacha20.c create mode 100644 net/wireguard/crypto/zinc/selftest/chacha20poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/curve25519.c create mode 100644 net/wireguard/crypto/zinc/selftest/poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/run.h create mode 100644 net/wireguard/device.c create mode 100644 net/wireguard/device.h create mode 100644 net/wireguard/main.c create mode 100644 net/wireguard/messages.h create mode 100644 net/wireguard/netlink.c create mode 100644 net/wireguard/netlink.h create mode 100644 net/wireguard/noise.c create mode 100644 net/wireguard/noise.h create mode 100644 net/wireguard/peer.c create mode 100644 net/wireguard/peer.h create mode 100644 net/wireguard/peerlookup.c create mode 100644 net/wireguard/peerlookup.h create mode 100644 net/wireguard/queueing.c create mode 100644 net/wireguard/queueing.h create mode 100644 net/wireguard/ratelimiter.c create mode 100644 net/wireguard/ratelimiter.h create mode 100644 net/wireguard/receive.c create mode 100644 net/wireguard/selftest/allowedips.c create mode 100644 net/wireguard/selftest/counter.c create mode 100644 net/wireguard/selftest/ratelimiter.c create mode 100644 net/wireguard/send.c create mode 100644 net/wireguard/socket.c create mode 100644 net/wireguard/socket.h create mode 100644 net/wireguard/timers.c create mode 100644 net/wireguard/timers.h create mode 100644 net/wireguard/uapi/wireguard.h create mode 100644 net/wireguard/version.h diff --git a/net/Kconfig b/net/Kconfig index d32d98d3e5773..92db8125600e1 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -91,6 +91,7 @@ config INET Short answer: say Y. if INET +source "net/wireguard/Kconfig" source "net/ipv4/Kconfig" source "net/ipv6/Kconfig" source "net/netlabel/Kconfig" diff --git a/net/Makefile b/net/Makefile index 1e4dfbdf5edc3..49e8e6c935d2c 100644 --- a/net/Makefile +++ b/net/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_NET) += $(tmp-y) obj-$(CONFIG_LLC) += llc/ obj-$(CONFIG_NET) += ethernet/ 802/ sched/ netlink/ bpf/ obj-$(CONFIG_NETFILTER) += netfilter/ +obj-$(CONFIG_WIREGUARD) += wireguard/ obj-$(CONFIG_INET) += ipv4/ obj-$(CONFIG_TLS) += tls/ obj-$(CONFIG_XFRM) += xfrm/ diff --git a/net/wireguard/Kconfig b/net/wireguard/Kconfig new file mode 100644 index 0000000000000..156e9dbfc0516 --- /dev/null +++ b/net/wireguard/Kconfig @@ -0,0 +1,33 @@ +config WIREGUARD + tristate "IP: WireGuard secure network tunnel" + depends on NET && INET + depends on IPV6 || !IPV6 + select NET_UDP_TUNNEL + select DST_CACHE + select CRYPTO + select CRYPTO_ALGAPI + select VFP + select VFPv3 if CPU_V7 + select NEON if CPU_V7 + select KERNEL_MODE_NEON if CPU_V7 + default m + help + WireGuard is a secure, fast, and easy to use replacement for IPsec + that uses modern cryptography and clever networking tricks. It's + designed to be fairly general purpose and abstract enough to fit most + use cases, while at the same time remaining extremely simple to + configure. See www.wireguard.com for more info. + + It's safe to say Y or M here, as the driver is very lightweight and + is only in use when an administrator chooses to add an interface. + +config WIREGUARD_DEBUG + bool "Debugging checks and verbose messages" + depends on WIREGUARD + help + This will write log messages for handshake and other events + that occur for a WireGuard interface. It will also perform some + extra validation checks and unit tests at various points. This is + only useful for debugging. + + Say N here unless you know what you're doing. diff --git a/net/wireguard/Makefile b/net/wireguard/Makefile new file mode 100644 index 0000000000000..364deb7053766 --- /dev/null +++ b/net/wireguard/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +ccflags-y := -O3 -fvisibility=hidden +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG -g +ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' +ccflags-y += -Wframe-larger-than=2048 + +wireguard-y := main.o noise.o device.o peer.o timers.o queueing.o send.o receive.o socket.o peerlookup.o allowedips.o ratelimiter.o cookie.o netlink.o + +include $(src)/crypto/Makefile.include +include $(src)/compat/Makefile.include + +obj-$(if $(KBUILD_EXTMOD),m,$(CONFIG_WIREGUARD)) := wireguard.o diff --git a/net/wireguard/allowedips.c b/net/wireguard/allowedips.c new file mode 100644 index 0000000000000..72667d5399c34 --- /dev/null +++ b/net/wireguard/allowedips.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "allowedips.h" +#include "peer.h" + +static void swap_endian(u8 *dst, const u8 *src, u8 bits) +{ + if (bits == 32) { + *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); + } else if (bits == 128) { + ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); + ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); + } +} + +static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, + u8 cidr, u8 bits) +{ + node->cidr = cidr; + node->bit_at_a = cidr / 8U; +#ifdef __LITTLE_ENDIAN + node->bit_at_a ^= (bits / 8U - 1U) % 8U; +#endif + node->bit_at_b = 7U - (cidr % 8U); + node->bitlen = bits; + memcpy(node->bits, src, bits / 8U); +} +#define CHOOSE_NODE(parent, key) \ + parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1] + +static void node_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct allowedips_node, rcu)); +} + +static void push_rcu(struct allowedips_node **stack, + struct allowedips_node __rcu *p, unsigned int *len) +{ + if (rcu_access_pointer(p)) { + WARN_ON(IS_ENABLED(DEBUG) && *len >= 128); + stack[(*len)++] = rcu_dereference_raw(p); + } +} + +static void root_free_rcu(struct rcu_head *rcu) +{ + struct allowedips_node *node, *stack[128] = { + container_of(rcu, struct allowedips_node, rcu) }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + kfree(node); + } +} + +static void root_remove_peer_lists(struct allowedips_node *root) +{ + struct allowedips_node *node, *stack[128] = { root }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + if (rcu_access_pointer(node->peer)) + list_del(&node->peer_list); + } +} + +static void walk_remove_by_peer(struct allowedips_node __rcu **top, + struct wg_peer *peer, struct mutex *lock) +{ +#define REF(p) rcu_access_pointer(p) +#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock)) +#define PUSH(p) ({ \ + WARN_ON(IS_ENABLED(DEBUG) && len >= 128); \ + stack[len++] = p; \ + }) + + struct allowedips_node __rcu **stack[128], **nptr; + struct allowedips_node *node, *prev; + unsigned int len; + + if (unlikely(!peer || !REF(*top))) + return; + + for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) { + nptr = stack[len - 1]; + node = DEREF(nptr); + if (!node) { + --len; + continue; + } + if (!prev || REF(prev->bit[0]) == node || + REF(prev->bit[1]) == node) { + if (REF(node->bit[0])) + PUSH(&node->bit[0]); + else if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else if (REF(node->bit[0]) == prev) { + if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else { + if (rcu_dereference_protected(node->peer, + lockdep_is_held(lock)) == peer) { + RCU_INIT_POINTER(node->peer, NULL); + list_del_init(&node->peer_list); + if (!node->bit[0] || !node->bit[1]) { + rcu_assign_pointer(*nptr, DEREF( + &node->bit[!REF(node->bit[0])])); + call_rcu(&node->rcu, node_free_rcu); + node = DEREF(nptr); + } + } + --len; + } + } + +#undef REF +#undef DEREF +#undef PUSH +} + +static unsigned int fls128(u64 a, u64 b) +{ + return a ? fls64(a) + 64U : fls64(b); +} + +static u8 common_bits(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + if (bits == 32) + return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); + else if (bits == 128) + return 128U - fls128( + *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], + *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); + return 0; +} + +static bool prefix_matches(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + /* This could be much faster if it actually just compared the common + * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and + * the rest, but it turns out that common_bits is already super fast on + * modern processors, even taking into account the unfortunate bswap. + * So, we just inline it like this instead. + */ + return common_bits(node, key, bits) >= node->cidr; +} + +static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, + const u8 *key) +{ + struct allowedips_node *node = trie, *found = NULL; + + while (node && prefix_matches(node, key, bits)) { + if (rcu_access_pointer(node->peer)) + found = node; + if (node->cidr == bits) + break; + node = rcu_dereference_bh(CHOOSE_NODE(node, key)); + } + return found; +} + +/* Returns a strong reference to a peer */ +static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, + const void *be_ip) +{ + /* Aligned so it can be passed to fls/fls64 */ + u8 ip[16] __aligned(__alignof(u64)); + struct allowedips_node *node; + struct wg_peer *peer = NULL; + + swap_endian(ip, be_ip, bits); + + rcu_read_lock_bh(); +retry: + node = find_node(rcu_dereference_bh(root), bits, ip); + if (node) { + peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer)); + if (!peer) + goto retry; + } + rcu_read_unlock_bh(); + return peer; +} + +static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, + u8 cidr, u8 bits, struct allowedips_node **rnode, + struct mutex *lock) +{ + struct allowedips_node *node = rcu_dereference_protected(trie, + lockdep_is_held(lock)); + struct allowedips_node *parent = NULL; + bool exact = false; + + while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) { + parent = node; + if (parent->cidr == cidr) { + exact = true; + break; + } + node = rcu_dereference_protected(CHOOSE_NODE(parent, key), + lockdep_is_held(lock)); + } + *rnode = parent; + return exact; +} + +static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + struct allowedips_node *node, *parent, *down, *newnode; + + if (unlikely(cidr > bits || !peer)) + return -EINVAL; + + if (!rcu_access_pointer(*trie)) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) + return -ENOMEM; + RCU_INIT_POINTER(node->peer, peer); + list_add_tail(&node->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(node, key, cidr, bits); + rcu_assign_pointer(*trie, node); + return 0; + } + if (node_placement(*trie, key, cidr, bits, &node, lock)) { + rcu_assign_pointer(node->peer, peer); + list_move_tail(&node->peer_list, &peer->allowedips_list); + return 0; + } + + newnode = kzalloc(sizeof(*newnode), GFP_KERNEL); + if (unlikely(!newnode)) + return -ENOMEM; + RCU_INIT_POINTER(newnode->peer, peer); + list_add_tail(&newnode->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(newnode, key, cidr, bits); + + if (!node) { + down = rcu_dereference_protected(*trie, lockdep_is_held(lock)); + } else { + down = rcu_dereference_protected(CHOOSE_NODE(node, key), + lockdep_is_held(lock)); + if (!down) { + rcu_assign_pointer(CHOOSE_NODE(node, key), newnode); + return 0; + } + } + cidr = min(cidr, common_bits(down, key, bits)); + parent = node; + + if (newnode->cidr == cidr) { + rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down); + if (!parent) + rcu_assign_pointer(*trie, newnode); + else + rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits), + newnode); + } else { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) { + kfree(newnode); + return -ENOMEM; + } + INIT_LIST_HEAD(&node->peer_list); + copy_and_assign_cidr(node, newnode->bits, cidr, bits); + + rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down); + rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode); + if (!parent) + rcu_assign_pointer(*trie, node); + else + rcu_assign_pointer(CHOOSE_NODE(parent, node->bits), + node); + } + return 0; +} + +void wg_allowedips_init(struct allowedips *table) +{ + table->root4 = table->root6 = NULL; + table->seq = 1; +} + +void wg_allowedips_free(struct allowedips *table, struct mutex *lock) +{ + struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; + + ++table->seq; + RCU_INIT_POINTER(table->root4, NULL); + RCU_INIT_POINTER(table->root6, NULL); + if (rcu_access_pointer(old4)) { + struct allowedips_node *node = rcu_dereference_protected(old4, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } + if (rcu_access_pointer(old6)) { + struct allowedips_node *node = rcu_dereference_protected(old6, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } +} + +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls */ + u8 key[4] __aligned(__alignof(u32)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 32); + return add(&table->root4, 32, key, cidr, peer, lock); +} + +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls64 */ + u8 key[16] __aligned(__alignof(u64)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 128); + return add(&table->root6, 128, key, cidr, peer, lock); +} + +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock) +{ + ++table->seq; + walk_remove_by_peer(&table->root4, peer, lock); + walk_remove_by_peer(&table->root6, peer, lock); +} + +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) +{ + const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); + swap_endian(ip, node->bits, node->bitlen); + memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); + if (node->cidr) + ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); + + *cidr = node->cidr; + return node->bitlen == 32 ? AF_INET : AF_INET6; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); + return NULL; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->saddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); + return NULL; +} + +#include "selftest/allowedips.c" diff --git a/net/wireguard/allowedips.h b/net/wireguard/allowedips.h new file mode 100644 index 0000000000000..e5c83cafcef4c --- /dev/null +++ b/net/wireguard/allowedips.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ALLOWEDIPS_H +#define _WG_ALLOWEDIPS_H + +#include +#include +#include + +struct wg_peer; + +struct allowedips_node { + struct wg_peer __rcu *peer; + struct allowedips_node __rcu *bit[2]; + /* While it may seem scandalous that we waste space for v4, + * we're alloc'ing to the nearest power of 2 anyway, so this + * doesn't actually make a difference. + */ + u8 bits[16] __aligned(__alignof(u64)); + u8 cidr, bit_at_a, bit_at_b, bitlen; + + /* Keep rarely used list at bottom to be beyond cache line. */ + union { + struct list_head peer_list; + struct rcu_head rcu; + }; +}; + +struct allowedips { + struct allowedips_node __rcu *root4; + struct allowedips_node __rcu *root6; + u64 seq; +}; + +void wg_allowedips_init(struct allowedips *table); +void wg_allowedips_free(struct allowedips *table, struct mutex *mutex); +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock); +/* The ip input pointer should be __aligned(__alignof(u64))) */ +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr); + +/* These return a strong reference to a peer: */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb); +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb); + +#ifdef DEBUG +bool wg_allowedips_selftest(void); +#endif + +#endif /* _WG_ALLOWEDIPS_H */ diff --git a/net/wireguard/compat/Makefile.include b/net/wireguard/compat/Makefile.include new file mode 100644 index 0000000000000..7bf345b5db524 --- /dev/null +++ b/net/wireguard/compat/Makefile.include @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) + +ccflags-y += -include $(kbuild-dir)/compat/compat.h +asflags-y += -include $(kbuild-dir)/compat/compat-asm.h + +ifeq ($(wildcard $(srctree)/include/linux/ptr_ring.h),) +ccflags-y += -I$(kbuild-dir)/compat/ptr_ring/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/siphash.h),) +ccflags-y += -I$(kbuild-dir)/compat/siphash/include +wireguard-y += compat/siphash/siphash.o +endif + +ifeq ($(wildcard $(srctree)/include/net/dst_cache.h),) +ccflags-y += -I$(kbuild-dir)/compat/dst_cache/include +wireguard-y += compat/dst_cache/dst_cache.o +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/intel-family.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/intel-family-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/fpu/api.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/fpu-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/simd.h)$(shell grep -s -F "generic-y += simd.h" "$(srctree)/arch/$(SRCARCH)/Makefile" "$(srctree)/arch/$(SRCARCH)/Makefile"),) +ccflags-y += -I$(kbuild-dir)/compat/simd-asm/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/simd.h),) +ccflags-y += -I$(kbuild-dir)/compat/simd/include +endif + +ifeq ($(wildcard $(srctree)/include/net/udp_tunnel.h),) +ccflags-y += -I$(kbuild-dir)/compat/udp_tunnel/include +wireguard-y += compat/udp_tunnel/udp_tunnel.o +endif + +ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h"),) +ccflags-y += -include $(kbuild-dir)/compat/memneq/include.h +wireguard-y += compat/memneq/memneq.o +endif + +ifeq ($(shell grep -s -F "addr_gen_mode" "$(srctree)/include/linux/ipv6.h"),) +ccflags-y += -DCOMPAT_CANNOT_USE_DEV_CNF +endif + +ifdef CONFIG_HZ +ifeq ($(wildcard $(srctree)/include/generated/timeconst.h),) +ccflags-y += $(shell echo 'define gcd(a,b){auto t;while(b){t=b;b=a%b;a=t;};return a;};hz=$(CONFIG_HZ);cd=gcd(hz,1000000);print "-DHZ_TO_USEC_NUM=",1000000/cd," -DHZ_TO_USEC_DEN=",hz/cd;halt;' | bc -q) +endif +endif + +ifeq ($(wildcard $(srctree)/arch/arm/include/asm/neon.h)$(CONFIG_ARM),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif +ifeq ($(wildcard $(srctree)/arch/arm64/include/asm/neon.h)$(CONFIG_ARM64),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif + +ifeq ($(CONFIG_X86_64),y) + ifeq ($(ssse3_instr),) + ssse3_instr := $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) + ccflags-y += $(ssse3_instr) + asflags-y += $(ssse3_instr) + endif + ifeq ($(avx_instr),) + avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1) + ccflags-y += $(avx_instr) + asflags-y += $(avx_instr) + endif + ifeq ($(avx2_instr),) + avx2_instr := $(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) + ccflags-y += $(avx2_instr) + asflags-y += $(avx2_instr) + endif + ifeq ($(avx512_instr),) + avx512_instr := $(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) + ccflags-y += $(avx512_instr) + asflags-y += $(avx512_instr) + endif + ifeq ($(bmi2_instr),) + bmi2_instr :=$(call as-instr,mulx %rax$(comma)%rax$(comma)%rax,-DCONFIG_AS_BMI2=1) + ccflags-y += $(bmi2_instr) + asflags-y += $(bmi2_instr) + endif + ifeq ($(adx_instr),) + adx_instr :=$(call as-instr,adcx %rax$(comma)%rax,-DCONFIG_AS_ADX=1) + ccflags-y += $(adx_instr) + asflags-y += $(adx_instr) + endif +endif diff --git a/net/wireguard/compat/checksum/checksum_partial_compat.h b/net/wireguard/compat/checksum/checksum_partial_compat.h new file mode 100644 index 0000000000000..3dfe5397a94f5 --- /dev/null +++ b/net/wireguard/compat/checksum/checksum_partial_compat.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include +#include + +#define IP6_MF 0x0001 +#define IP6_OFFSET 0xFFF8 +static inline int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, unsigned int max) +{ + if (skb_headlen(skb) >= len) + return 0; + if (max > skb->len) + max = skb->len; + if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) + return -ENOMEM; + if (skb_headlen(skb) < len) + return -EPROTO; + return 0; +} +#define MAX_IP_HDR_LEN 128 +static inline int skb_checksum_setup_ip(struct sk_buff *skb, bool recalculate) +{ + unsigned int off; + bool fragment; + int err; + fragment = false; + err = skb_maybe_pull_tail(skb, sizeof(struct iphdr), MAX_IP_HDR_LEN); + if (err < 0) + goto out; + if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) + fragment = true; + off = ip_hdrlen(skb); + err = -EPROTO; + if (fragment) + goto out; + switch (ip_hdr(skb)->protocol) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +#define MAX_IPV6_HDR_LEN 256 +#define OPT_HDR(type, skb, off) \ + (type *)(skb_network_header(skb) + (off)) +static inline int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) +{ + int err; + u8 nexthdr; + unsigned int off; + unsigned int len; + bool fragment; + bool done; + fragment = false; + done = false; + off = sizeof(struct ipv6hdr); + err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + nexthdr = ipv6_hdr(skb)->nexthdr; + len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); + while (off <= len && !done) { + switch (nexthdr) { + case IPPROTO_DSTOPTS: + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: { + struct ipv6_opt_hdr *hp; + + err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); + nexthdr = hp->nexthdr; + off += ipv6_optlen(hp); + break; + } + case IPPROTO_FRAGMENT: { + struct frag_hdr *hp; + err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct frag_hdr, skb, off); + if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) + fragment = true; + nexthdr = hp->nexthdr; + off += sizeof(struct frag_hdr); + break; + } + default: + done = true; + break; + } + } + err = -EPROTO; + if (!done || fragment) + goto out; + switch (nexthdr) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +static inline int skb_checksum_setup(struct sk_buff *skb, bool recalculate) +{ + int err; + switch (skb->protocol) { + case htons(ETH_P_IP): + err = skb_checksum_setup_ip(skb, recalculate); + break; + + case htons(ETH_P_IPV6): + err = skb_checksum_setup_ipv6(skb, recalculate); + break; + default: + err = -EPROTO; + break; + } + return err; +} diff --git a/net/wireguard/compat/compat-asm.h b/net/wireguard/compat/compat-asm.h new file mode 100644 index 0000000000000..bafd70beb68f4 --- /dev/null +++ b/net/wireguard/compat/compat-asm.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPATASM_H +#define _WG_COMPATASM_H + +#include +#include +#include + +/* PaX compatibility */ +#if defined(RAP_PLUGIN) +#undef ENTRY +#define ENTRY RAP_ENTRY +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo + .macro ret\c, reg +#if __LINUX_ARM_ARCH__ < 6 + mov\c pc, \reg +#else + .ifeqs "\reg", "lr" + bx\c \reg + .else + mov\c pc, \reg + .endif +#endif + .endm + .endr +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#include +#define lspush push +#define lspull pull +#undef push +#undef pull +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define SYM_FUNC_START ENTRY +#define SYM_FUNC_END ENDPROC +#endif + +#endif /* _WG_COMPATASM_H */ diff --git a/net/wireguard/compat/compat.h b/net/wireguard/compat/compat.h new file mode 100644 index 0000000000000..391dfd39b2ba3 --- /dev/null +++ b/net/wireguard/compat/compat.h @@ -0,0 +1,945 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPAT_H +#define _WG_COMPAT_H + +#include +#include +#include +#include + +#ifdef RHEL_MAJOR +#if RHEL_MAJOR == 7 +#define ISRHEL7 +#elif RHEL_MAJOR == 8 +#define ISRHEL8 +#endif +#endif +#ifdef UTS_UBUNTU_RELEASE_ABI +#if LINUX_VERSION_CODE == KERNEL_VERSION(3, 13, 11) +#define ISUBUNTU1404 +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#define ISUBUNTU1604 +#endif +#endif +#ifdef CONFIG_SUSE_KERNEL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#define ISOPENSUSE42 +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) +#define ISOPENSUSE15 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#error "WireGuard requires Linux >= 3.10" +#endif + +#if defined(ISRHEL7) +#include +#define headers_end headers_start +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define headers_start data +#define headers_end data +#endif + +#include +#include +#ifndef __ro_after_init +#define __ro_after_init __read_mostly +#endif + +#include +#ifndef READ_ONCE +#define READ_ONCE ACCESS_ONCE +#endif +#ifndef WRITE_ONCE +#ifdef ACCESS_ONCE_RW +#define WRITE_ONCE(p, v) (ACCESS_ONCE_RW(p) = (v)) +#else +#define WRITE_ONCE(p, v) (ACCESS_ONCE(p) = (v)) +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include "udp_tunnel/udp_tunnel_partial_compat.h" +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(DEBUG) && defined(net_dbg_ratelimited) +#undef net_dbg_ratelimited +#define net_dbg_ratelimited(fmt, ...) do { if (0) no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef RCU_LOCKDEP_WARN +#define RCU_LOCKDEP_WARN(cond, message) rcu_lockdep_assert(!(cond), message) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(ISRHEL7) +#define ipv6_dst_lookup(a, b, c, d) ipv6_dst_lookup(b, c, d) +#endif + +#if (LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 17) && LINUX_VERSION_CODE > KERNEL_VERSION(3, 19, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 27) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 40) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 54))) && !defined(ISUBUNTU1404) && (!defined(ISRHEL7) || RHEL_MINOR < 7) /* TODO: remove < 7 workaround once CentOS 7.7 comes out. */ +#include +#include +#define IP6_ECN_set_ce(a, b) IP6_ECN_set_ce(b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISRHEL7) +#include +struct ipv6_stub_type { + void *udpv6_encap_enable; + int (*ipv6_dst_lookup)(struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6); +}; +static const struct ipv6_stub_type ipv6_stub_impl = { + .udpv6_encap_enable = (void *)1, + .ipv6_dst_lookup = ip6_dst_lookup +}; +static const struct ipv6_stub_type *ipv6_stub = &ipv6_stub_impl; +#endif + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISOPENSUSE42) && !defined(ISRHEL7) +#include +static inline bool ipv6_mod_enabled(void) +{ + return ipv6_stub != NULL && ipv6_stub->udpv6_encap_enable != NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_reset_redirect(struct sk_buff *skb) +{ +#ifdef CONFIG_NET_CLS_ACT + skb->tc_verd = 0; +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#include +#include +static inline u32 __compat_get_random_u32(void) +{ + static siphash_key_t key; + static u32 counter = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + static bool has_seeded = false; + if (unlikely(!has_seeded)) { + get_random_bytes(&key, sizeof(key)); + has_seeded = true; + } +#else + get_random_once(&key, sizeof(key)); +#endif + return siphash_2u32(counter++, get_random_int(), &key); +} +#define get_random_u32 __compat_get_random_u32 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) && !defined(ISRHEL7) +static inline void netif_keep_dst(struct net_device *dev) +{ + dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; +} +#define COMPAT_CANNOT_USE_CSUM_LEVEL +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define pcpu_sw_netstats pcpu_tstats +#endif +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats alloc_percpu +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats(type) \ +({ \ + typeof(type) __percpu *pcpu_stats = alloc_percpu(type); \ + if (pcpu_stats) { \ + int __cpu; \ + for_each_possible_cpu(__cpu) { \ + typeof(type) *stat; \ + stat = per_cpu_ptr(pcpu_stats, __cpu); \ + u64_stats_init(&stat->syncp); \ + } \ + } \ + pcpu_stats; \ +}) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include "checksum/checksum_partial_compat.h" +static inline void *__compat_pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) +{ + if (tail != skb) { + skb->data_len += len; + skb->len += len; + } + return skb_put(tail, len); +} +#define pskb_put __compat_pskb_put +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_scrub_packet(struct sk_buff *skb, bool xnet) +{ +#ifdef CONFIG_CAVIUM_OCTEON_IPFWD_OFFLOAD + memset(&skb->cvm_info, 0, sizeof(skb->cvm_info)); + skb->cvm_reserved = 0; +#endif + skb->tstamp.tv64 = 0; + skb->pkt_type = PACKET_HOST; + skb->skb_iif = 0; + skb_dst_drop(skb); + secpath_reset(skb); + nf_reset(skb); + nf_reset_trace(skb); + if (!xnet) + return; + skb_orphan(skb); + skb->mark = 0; +} +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) || defined(ISUBUNTU1404)) && !defined(ISRHEL7) +#include +static inline u32 __compat_prandom_u32_max(u32 ep_ro) +{ + return (u32)(((u64)prandom_u32() * ep_ro) >> 32); +} +#define prandom_u32_max __compat_prandom_u32_max +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 75) && !defined(ISRHEL7) +#ifndef U8_MAX +#define U8_MAX ((u8)~0U) +#endif +#ifndef S8_MAX +#define S8_MAX ((s8)(U8_MAX >> 1)) +#endif +#ifndef S8_MIN +#define S8_MIN ((s8)(-S8_MAX - 1)) +#endif +#ifndef U16_MAX +#define U16_MAX ((u16)~0U) +#endif +#ifndef S16_MAX +#define S16_MAX ((s16)(U16_MAX >> 1)) +#endif +#ifndef S16_MIN +#define S16_MIN ((s16)(-S16_MAX - 1)) +#endif +#ifndef U32_MAX +#define U32_MAX ((u32)~0U) +#endif +#ifndef S32_MAX +#define S32_MAX ((s32)(U32_MAX >> 1)) +#endif +#ifndef S32_MIN +#define S32_MIN ((s32)(-S32_MAX - 1)) +#endif +#ifndef U64_MAX +#define U64_MAX ((u64)~0ULL) +#endif +#ifndef S64_MAX +#define S64_MAX ((s64)(U64_MAX >> 1)) +#endif +#ifndef S64_MIN +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 60) && !defined(ISRHEL7) +/* Making this static may very well invalidate its usefulness, + * but so it goes with compat code. */ +static inline void memzero_explicit(void *s, size_t count) +{ + memset(s, 0, count); + barrier(); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && !defined(ISRHEL7) +static const struct in6_addr __compat_in6addr_any = IN6ADDR_ANY_INIT; +#define in6addr_any __compat_in6addr_any +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISOPENSUSE15) +#include +#include +#include +struct rng_initializer { + struct completion done; + struct random_ready_callback cb; +}; +static inline void rng_initialized_callback(struct random_ready_callback *cb) +{ + complete(&container_of(cb, struct rng_initializer, cb)->done); +} +static inline int wait_for_random_bytes(void) +{ + static bool rng_is_initialized = false; + int ret; + if (unlikely(!rng_is_initialized)) { + struct rng_initializer rng = { + .done = COMPLETION_INITIALIZER(rng.done), + .cb = { .owner = THIS_MODULE, .func = rng_initialized_callback } + }; + ret = add_random_ready_callback(&rng.cb); + if (!ret) { + ret = wait_for_completion_interruptible(&rng.done); + if (ret) { + del_random_ready_callback(&rng.cb); + return ret; + } + } else if (ret != -EALREADY) + return ret; + rng_is_initialized = true; + } + return 0; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline int wait_for_random_bytes(void) +{ + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISRHEL8) +#include +#include +struct rng_is_initialized_callback { + struct random_ready_callback cb; + atomic_t *rng_state; +}; +static inline void rng_is_initialized_callback(struct random_ready_callback *cb) +{ + struct rng_is_initialized_callback *rdy = container_of(cb, struct rng_is_initialized_callback, cb); + atomic_set(rdy->rng_state, 2); + kfree(rdy); +} +static inline bool rng_is_initialized(void) +{ + static atomic_t rng_state = ATOMIC_INIT(0); + + if (atomic_read(&rng_state) == 2) + return true; + + if (atomic_cmpxchg(&rng_state, 0, 1) == 0) { + int ret; + struct rng_is_initialized_callback *rdy = kmalloc(sizeof(*rdy), GFP_ATOMIC); + if (!rdy) { + atomic_set(&rng_state, 0); + return false; + } + rdy->cb.owner = THIS_MODULE; + rdy->cb.func = rng_is_initialized_callback; + rdy->rng_state = &rng_state; + ret = add_random_ready_callback(&rdy->cb); + if (ret) + kfree(rdy); + if (ret == -EALREADY) { + atomic_set(&rng_state, 2); + return true; + } else if (ret) + atomic_set(&rng_state, 0); + return false; + } + return false; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline bool rng_is_initialized(void) +{ + return true; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +static inline int get_random_bytes_wait(void *buf, int nbytes) +{ + int ret = wait_for_random_bytes(); + if (unlikely(ret)) + return ret; + get_random_bytes(buf, nbytes); + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#define system_power_efficient_wq system_unbound_wq +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#include +#ifndef ktime_get_real_ts64 +#define timespec64 timespec +#define ktime_get_real_ts64 ktime_get_real_ts +#endif +#else +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +static inline u64 __compat_jiffies64_to_nsecs(u64 j) +{ +#if !(NSEC_PER_SEC % HZ) + return (NSEC_PER_SEC / HZ) * j; +#else + return div_u64(j * HZ_TO_USEC_NUM, HZ_TO_USEC_DEN) * 1000; +#endif +} +#define jiffies64_to_nsecs __compat_jiffies64_to_nsecs +#endif +static inline u64 ktime_get_coarse_boottime_ns(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return ktime_to_ns(ktime_get_boottime()); +#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 12) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 53) + return ktime_to_ns(ktime_mono_to_any(ns_to_ktime(jiffies64_to_nsecs(get_jiffies_64())), TK_OFFS_BOOT)); +#else + return ktime_to_ns(ktime_get_coarse_boottime()); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +static inline __be32 __compat_confirm_addr_indev(struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + int same = 0; + __be32 addr = 0; + for_ifa(in_dev) { + if (!addr && (local == ifa->ifa_local || !local) && ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + if (same) + break; + } + if (!same) { + same = (!local || inet_ifa_match(local, ifa)) && (!dst || inet_ifa_match(dst, ifa)); + if (same && addr) { + if (local || !dst) + break; + if (inet_ifa_match(addr, ifa)) + break; + if (ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + break; + } + same = 0; + } + } + } endfor_ifa(in_dev); + return same ? addr : 0; +} +static inline __be32 __compat_inet_confirm_addr(struct net *net, struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + __be32 addr = 0; + struct net_device *dev; + if (in_dev) + return __compat_confirm_addr_indev(in_dev, dst, local, scope); + rcu_read_lock(); + for_each_netdev_rcu(net, dev) { + in_dev = __in_dev_get_rcu(dev); + if (in_dev) { + addr = __compat_confirm_addr_indev(in_dev, dst, local, scope); + if (addr) + break; + } + } + rcu_read_unlock(); + return addr; +} +#define inet_confirm_addr __compat_inet_confirm_addr +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#include +static inline void *__compat_kvmalloc(size_t size, gfp_t flags) +{ + gfp_t kmalloc_flags = flags; + void *ret; + if (size > PAGE_SIZE) { + kmalloc_flags |= __GFP_NOWARN; + if (!(kmalloc_flags & __GFP_REPEAT) || (size <= PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + kmalloc_flags |= __GFP_NORETRY; + } + ret = kmalloc(size, kmalloc_flags); + if (ret || size <= PAGE_SIZE) + return ret; + return __vmalloc(size, flags, PAGE_KERNEL); +} +static inline void *__compat_kvzalloc(size_t size, gfp_t flags) +{ + return __compat_kvmalloc(size, flags | __GFP_ZERO); +} +#define kvmalloc __compat_kvmalloc +#define kvzalloc __compat_kvzalloc +#endif + +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 41)) && !defined(ISUBUNTU1404) +#include +#include +static inline void __compat_kvfree(const void *addr) +{ + if (is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} +#define kvfree __compat_kvfree +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 9) +#include +#define priv_destructor destructor +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +#define wg_newlink(a,b,c,d,e) wg_newlink(a,b,c,d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#define nlmsg_parse(a, b, c, d, e, f) nlmsg_parse(a, b, c, d, e) +#define nla_parse_nested(a, b, c, d, e) nla_parse_nested(a, b, c, d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +static inline struct nlattr **genl_family_attrbuf(const struct genl_family *family) +{ + return family->attrbuf; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define PTR_ERR_OR_ZERO(p) PTR_RET(p) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) +#include +#define nla_put_u64_64bit(a, b, c, d) nla_put_u64(a, b, c) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#include +#ifndef GENL_UNS_ADMIN_PERM +#define GENL_UNS_ADMIN_PERM GENL_ADMIN_PERM +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops, ARRAY_SIZE(genl_ops)) +#define COMPAT_CANNOT_USE_CONST_GENL_OPS +#else +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops) +#endif +#define COMPAT_CANNOT_USE_GENL_NOPS +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 2) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 16) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 65) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 101) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 84) +#define __COMPAT_NETLINK_DUMP_BLOCK { \ + int ret; \ + skb->end -= nlmsg_total_size(sizeof(int)); \ + ret = wg_get_device_dump_real(skb, cb); \ + skb->end += nlmsg_total_size(sizeof(int)); \ + return ret; \ +} +#define __COMPAT_NETLINK_DUMP_OVERRIDE +#else +#define __COMPAT_NETLINK_DUMP_BLOCK return wg_get_device_dump_real(skb, cb); +#endif +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 25) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 87) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + struct wg_device *wg = (struct wg_device *)cb->args[0]; \ + if (!wg) { \ + int ret = wg_get_device_start(cb); \ + if (ret) \ + return ret; \ + } \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#define COMPAT_CANNOT_USE_NETLINK_START +#elif defined(__COMPAT_NETLINK_DUMP_OVERRIDE) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define COMPAT_CANNOT_USE_IN6_DEV_GET +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define COMPAT_CANNOT_USE_IFF_NO_QUEUE +#endif + +#if defined(CONFIG_X86_64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +#include +#include +static inline int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) +{ + return boot_cpu_has(X86_FEATURE_XSAVE) && xgetbv(XCR_XFEATURE_ENABLED_MASK) & xfeatures_needed; +} +#endif +#ifndef XFEATURE_MASK_YMM +#define XFEATURE_MASK_YMM XSTATE_YMM +#endif +#ifndef XFEATURE_MASK_SSE +#define XFEATURE_MASK_SSE XSTATE_SSE +#endif +#ifndef XSTATE_AVX512 +#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM) +#endif +#ifndef XFEATURE_MASK_AVX512 +#define XFEATURE_MASK_AVX512 XSTATE_AVX512 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && defined(CONFIG_X86_64) +/* This is incredibly dumb and reckless, but as it turns out, there's + * not really hardware Linux runs properly on that supports F but not BW + * and VL, so in practice this isn't so bad. Plus, this is compat layer, + * so the bar remains fairly low. + */ +#include +#ifndef X86_FEATURE_AVX512BW +#define X86_FEATURE_AVX512BW X86_FEATURE_AVX512F +#endif +#ifndef X86_FEATURE_AVX512VL +#define X86_FEATURE_AVX512VL X86_FEATURE_AVX512F +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) +struct __compat_dummy_container { char dev; }; +#define netdev_notifier_info net_device *)data); __attribute((unused)) char __compat_dummy_variable = ((struct __compat_dummy_container +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#define from_timer(var, callback_timer, timer_fieldname) container_of(callback_timer, typeof(*var), timer_fieldname) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 3) +#define COMPAT_CANNOT_USE_AVX512 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define genl_dump_check_consistent(a, b) genl_dump_check_consistent(a, b, &genl_family) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISRHEL7) && !defined(ISOPENSUSE15) +static inline void *skb_put_data(struct sk_buff *skb, const void *data, unsigned int len) +{ + void *tmp = skb_put(skb, len); + memcpy(tmp, data, len); + return tmp; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) && !defined(ISRHEL7) +#define napi_complete_done(n, work_done) napi_complete(n) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#include +/* NAPI_STATE_SCHED gets set by netif_napi_add anyway, so this is safe. + * Also, kernels without NAPI_STATE_NO_BUSY_POLL don't have a call to + * napi_hash_add inside of netif_napi_add. + */ +#define NAPI_STATE_NO_BUSY_POLL NAPI_STATE_SCHED +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) ({ int __compat_p1 = atomic_read(v); smp_rmb(); __compat_p1; }) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) ({ smp_wmb(); atomic_set(v, i); }) +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0) +static inline void le32_to_cpu_array(u32 *buf, unsigned int words) +{ + while (words--) { + __le32_to_cpus(buf); + buf++; + } +} +static inline void cpu_to_le32_array(u32 *buf, unsigned int words) +{ + while (words--) { + __cpu_to_le32s(buf); + buf++; + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#include +static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2, + unsigned int size) +{ + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + __builtin_constant_p(size) && + (size % sizeof(unsigned long)) == 0) { + unsigned long *d = (unsigned long *)dst; + unsigned long *s1 = (unsigned long *)src1; + unsigned long *s2 = (unsigned long *)src2; + + while (size > 0) { + *d++ = *s1++ ^ *s2++; + size -= sizeof(unsigned long); + } + } else { + if (unlikely(dst != src1)) + memmove(dst, src1, size); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + crypto_xor(dst, src2, size); +#else + __crypto_xor(dst, src2, size); +#endif + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define read_cpuid_part() read_cpuid_part_number() +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) && !defined(ISRHEL7) +#define hlist_add_behind(a, b) hlist_add_after(b, a) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) +#define totalram_pages() totalram_pages +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0) +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#ifdef __kernel_timespec +#undef __kernel_timespec +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#define skb_probe_transport_header(a) skb_probe_transport_header(a, 0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && !defined(ISRHEL7) +#define ignore_df local_df +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +/* Note that all intentional uses of the non-_bh variety need to explicitly + * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU. + */ +#include +static __always_inline void old_synchronize_rcu(void) +{ + synchronize_rcu(); +} +static __always_inline void old_call_rcu(void *a, void *b) +{ + call_rcu(a, b); +} +static __always_inline void old_rcu_barrier(void) +{ + rcu_barrier(); +} +#ifdef synchronize_rcu +#undef synchronize_rcu +#endif +#ifdef call_rcu +#undef call_rcu +#endif +#ifdef rcu_barrier +#undef rcu_barrier +#endif +#define synchronize_rcu synchronize_rcu_bh +#define call_rcu call_rcu_bh +#define rcu_barrier rcu_barrier_bh +#define COMPAT_CANNOT_DEPRECIATE_BH_RCU +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 10) && !defined(ISRHEL8) +static inline void skb_mark_not_on_list(struct sk_buff *skb) +{ + skb->next = NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && !defined(ISRHEL8) +#define NLA_EXACT_LEN NLA_UNSPEC +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) +#define NLA_MIN_LEN NLA_UNSPEC +#define COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) && defined(__aarch64__) +#define cpu_have_named_feature(name) (elf_hwcap & (HWCAP_ ## name)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) +#include +#ifndef offsetofend +#define offsetofend(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define genl_dumpit_info(cb) ({ \ + struct { struct nlattr **attrs; } *a = (void *)((u8 *)cb->args + offsetofend(struct dump_ctx, next_allowedip)); \ + BUILD_BUG_ON(sizeof(cb->args) < offsetofend(struct dump_ctx, next_allowedip) + sizeof(*a)); \ + a->attrs = genl_family_attrbuf(&genl_family); \ + if (nlmsg_parse(cb->nlh, GENL_HDRLEN + genl_family.hdrsize, a->attrs, genl_family.maxattr, device_policy, NULL) < 0) \ + memset(a->attrs, 0, (genl_family.maxattr + 1) * sizeof(struct nlattr *)); \ + a; \ +}) +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 18) +#define ipv6_dst_lookup_flow(a, b, c, d) ipv6_dst_lookup(a, b, &dst, c) + (void *)0 ?: dst +#endif + +#if defined(ISUBUNTU1604) +#include +#ifndef _WG_LINUX_SIPHASH_H +#define hsiphash_2u32 siphash_2u32 +#define hsiphash_3u32 siphash_3u32 +#define hsiphash_key_t siphash_key_t +#endif +#endif + +#ifdef CONFIG_VE +#include +#ifdef NETIF_F_VIRTUAL +#undef NETIF_F_LLTX +#define NETIF_F_LLTX (__NETIF_F(LLTX) | __NETIF_F(VIRTUAL)) +#endif +#endif + +/* https://github.com/ClangBuiltLinux/linux/issues/7 */ +#if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000) +#include +#undef BUILD_BUG_ON +#define BUILD_BUG_ON(x) +#endif + +/* https://lkml.kernel.org/r/20170624021727.17835-1-Jason@zx2c4.com */ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#endif +static inline void new_icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb_in, &ctinfo); + if (skb_network_header(skb_in) < skb_in->head || (skb_network_header(skb_in) + sizeof(struct iphdr)) > skb_tail_pointer(skb_in)) + return; + if (ct) + ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; + icmp_send(skb_in, type, code, info); +} +static inline void new_icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb, &ctinfo); + if (skb_network_header(skb) < skb->head || (skb_network_header(skb) + sizeof(struct ipv6hdr)) > skb_tail_pointer(skb)) + return; + if (ct) + ipv6_hdr(skb)->saddr = ct->tuplehash[0].tuple.src.u3.in6; + icmpv6_send(skb, type, code, info); +} +#define icmp_send(a,b,c,d) new_icmp_send(a,b,c,d) +#define icmpv6_send(a,b,c,d) new_icmpv6_send(a,b,c,d) +#endif + +/* PaX compatibility */ +#ifdef CONSTIFY_PLUGIN +#include +#undef __read_mostly +#define __read_mostly +#endif +#if (defined(RAP_PLUGIN) || defined(CONFIG_CFI_CLANG)) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define wg_expired_retransmit_handshake(a) wg_expired_retransmit_handshake(unsigned long timer) +#define wg_expired_send_keepalive(a) wg_expired_send_keepalive(unsigned long timer) +#define wg_expired_new_handshake(a) wg_expired_new_handshake(unsigned long timer) +#define wg_expired_zero_key_material(a) wg_expired_zero_key_material(unsigned long timer) +#define wg_expired_send_persistent_keepalive(a) wg_expired_send_persistent_keepalive(unsigned long timer) +#undef timer_setup +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#undef from_timer +#define from_timer(var, callback_timer, timer_fieldname) container_of((struct timer_list *)callback_timer, typeof(*var), timer_fieldname) +#endif + +#endif /* _WG_COMPAT_H */ diff --git a/net/wireguard/compat/dst_cache/dst_cache.c b/net/wireguard/compat/dst_cache/dst_cache.c new file mode 100644 index 0000000000000..7ec22f768a8fb --- /dev/null +++ b/net/wireguard/compat/dst_cache/dst_cache.c @@ -0,0 +1,175 @@ +/* + * net/core/dst_cache.c - dst entry cache + * + * Copyright (c) 2016 Paolo Abeni + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 50) +static inline u32 rt6_get_cookie(const struct rt6_info *rt) +{ + if ((unlikely(rt->dst.flags & DST_NOCACHE) && rt->dst.from)) + rt = (struct rt6_info *)(rt->dst.from); + + return rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0; +} +#endif +#endif +#include + +struct dst_cache_pcpu { + unsigned long refresh_ts; + struct dst_entry *dst; + u32 cookie; + union { + struct in_addr in_saddr; + struct in6_addr in6_saddr; + }; +}; + +static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, + struct dst_entry *dst, u32 cookie) +{ + dst_release(dst_cache->dst); + if (dst) + dst_hold(dst); + + dst_cache->cookie = cookie; + dst_cache->dst = dst; +} + +static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, + struct dst_cache_pcpu *idst) +{ + struct dst_entry *dst; + + dst = idst->dst; + if (!dst) + goto fail; + + /* the cache already hold a dst reference; it can't go away */ + dst_hold(dst); + + if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) || + (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) { + dst_cache_per_cpu_dst_set(idst, NULL, 0); + dst_release(dst); + goto fail; + } + return dst; + +fail: + idst->refresh_ts = jiffies; + return NULL; +} + +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) +{ + if (!dst_cache->cache) + return NULL; + + return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); +} + +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in_saddr.s_addr; + return container_of(dst, struct rtable, dst); +} + +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(idst, dst, 0); + idst->in_saddr.s_addr = saddr; +} + +#if IS_ENABLED(CONFIG_IPV6) +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst, + rt6_get_cookie((struct rt6_info *)dst)); + idst->in6_saddr = *addr; +} + +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in6_saddr; + return dst; +} +#endif + +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) + BUG_ON(gfp & GFP_ATOMIC); + dst_cache->cache = alloc_percpu(struct dst_cache_pcpu); +#else + dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu, + gfp | __GFP_ZERO); +#endif + if (!dst_cache->cache) + return -ENOMEM; + + dst_cache_reset(dst_cache); + return 0; +} + +void dst_cache_destroy(struct dst_cache *dst_cache) +{ + int i; + + if (!dst_cache->cache) + return; + + for_each_possible_cpu(i) + dst_release(per_cpu_ptr(dst_cache->cache, i)->dst); + + free_percpu(dst_cache->cache); +} diff --git a/net/wireguard/compat/dst_cache/include/net/dst_cache.h b/net/wireguard/compat/dst_cache/include/net/dst_cache.h new file mode 100644 index 0000000000000..48021c0d6be14 --- /dev/null +++ b/net/wireguard/compat/dst_cache/include/net/dst_cache.h @@ -0,0 +1,97 @@ +#ifndef _WG_NET_DST_CACHE_H +#define _WG_NET_DST_CACHE_H + +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#endif + +struct dst_cache { + struct dst_cache_pcpu __percpu *cache; + unsigned long reset_ts; +}; + +/** + * dst_cache_get - perform cache lookup + * @dst_cache: the cache + * + * The caller should use dst_cache_get_ip4() if it need to retrieve the + * source address to be used when xmitting to the cached dst. + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache); + +/** + * dst_cache_get_ip4 - perform cache lookup and fetch ipv4 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr); + +/** + * dst_cache_set_ip4 - store the ipv4 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr); + +#if IS_ENABLED(CONFIG_IPV6) + +/** + * dst_cache_set_ip6 - store the ipv6 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr); + +/** + * dst_cache_get_ip6 - perform cache lookup and fetch ipv6 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr); +#endif + +/** + * dst_cache_reset - invalidate the cache contents + * @dst_cache: the cache + * + * This do not free the cached dst to avoid races and contentions. + * the dst will be freed on later cache lookup. + */ +static inline void dst_cache_reset(struct dst_cache *dst_cache) +{ + dst_cache->reset_ts = jiffies; +} + +/** + * dst_cache_init - initialize the cache, allocating the required storage + * @dst_cache: the cache + * @gfp: allocation flags + */ +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp); + +/** + * dst_cache_destroy - empty the cache and free the allocated storage + * @dst_cache: the cache + * + * No synchronization is enforced: it must be called only when the cache + * is unused. + */ +void dst_cache_destroy(struct dst_cache *dst_cache); + +#endif /* _WG_NET_DST_CACHE_H */ diff --git a/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h new file mode 100644 index 0000000000000..f3f9117bcb885 --- /dev/null +++ b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h @@ -0,0 +1 @@ +#include diff --git a/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h new file mode 100644 index 0000000000000..35a6bc4da8adf --- /dev/null +++ b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_INTEL_FAMILY_H +#define _ASM_X86_INTEL_FAMILY_H + +/* + * "Big Core" Processors (Branded as Core, Xeon, etc...) + * + * The "_X" parts are generally the EP and EX Xeons, or the + * "Extreme" ones, like Broadwell-E. + * + * Things ending in "2" are usually because we have no better + * name for them. There's no processor called "SILVERMONT2". + */ + +#define INTEL_FAM6_CORE_YONAH 0x0E + +#define INTEL_FAM6_CORE2_MEROM 0x0F +#define INTEL_FAM6_CORE2_MEROM_L 0x16 +#define INTEL_FAM6_CORE2_PENRYN 0x17 +#define INTEL_FAM6_CORE2_DUNNINGTON 0x1D + +#define INTEL_FAM6_NEHALEM 0x1E +#define INTEL_FAM6_NEHALEM_G 0x1F /* Auburndale / Havendale */ +#define INTEL_FAM6_NEHALEM_EP 0x1A +#define INTEL_FAM6_NEHALEM_EX 0x2E + +#define INTEL_FAM6_WESTMERE 0x25 +#define INTEL_FAM6_WESTMERE_EP 0x2C +#define INTEL_FAM6_WESTMERE_EX 0x2F + +#define INTEL_FAM6_SANDYBRIDGE 0x2A +#define INTEL_FAM6_SANDYBRIDGE_X 0x2D +#define INTEL_FAM6_IVYBRIDGE 0x3A +#define INTEL_FAM6_IVYBRIDGE_X 0x3E + +#define INTEL_FAM6_HASWELL_CORE 0x3C +#define INTEL_FAM6_HASWELL_X 0x3F +#define INTEL_FAM6_HASWELL_ULT 0x45 +#define INTEL_FAM6_HASWELL_GT3E 0x46 + +#define INTEL_FAM6_BROADWELL_CORE 0x3D +#define INTEL_FAM6_BROADWELL_GT3E 0x47 +#define INTEL_FAM6_BROADWELL_X 0x4F +#define INTEL_FAM6_BROADWELL_XEON_D 0x56 + +#define INTEL_FAM6_SKYLAKE_MOBILE 0x4E +#define INTEL_FAM6_SKYLAKE_DESKTOP 0x5E +#define INTEL_FAM6_SKYLAKE_X 0x55 +#define INTEL_FAM6_KABYLAKE_MOBILE 0x8E +#define INTEL_FAM6_KABYLAKE_DESKTOP 0x9E + +/* "Small Core" Processors (Atom) */ + +#define INTEL_FAM6_ATOM_PINEVIEW 0x1C +#define INTEL_FAM6_ATOM_LINCROFT 0x26 +#define INTEL_FAM6_ATOM_PENWELL 0x27 +#define INTEL_FAM6_ATOM_CLOVERVIEW 0x35 +#define INTEL_FAM6_ATOM_CEDARVIEW 0x36 +#define INTEL_FAM6_ATOM_SILVERMONT1 0x37 /* BayTrail/BYT / Valleyview */ +#define INTEL_FAM6_ATOM_SILVERMONT2 0x4D /* Avaton/Rangely */ +#define INTEL_FAM6_ATOM_AIRMONT 0x4C /* CherryTrail / Braswell */ +#define INTEL_FAM6_ATOM_MERRIFIELD 0x4A /* Tangier */ +#define INTEL_FAM6_ATOM_MOOREFIELD 0x5A /* Anniedale */ +#define INTEL_FAM6_ATOM_GOLDMONT 0x5C +#define INTEL_FAM6_ATOM_DENVERTON 0x5F /* Goldmont Microserver */ +#define INTEL_FAM6_ATOM_GEMINI_LAKE 0x7A + +/* Xeon Phi */ + +#define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ +#define INTEL_FAM6_XEON_PHI_KNM 0x85 /* Knights Mill */ + +#endif /* _ASM_X86_INTEL_FAMILY_H */ diff --git a/net/wireguard/compat/memneq/include.h b/net/wireguard/compat/memneq/include.h new file mode 100644 index 0000000000000..2d18acd9b6c87 --- /dev/null +++ b/net/wireguard/compat/memneq/include.h @@ -0,0 +1,5 @@ +extern noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size); +static inline int crypto_memneq(const void *a, const void *b, size_t size) +{ + return __crypto_memneq(a, b, size) != 0UL ? 1 : 0; +} diff --git a/net/wireguard/compat/memneq/memneq.c b/net/wireguard/compat/memneq/memneq.c new file mode 100644 index 0000000000000..1c427d4055379 --- /dev/null +++ b/net/wireguard/compat/memneq/memneq.c @@ -0,0 +1,170 @@ +/* + * Constant-time equality testing of memory regions. + * + * Authors: + * + * James Yonan + * Daniel Borkmann + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called LICENSE.GPL. + * + * BSD LICENSE + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of OpenVPN Technologies nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +/* Make the optimizer believe the variable can be manipulated arbitrarily. */ +#define COMPILER_OPTIMIZER_HIDE_VAR(var) asm("" : "=r" (var) : "0" (var)) + +#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ + +/* Generic path for arbitrary size */ +static inline unsigned long +__crypto_memneq_generic(const void *a, const void *b, size_t size) +{ + unsigned long neq = 0; + +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + while (size >= sizeof(unsigned long)) { + neq |= *(unsigned long *)a ^ *(unsigned long *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += sizeof(unsigned long); + b += sizeof(unsigned long); + size -= sizeof(unsigned long); + } +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + while (size > 0) { + neq |= *(unsigned char *)a ^ *(unsigned char *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += 1; + b += 1; + size -= 1; + } + return neq; +} + +/* Loop-free fast-path for frequently used 16-byte size */ +static inline unsigned long __crypto_memneq_16(const void *a, const void *b) +{ + unsigned long neq = 0; + +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (sizeof(unsigned long) == 8) { + neq |= *(unsigned long *)(a) ^ *(unsigned long *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned long *)(a+8) ^ *(unsigned long *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else if (sizeof(unsigned int) == 4) { + neq |= *(unsigned int *)(a) ^ *(unsigned int *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+4) ^ *(unsigned int *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+8) ^ *(unsigned int *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+12) ^ *(unsigned int *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + { + neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } + + return neq; +} + +/* Compare two areas of memory without leaking timing information, + * and with special optimizations for common sizes. Users should + * not call this function directly, but should instead use + * crypto_memneq defined in crypto/algapi.h. + */ +noinline unsigned long __crypto_memneq(const void *a, const void *b, + size_t size) +{ + switch (size) { + case 16: + return __crypto_memneq_16(a, b); + default: + return __crypto_memneq_generic(a, b, size); + } +} + +#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ diff --git a/net/wireguard/compat/neon-arm/include/asm/neon.h b/net/wireguard/compat/neon-arm/include/asm/neon.h new file mode 100644 index 0000000000000..980d831e201a4 --- /dev/null +++ b/net/wireguard/compat/neon-arm/include/asm/neon.h @@ -0,0 +1,7 @@ +#ifndef _ARCH_ARM_ASM_NEON +#define _ARCH_ARM_ASM_NEON +#define kernel_neon_begin() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#define kernel_neon_end() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#endif diff --git a/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h new file mode 100644 index 0000000000000..37b4bb2545b32 --- /dev/null +++ b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h @@ -0,0 +1,640 @@ +/* + * Definitions for the 'struct ptr_ring' datastructure. + * + * Author: + * Michael S. Tsirkin + * + * Copyright (C) 2016 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This is a limited-size FIFO maintaining pointers in FIFO order, with + * one CPU producing entries and another consuming entries from a FIFO. + * + * This implementation tries to minimize cache-contention when there is a + * single producer and a single consumer CPU. + */ + +#ifndef _LINUX_PTR_RING_H +#define _LINUX_PTR_RING_H 1 + +#ifdef __KERNEL__ +#include +#include +#include +#include +#include +#include +#include +#endif + +struct ptr_ring { + int producer ____cacheline_aligned_in_smp; + spinlock_t producer_lock; + int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */ + int consumer_tail; /* next entry to invalidate */ + spinlock_t consumer_lock; + /* Shared consumer/producer data */ + /* Read-only by both the producer and the consumer */ + int size ____cacheline_aligned_in_smp; /* max entries in queue */ + int batch; /* number of entries to consume in a batch */ + void **queue; +}; + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). If ring is ever resized, callers must hold + * producer_lock - see e.g. ptr_ring_full. Otherwise, if callers don't hold + * producer_lock, the next call to __ptr_ring_produce may fail. + */ +static inline bool __ptr_ring_full(struct ptr_ring *r) +{ + return r->queue[r->producer]; +} + +static inline bool ptr_ring_full(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_full(r); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_full_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must hold producer_lock. + */ +static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + if (unlikely(!r->size) || r->queue[r->producer]) + return -ENOSPC; + + r->queue[r->producer++] = ptr; + if (unlikely(r->producer >= r->size)) + r->producer = 0; + return 0; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * consume in interrupt or BH context, you must disable interrupts/BH when + * calling this. + */ +static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if they dereference the pointer - see e.g. PTR_RING_PEEK_CALL. + * If ring is never resized, and if the pointer is merely + * tested, there's no need to take the lock - see e.g. __ptr_ring_empty. + */ +static inline void *__ptr_ring_peek(struct ptr_ring *r) +{ + if (likely(r->size)) + return r->queue[r->consumer_head]; + return NULL; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if the ring is ever resized - see e.g. ptr_ring_empty. + */ +static inline bool __ptr_ring_empty(struct ptr_ring *r) +{ + return !__ptr_ring_peek(r); +} + +static inline bool ptr_ring_empty(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_empty(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_empty_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Must only be called after __ptr_ring_peek returned !NULL */ +static inline void __ptr_ring_discard_one(struct ptr_ring *r) +{ + /* Fundamentally, what we want to do is update consumer + * index and zero out the entry so producer can reuse it. + * Doing it naively at each consume would be as simple as: + * r->queue[r->consumer++] = NULL; + * if (unlikely(r->consumer >= r->size)) + * r->consumer = 0; + * but that is suboptimal when the ring is full as producer is writing + * out new entries in the same cache line. Defer these updates until a + * batch of entries has been consumed. + */ + int head = r->consumer_head++; + + /* Once we have processed enough entries invalidate them in + * the ring all at once so producer can reuse their space in the ring. + * We also do this when we reach end of the ring - not mandatory + * but helps keep the implementation simple. + */ + if (unlikely(r->consumer_head - r->consumer_tail >= r->batch || + r->consumer_head >= r->size)) { + /* Zero out entries in the reverse order: this way we touch the + * cache line that producer might currently be reading the last; + * producer won't make progress and touch other cache lines + * besides the first one until we write out all entries. + */ + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + } + if (unlikely(r->consumer_head >= r->size)) { + r->consumer_head = 0; + r->consumer_tail = 0; + } +} + +static inline void *__ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + ptr = __ptr_ring_peek(r); + if (ptr) + __ptr_ring_discard_one(r); + + return ptr; +} + +static inline int __ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + void *ptr; + int i; + + for (i = 0; i < n; i++) { + ptr = __ptr_ring_consume(r); + if (!ptr) + break; + array[i] = ptr; + } + + return i; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * call this in interrupt or BH context, you must disable interrupts/BH when + * producing. + */ +static inline void *ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + spin_lock(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_irq(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_irq(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_irq(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_any(struct ptr_ring *r) +{ + unsigned long flags; + void *ptr; + + spin_lock_irqsave(&r->consumer_lock, flags); + ptr = __ptr_ring_consume(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ptr; +} + +static inline void *ptr_ring_consume_bh(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_bh(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_bh(&r->consumer_lock); + + return ptr; +} + +static inline int ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_any(struct ptr_ring *r, + void **array, int n) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Cast to structure type and call a function without discarding from FIFO. + * Function must return a value. + * Callers must take consumer_lock. + */ +#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r))) + +#define PTR_RING_PEEK_CALL(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_BH(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + unsigned long __PTR_RING_PEEK_CALL_f;\ + \ + spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) +{ + return kcalloc(size, sizeof(void *), gfp); +} + +static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) +{ + r->size = size; + r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue)); + /* We need to set batch at least to 1 to make logic + * in __ptr_ring_discard_one work correctly. + * Batching too much (because ring is small) would cause a lot of + * burstiness. Needs tuning, for now disable batching. + */ + if (r->batch > r->size / 2 || !r->batch) + r->batch = 1; +} + +static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) +{ + r->queue = __ptr_ring_init_queue_alloc(size, gfp); + if (!r->queue) + return -ENOMEM; + + __ptr_ring_set_size(r, size); + r->producer = r->consumer_head = r->consumer_tail = 0; + spin_lock_init(&r->producer_lock); + spin_lock_init(&r->consumer_lock); + + return 0; +} + +/* + * Return entries into ring. Destroy entries that don't fit. + * + * Note: this is expected to be a rare slow path operation. + * + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n, + void (*destroy)(void *)) +{ + unsigned long flags; + int head; + + spin_lock_irqsave(&r->consumer_lock, flags); + spin_lock(&r->producer_lock); + + if (!r->size) + goto done; + + /* + * Clean out buffered entries (for simplicity). This way following code + * can test entries for NULL and if not assume they are valid. + */ + head = r->consumer_head - 1; + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + + /* + * Go over entries in batch, start moving head back and copy entries. + * Stop when we run into previously unconsumed entries. + */ + while (n) { + head = r->consumer_head - 1; + if (head < 0) + head = r->size - 1; + if (r->queue[head]) { + /* This batch entry will have to be destroyed. */ + goto done; + } + r->queue[head] = batch[--n]; + r->consumer_tail = r->consumer_head = head; + } + +done: + /* Destroy all entries left in the batch. */ + while (n) + destroy(batch[--n]); + spin_unlock(&r->producer_lock); + spin_unlock_irqrestore(&r->consumer_lock, flags); +} + +static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, + int size, gfp_t gfp, + void (*destroy)(void *)) +{ + int producer = 0; + void **old; + void *ptr; + + while ((ptr = __ptr_ring_consume(r))) + if (producer < size) + queue[producer++] = ptr; + else if (destroy) + destroy(ptr); + + __ptr_ring_set_size(r, size); + r->producer = producer; + r->consumer_head = 0; + r->consumer_tail = 0; + old = r->queue; + r->queue = queue; + + return old; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, + void (*destroy)(void *)) +{ + unsigned long flags; + void **queue = __ptr_ring_init_queue_alloc(size, gfp); + void **old; + + if (!queue) + return -ENOMEM; + + spin_lock_irqsave(&(r)->consumer_lock, flags); + spin_lock(&(r)->producer_lock); + + old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy); + + spin_unlock(&(r)->producer_lock); + spin_unlock_irqrestore(&(r)->consumer_lock, flags); + + kfree(old); + + return 0; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, + unsigned int nrings, + int size, + gfp_t gfp, void (*destroy)(void *)) +{ + unsigned long flags; + void ***queues; + int i; + + queues = kmalloc_array(nrings, sizeof(*queues), gfp); + if (!queues) + goto noqueues; + + for (i = 0; i < nrings; ++i) { + queues[i] = __ptr_ring_init_queue_alloc(size, gfp); + if (!queues[i]) + goto nomem; + } + + for (i = 0; i < nrings; ++i) { + spin_lock_irqsave(&(rings[i])->consumer_lock, flags); + spin_lock(&(rings[i])->producer_lock); + queues[i] = __ptr_ring_swap_queue(rings[i], queues[i], + size, gfp, destroy); + spin_unlock(&(rings[i])->producer_lock); + spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags); + } + + for (i = 0; i < nrings; ++i) + kfree(queues[i]); + + kfree(queues); + + return 0; + +nomem: + while (--i >= 0) + kfree(queues[i]); + + kfree(queues); + +noqueues: + return -ENOMEM; +} + +static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) +{ + void *ptr; + + if (destroy) + while ((ptr = ptr_ring_consume(r))) + destroy(ptr); + kfree(r->queue); +} + +#endif /* _LINUX_PTR_RING_H */ diff --git a/net/wireguard/compat/simd-asm/include/asm/simd.h b/net/wireguard/compat/simd-asm/include/asm/simd.h new file mode 100644 index 0000000000000..a975b38b55781 --- /dev/null +++ b/net/wireguard/compat/simd-asm/include/asm/simd.h @@ -0,0 +1,21 @@ +#ifndef _COMPAT_ASM_SIMD_H +#define _COMPAT_ASM_SIMD_H + +#if defined(CONFIG_X86_64) +#include +#endif + +static __must_check inline bool may_use_simd(void) +{ +#if defined(CONFIG_X86_64) + return irq_fpu_usable(); +#elif defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) + return true; +#elif defined(CONFIG_ARM) && defined(CONFIG_KERNEL_MODE_NEON) + return !in_nmi() && !in_irq() && !in_serving_softirq(); +#else + return false; +#endif +} + +#endif diff --git a/net/wireguard/compat/simd/include/linux/simd.h b/net/wireguard/compat/simd/include/linux/simd.h new file mode 100644 index 0000000000000..c75c72471e9ec --- /dev/null +++ b/net/wireguard/compat/simd/include/linux/simd.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SIMD_H +#define _WG_SIMD_H + +#include +#include +#if defined(CONFIG_X86_64) +#include +#include +#elif defined(CONFIG_KERNEL_MODE_NEON) +#include +#endif + +typedef enum { + HAVE_NO_SIMD = 1 << 0, + HAVE_FULL_SIMD = 1 << 1, + HAVE_SIMD_IN_USE = 1 << 31 +} simd_context_t; + +#define DONT_USE_SIMD ((simd_context_t []){ HAVE_NO_SIMD }) + +static inline void simd_get(simd_context_t *ctx) +{ + *ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD; +} + +static inline void simd_put(simd_context_t *ctx) +{ +#if defined(CONFIG_X86_64) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_fpu_end(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_neon_end(); +#endif + *ctx = HAVE_NO_SIMD; +} + +static inline bool simd_relax(simd_context_t *ctx) +{ +#ifdef CONFIG_PREEMPT + if ((*ctx & HAVE_SIMD_IN_USE) && need_resched()) { + simd_put(ctx); + simd_get(ctx); + return true; + } +#endif + return false; +} + +static __must_check inline bool simd_use(simd_context_t *ctx) +{ + if (!(*ctx & HAVE_FULL_SIMD)) + return false; + if (*ctx & HAVE_SIMD_IN_USE) + return true; +#if defined(CONFIG_X86_64) + kernel_fpu_begin(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + kernel_neon_begin(); +#endif + *ctx |= HAVE_SIMD_IN_USE; + return true; +} + +#endif /* _WG_SIMD_H */ diff --git a/net/wireguard/compat/siphash/include/linux/siphash.h b/net/wireguard/compat/siphash/include/linux/siphash.h new file mode 100644 index 0000000000000..1e5e337d15bf6 --- /dev/null +++ b/net/wireguard/compat/siphash/include/linux/siphash.h @@ -0,0 +1,140 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#ifndef _WG_LINUX_SIPHASH_H +#define _WG_LINUX_SIPHASH_H + +#include +#include + +#define SIPHASH_ALIGNMENT __alignof__(u64) +typedef struct { + u64 key[2]; +} siphash_key_t; + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key); +#endif + +u64 siphash_1u64(const u64 a, const siphash_key_t *key); +u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key); +u64 siphash_3u64(const u64 a, const u64 b, const u64 c, + const siphash_key_t *key); +u64 siphash_4u64(const u64 a, const u64 b, const u64 c, const u64 d, + const siphash_key_t *key); +u64 siphash_1u32(const u32 a, const siphash_key_t *key); +u64 siphash_3u32(const u32 a, const u32 b, const u32 c, + const siphash_key_t *key); + +static inline u64 siphash_2u32(const u32 a, const u32 b, + const siphash_key_t *key) +{ + return siphash_1u64((u64)b << 32 | a, key); +} +static inline u64 siphash_4u32(const u32 a, const u32 b, const u32 c, + const u32 d, const siphash_key_t *key) +{ + return siphash_2u64((u64)b << 32 | a, (u64)d << 32 | c, key); +} + + +static inline u64 ___siphash_aligned(const __le64 *data, size_t len, + const siphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return siphash_1u32(le32_to_cpup((const __le32 *)data), key); + if (__builtin_constant_p(len) && len == 8) + return siphash_1u64(le64_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 16) + return siphash_2u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 24) + return siphash_3u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 32) + return siphash_4u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), le64_to_cpu(data[3]), + key); + return __siphash_aligned(data, len, key); +} + +/** + * siphash - compute 64-bit siphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the siphash key + */ +static inline u64 siphash(const void *data, size_t len, + const siphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT)) + return __siphash_unaligned(data, len, key); +#endif + return ___siphash_aligned(data, len, key); +} + +#define HSIPHASH_ALIGNMENT __alignof__(unsigned long) +typedef struct { + unsigned long key[2]; +} hsiphash_key_t; + +u32 __hsiphash_aligned(const void *data, size_t len, + const hsiphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key); +#endif + +u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key); +u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key); +u32 hsiphash_3u32(const u32 a, const u32 b, const u32 c, + const hsiphash_key_t *key); +u32 hsiphash_4u32(const u32 a, const u32 b, const u32 c, const u32 d, + const hsiphash_key_t *key); + +static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len, + const hsiphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return hsiphash_1u32(le32_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 8) + return hsiphash_2u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 12) + return hsiphash_3u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 16) + return hsiphash_4u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), le32_to_cpu(data[3]), + key); + return __hsiphash_aligned(data, len, key); +} + +/** + * hsiphash - compute 32-bit hsiphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the hsiphash key + */ +static inline u32 hsiphash(const void *data, size_t len, + const hsiphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT)) + return __hsiphash_unaligned(data, len, key); +#endif + return ___hsiphash_aligned(data, len, key); +} + +#endif /* _WG_LINUX_SIPHASH_H */ diff --git a/net/wireguard/compat/siphash/siphash.c b/net/wireguard/compat/siphash/siphash.c new file mode 100644 index 0000000000000..58855328e6e0d --- /dev/null +++ b/net/wireguard/compat/siphash/siphash.c @@ -0,0 +1,539 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#ifdef __LITTLE_ENDIAN +#define bytemask_from_count(cnt) (~(~0ul << (cnt)*8)) +#else +#define bytemask_from_count(cnt) (~(~0ul >> (cnt)*8)) +#endif +#endif + +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 +#include +#include +#endif + +#define SIPROUND \ + do { \ + v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \ + v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \ + } while (0) + +#define PREAMBLE(len) \ + u64 v0 = 0x736f6d6570736575ULL; \ + u64 v1 = 0x646f72616e646f6dULL; \ + u64 v2 = 0x6c7967656e657261ULL; \ + u64 v3 = 0x7465646279746573ULL; \ + u64 b = ((u64)(len)) << 56; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define POSTAMBLE \ + v3 ^= b; \ + SIPROUND; \ + SIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} +#endif + +/** + * siphash_1u64 - compute 64-bit siphash PRF value of a u64 + * @first: first u64 + * @key: the siphash key + */ +u64 siphash_1u64(const u64 first, const siphash_key_t *key) +{ + PREAMBLE(8) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + POSTAMBLE +} + +/** + * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64 + * @first: first u64 + * @second: second u64 + * @key: the siphash key + */ +u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key) +{ + PREAMBLE(16) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + POSTAMBLE +} + +/** + * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @key: the siphash key + */ +u64 siphash_3u64(const u64 first, const u64 second, const u64 third, + const siphash_key_t *key) +{ + PREAMBLE(24) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + POSTAMBLE +} + +/** + * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @forth: forth u64 + * @key: the siphash key + */ +u64 siphash_4u64(const u64 first, const u64 second, const u64 third, + const u64 forth, const siphash_key_t *key) +{ + PREAMBLE(32) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + v3 ^= forth; + SIPROUND; + SIPROUND; + v0 ^= forth; + POSTAMBLE +} + +u64 siphash_1u32(const u32 first, const siphash_key_t *key) +{ + PREAMBLE(4) + b |= first; + POSTAMBLE +} + +u64 siphash_3u32(const u32 first, const u32 second, const u32 third, + const siphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + PREAMBLE(12) + v3 ^= combined; + SIPROUND; + SIPROUND; + v0 ^= combined; + b |= third; + POSTAMBLE +} + +#if BITS_PER_LONG == 64 +/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for + * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3. + */ + +#define HSIPROUND SIPROUND +#define HPREAMBLE(len) PREAMBLE(len) +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + b |= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(8) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(12) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + b |= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(16) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + combined = (u64)forth << 32 | third; + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} +#else +#define HSIPROUND \ + do { \ + v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \ + v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \ + v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \ + v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \ + } while (0) + +#define HPREAMBLE(len) \ + u32 v0 = 0; \ + u32 v1 = 0; \ + u32 v2 = 0x6c796765U; \ + u32 v3 = 0x74656462U; \ + u32 b = ((u32)(len)) << 24; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return v1 ^ v3; + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = le32_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = get_unaligned_le32(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + v3 ^= first; + HSIPROUND; + v0 ^= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + HPREAMBLE(8) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + HPREAMBLE(12) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + HPREAMBLE(16) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + v3 ^= forth; + HSIPROUND; + v0 ^= forth; + HPOSTAMBLE +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h new file mode 100644 index 0000000000000..8999527d69521 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h @@ -0,0 +1,94 @@ +#ifndef _WG_NET_UDP_TUNNEL_H +#define _WG_NET_UDP_TUNNEL_H + +#include +#include + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#endif + +struct udp_port_cfg { + u8 family; + + /* Used only for kernel-created sockets */ + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, + use_udp6_tx_checksums:1, + use_udp6_rx_checksums:1, + ipv6_v6only:1; +}; + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); +#else +static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + return 0; +} +#endif + +static inline int udp_sock_create(struct net *net, + struct udp_port_cfg *cfg, + struct socket **sockp) +{ + if (cfg->family == AF_INET) + return udp_sock_create4(net, cfg, sockp); + + if (cfg->family == AF_INET6) + return udp_sock_create6(net, cfg, sockp); + + return -EPFNOSUPPORT; +} + +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); + +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; + +/* Setup the given (UDP) sock to receive UDP encapsulated packets */ +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *sock_cfg); + +/* Transmit the skb using UDP encapsulation. */ +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck); +#endif + +void udp_tunnel_sock_release(struct socket *sock); + +#endif /* _WG_NET_UDP_TUNNEL_H */ diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel.c b/net/wireguard/compat/udp_tunnel/udp_tunnel.c new file mode 100644 index 0000000000000..ae435660931c8 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel.c @@ -0,0 +1,387 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) +#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk))) +#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr) +#endif + +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) + ,int unused_vulnerable_length_param +#endif + ) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + int err; + struct socket *sock = NULL; + struct sockaddr_in udp_addr; + + err = __sock_create(net, AF_INET, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->local_ip; + udp_addr.sin_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->peer_ip; + udp_addr.sin_port = cfg->peer_udp_port; + err = kernel_connect(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr), 0); + if (err < 0) + goto error; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + sock->sk->sk_no_check_tx = !cfg->use_udp_checksums; +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + inet_sk(sock->sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sock->sk, cfg->sk_user_data); + /* We force the cast in this awful way, due to various Android kernels + * backporting things stupidly. */ + *(void **)&sock->sk->sk_data_ready = (void *)__compat_sk_data_ready; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v4_check(int len, __be32 saddr, + __be32 daddr, __wsum base) +{ + return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_UDP, base); +} + +static void udp_set_csum(bool nocheck, struct sk_buff *skb, + __be32 saddr, __be32 daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_V4_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v4_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} + +#endif + +static void __compat_fake_destructor(struct sk_buff *skb) +{ +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static void __compat_iptunnel_xmit(struct rtable *rt, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 proto, + __u8 tos, __u8 ttl, __be16 df, bool xnet) +{ + struct iphdr *iph; + struct pcpu_tstats *tstats = this_cpu_ptr(skb->dev->tstats); + + skb_scrub_packet(skb, xnet); + + skb->rxhash = 0; + skb_dst_set(skb, &rt->dst); + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + + /* Push down and install the IP header. */ + skb_push(skb, sizeof(struct iphdr)); + skb_reset_network_header(skb); + + iph = ip_hdr(skb); + + iph->version = 4; + iph->ihl = sizeof(struct iphdr) >> 2; + iph->frag_off = df; + iph->protocol = proto; + iph->tos = tos; + iph->daddr = dst; + iph->saddr = src; + iph->ttl = ttl; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 53) + __ip_select_ident(iph, &rt->dst, (skb_shinfo(skb)->gso_segs ?: 1) - 1); +#else + __ip_select_ident(iph, skb_shinfo(skb)->gso_segs ?: 1); +#endif + + iptunnel_xmit(skb, skb->dev); + u64_stats_update_begin(&tstats->syncp); + tstats->tx_bytes -= 8; + u64_stats_update_end(&tstats->syncp); +} +#define iptunnel_xmit __compat_iptunnel_xmit +#endif + +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck) +{ + struct udphdr *uh; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + struct net_device *dev = skb->dev; + int ret; +#endif + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + + udp_set_csum(nocheck, skb, src, dst, skb->len); + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + ret = +#endif + iptunnel_xmit( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) + sk, +#endif + rt, skb, src, dst, IPPROTO_UDP, tos, ttl, df, xnet); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + if (ret) + iptunnel_xmit_stats(ret - 8, &dev->stats, dev->tstats); +#endif +} + +void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); +} + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + struct sockaddr_in6 udp6_addr; + int err; + struct socket *sock = NULL; + + err = __sock_create(net, AF_INET6, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + if (cfg->ipv6_v6only) { + int val = 1; + + err = kernel_setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + (char *) &val, sizeof(val)); + if (err < 0) + goto error; + } + + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->local_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->peer_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->peer_udp_port; + err = kernel_connect(sock, + (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr), 0); + } + if (err < 0) + goto error; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + udp_set_no_check6_tx(sock->sk, !cfg->use_udp6_tx_checksums); + udp_set_no_check6_rx(sock->sk, !cfg->use_udp6_rx_checksums); +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v6_check(int len, + const struct in6_addr *saddr, + const struct in6_addr *daddr, + __wsum base) +{ + return csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, base); +} +static void udp6_set_csum(bool nocheck, struct sk_buff *skb, + const struct in6_addr *saddr, + const struct in6_addr *daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_IPV6_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v6_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} +#endif + +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + + uh->len = htons(skb->len); + + skb_dst_set(skb, dst); + + udp6_set_csum(nocheck, skb, saddr, daddr, skb->len); + + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, label); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; + + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h new file mode 100644 index 0000000000000..0605896e902f7 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define udp_sock_create4 udp_sock_create +#define udp_sock_create6 udp_sock_create +#include +#include +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#endif +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} +static inline void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + struct sock *sk = sock->sk; + inet_sk(sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sk, cfg->sk_user_data); + sk->sk_data_ready = __compat_sk_data_ready; +} +static inline void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sk_release_kernel(sock->sk); +} +static inline int udp_tunnel_xmit_skb(struct socket *sock, struct rtable *rt, + struct sk_buff *skb, __be32 src, __be32 dst, + __u8 tos, __u8 ttl, __be16 df, __be16 src_port, + __be16 dst_port, bool xnet) +{ + struct udphdr *uh; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + udp_set_csum(sock->sk->sk_no_check_tx, skb, src, dst, skb->len); + return iptunnel_xmit(sock->sk, rt, skb, src, dst, IPPROTO_UDP, + tos, ttl, df, xnet); +} +#if IS_ENABLED(CONFIG_IPV6) +static inline int udp_tunnel6_xmit_skb(struct socket *sock, struct dst_entry *dst, + struct sk_buff *skb, struct net_device *dev, + struct in6_addr *saddr, struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be16 src_port, + __be16 dst_port) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + struct sock *sk = sock->sk; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED + | IPSKB_REROUTED); + skb_dst_set(skb, dst); + udp6_set_csum(udp_get_no_check6_tx(sk), skb, &inet6_sk(sk)->saddr, + &sk->sk_v6_daddr, skb->len); + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, htonl(0)); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +#include +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; ret__ = udp_tunnel_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, i, j, k); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, j, k); +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0) +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +#endif +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); ret__ = udp_tunnel_xmit_skb(a, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); udp_tunnel6_xmit_skb(a, c, d, e, f, g, h, j, k, l); } while(0) +#endif +#else + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); iptunnel_xmit_stats(ret__, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && IS_ENABLED(CONFIG_IPV6) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, j, k, l) +#endif + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +struct __compat_udp_port_cfg { + u8 family; + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, use_udp6_tx_checksums:1, use_udp6_rx_checksums:1, ipv6_v6only:1; +}; +static inline int __maybe_unused __compat_udp_sock_create(struct net *net, struct __compat_udp_port_cfg *cfg, struct socket **sockp) +{ + struct udp_port_cfg old_cfg = { + .family = cfg->family, + .local_ip = cfg->local_ip, +#if IS_ENABLED(CONFIG_IPV6) + .local_ip6 = cfg->local_ip6, +#endif + .peer_ip = cfg->peer_ip, +#if IS_ENABLED(CONFIG_IPV6) + .peer_ip6 = cfg->peer_ip6, +#endif + .local_udp_port = cfg->local_udp_port, + .peer_udp_port = cfg->peer_udp_port, + .use_udp_checksums = cfg->use_udp_checksums, + .use_udp6_tx_checksums = cfg->use_udp6_tx_checksums, + .use_udp6_rx_checksums = cfg->use_udp6_rx_checksums + }; + if (cfg->family == AF_INET) + return udp_sock_create4(net, &old_cfg, sockp); + +#if IS_ENABLED(CONFIG_IPV6) + if (cfg->family == AF_INET6) { + int ret; + int old_bindv6only; + struct net *nobns; + + if (cfg->ipv6_v6only) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) + nobns = &init_net; +#else + nobns = net; +#endif + /* Since udp_port_cfg only learned of ipv6_v6only in 4.3, we do this horrible + * hack here and set the sysctl variable temporarily to something that will + * set the right option for us in sock_create. It's super racey! */ + old_bindv6only = nobns->ipv6.sysctl.bindv6only; + nobns->ipv6.sysctl.bindv6only = 1; + } + ret = udp_sock_create6(net, &old_cfg, sockp); + if (cfg->ipv6_v6only) + nobns->ipv6.sysctl.bindv6only = old_bindv6only; + return ret; + } +#endif + return -EPFNOSUPPORT; +} +#define udp_port_cfg __compat_udp_port_cfg +#define udp_sock_create(a, b, c) __compat_udp_sock_create(a, b, c) +#endif diff --git a/net/wireguard/cookie.c b/net/wireguard/cookie.c new file mode 100644 index 0000000000000..8b7d1fe0cdf4b --- /dev/null +++ b/net/wireguard/cookie.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "cookie.h" +#include "peer.h" +#include "device.h" +#include "messages.h" +#include "ratelimiter.h" +#include "timers.h" + +#include +#include + +#include +#include + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg) +{ + init_rwsem(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + checker->device = wg; +} + +enum { COOKIE_KEY_LABEL_LEN = 8 }; +static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----"; +static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--"; + +static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 pubkey[NOISE_PUBLIC_KEY_LEN], + const u8 label[COOKIE_KEY_LABEL_LEN]) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); + blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); + blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN); + blake2s_final(&blake, key); +} + +/* Must hold peer->handshake.static_identity->lock */ +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker) +{ + if (likely(checker->device->static_identity.has_identity)) { + precompute_key(checker->cookie_encryption_key, + checker->device->static_identity.static_public, + cookie_key_label); + precompute_key(checker->message_mac1_key, + checker->device->static_identity.static_public, + mac1_key_label); + } else { + memset(checker->cookie_encryption_key, 0, + NOISE_SYMMETRIC_KEY_LEN); + memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN); + } +} + +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer) +{ + precompute_key(peer->latest_cookie.cookie_decryption_key, + peer->handshake.remote_static, cookie_key_label); + precompute_key(peer->latest_cookie.message_mac1_key, + peer->handshake.remote_static, mac1_key_label); +} + +void wg_cookie_init(struct cookie *cookie) +{ + memset(cookie, 0, sizeof(*cookie)); + init_rwsem(&cookie->lock); +} + +static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, + const u8 key[NOISE_SYMMETRIC_KEY_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac1); + blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); +} + +static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, + const u8 cookie[COOKIE_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac2); + blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); +} + +static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, + struct cookie_checker *checker) +{ + struct blake2s_state state; + + if (wg_birthdate_has_expired(checker->secret_birthdate, + COOKIE_SECRET_MAX_AGE)) { + down_write(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + up_write(&checker->secret_lock); + } + + down_read(&checker->secret_lock); + + blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); + if (skb->protocol == htons(ETH_P_IP)) + blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, + sizeof(struct in_addr)); + else if (skb->protocol == htons(ETH_P_IPV6)) + blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, + sizeof(struct in6_addr)); + blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); + blake2s_final(&state, cookie); + + up_read(&checker->secret_lock); +} + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie) +{ + struct message_macs *macs = (struct message_macs *) + (skb->data + skb->len - sizeof(*macs)); + enum cookie_mac_state ret; + u8 computed_mac[COOKIE_LEN]; + u8 cookie[COOKIE_LEN]; + + ret = INVALID_MAC; + compute_mac1(computed_mac, skb->data, skb->len, + checker->message_mac1_key); + if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_BUT_NO_COOKIE; + + if (!check_cookie) + goto out; + + make_cookie(cookie, skb, checker); + + compute_mac2(computed_mac, skb->data, skb->len, cookie); + if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; + if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev))) + goto out; + + ret = VALID_MAC_WITH_COOKIE; + +out: + return ret; +} + +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)message + len - sizeof(*macs)); + + down_write(&peer->latest_cookie.lock); + compute_mac1(macs->mac1, message, len, + peer->latest_cookie.message_mac1_key); + memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN); + peer->latest_cookie.have_sent_mac1 = true; + up_write(&peer->latest_cookie.lock); + + down_read(&peer->latest_cookie.lock); + if (peer->latest_cookie.is_valid && + !wg_birthdate_has_expired(peer->latest_cookie.birthdate, + COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) + compute_mac2(macs->mac2, message, len, + peer->latest_cookie.cookie); + else + memset(macs->mac2, 0, COOKIE_LEN); + up_read(&peer->latest_cookie.lock); +} + +void wg_cookie_message_create(struct message_handshake_cookie *dst, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)skb->data + skb->len - sizeof(*macs)); + u8 cookie[COOKIE_LEN]; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE); + dst->receiver_index = index; + get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN); + + make_cookie(cookie, skb, checker); + xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN, + macs->mac1, COOKIE_LEN, dst->nonce, + checker->cookie_encryption_key); +} + +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL; + u8 cookie[COOKIE_LEN]; + bool ret; + + if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable, + INDEX_HASHTABLE_HANDSHAKE | + INDEX_HASHTABLE_KEYPAIR, + src->receiver_index, &peer))) + return; + + down_read(&peer->latest_cookie.lock); + if (unlikely(!peer->latest_cookie.have_sent_mac1)) { + up_read(&peer->latest_cookie.lock); + goto out; + } + ret = xchacha20poly1305_decrypt( + cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie), + peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce, + peer->latest_cookie.cookie_decryption_key); + up_read(&peer->latest_cookie.lock); + + if (ret) { + down_write(&peer->latest_cookie.lock); + memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN); + peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns(); + peer->latest_cookie.is_valid = true; + peer->latest_cookie.have_sent_mac1 = false; + up_write(&peer->latest_cookie.lock); + } else { + net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n", + wg->dev->name); + } + +out: + wg_peer_put(peer); +} diff --git a/net/wireguard/cookie.h b/net/wireguard/cookie.h new file mode 100644 index 0000000000000..c4bd61ca03f24 --- /dev/null +++ b/net/wireguard/cookie.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COOKIE_H +#define _WG_COOKIE_H + +#include "messages.h" +#include + +struct wg_peer; + +struct cookie_checker { + u8 secret[NOISE_HASH_LEN]; + u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + u64 secret_birthdate; + struct rw_semaphore secret_lock; + struct wg_device *device; +}; + +struct cookie { + u64 birthdate; + bool is_valid; + u8 cookie[COOKIE_LEN]; + bool have_sent_mac1; + u8 last_mac1_sent[COOKIE_LEN]; + u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + struct rw_semaphore lock; +}; + +enum cookie_mac_state { + INVALID_MAC, + VALID_MAC_BUT_NO_COOKIE, + VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, + VALID_MAC_WITH_COOKIE +}; + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg); +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer); +void wg_cookie_init(struct cookie *cookie); + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie); +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer); + +void wg_cookie_message_create(struct message_handshake_cookie *src, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker); +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg); + +#endif /* _WG_COOKIE_H */ diff --git a/net/wireguard/crypto/Makefile.include b/net/wireguard/crypto/Makefile.include new file mode 100644 index 0000000000000..f2a312e96d889 --- /dev/null +++ b/net/wireguard/crypto/Makefile.include @@ -0,0 +1,57 @@ +ifeq ($(CONFIG_X86_64)$(if $(CONFIG_UML),y,n),yn) +CONFIG_ZINC_ARCH_X86_64 := y +endif +ifeq ($(CONFIG_ARM)$(if $(CONFIG_CPU_32v3),y,n),yn) +CONFIG_ZINC_ARCH_ARM := y +endif +ifeq ($(CONFIG_ARM64),y) +CONFIG_ZINC_ARCH_ARM64 := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy) +CONFIG_ZINC_ARCH_MIPS := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy) +CONFIG_ZINC_ARCH_MIPS64 := y +endif + +zinc-y += chacha20/chacha20.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o +AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots + +zinc-y += poly1305/poly1305.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o +AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots +zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o + +zinc-y += chacha20poly1305.o + +zinc-y += blake2s/blake2s.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o + +zinc-y += curve25519/curve25519.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) +targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) + +# Old kernels don't set this, which causes trouble. +.SECONDARY: + +wireguard-y += $(addprefix crypto/zinc/,$(zinc-y)) +ccflags-y += -I$(kbuild-dir)/crypto/include +ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64 +ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM +ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64 +ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS +ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64 +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST diff --git a/net/wireguard/crypto/include/zinc/blake2s.h b/net/wireguard/crypto/include/zinc/blake2s.h new file mode 100644 index 0000000000000..2ca0bc30750d9 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/blake2s.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_BLAKE2S_H +#define _ZINC_BLAKE2S_H + +#include +#include +#include + +enum blake2s_lengths { + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32 +}; + +struct blake2s_state { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +void blake2s_init(struct blake2s_state *state, const size_t outlen); +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen); +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen); +void blake2s_final(struct blake2s_state *state, u8 *out); + +static inline void blake2s(u8 *out, const u8 *in, const u8 *key, + const size_t outlen, const size_t inlen, + const size_t keylen) +{ + struct blake2s_state state; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE || + (!key && keylen))); + + if (keylen) + blake2s_init_key(&state, outlen, key, keylen); + else + blake2s_init(&state, outlen); + + blake2s_update(&state, in, inlen); + blake2s_final(&state, out); +} + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen); + +#endif /* _ZINC_BLAKE2S_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20.h b/net/wireguard/crypto/include/zinc/chacha20.h new file mode 100644 index 0000000000000..1b0083d871fb6 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20_H +#define _ZINC_CHACHA20_H + +#include +#include +#include +#include + +enum chacha20_lengths { + CHACHA20_NONCE_SIZE = 16, + CHACHA20_KEY_SIZE = 32, + CHACHA20_KEY_WORDS = CHACHA20_KEY_SIZE / sizeof(u32), + CHACHA20_BLOCK_SIZE = 64, + CHACHA20_BLOCK_WORDS = CHACHA20_BLOCK_SIZE / sizeof(u32), + HCHACHA20_NONCE_SIZE = CHACHA20_NONCE_SIZE, + HCHACHA20_KEY_SIZE = CHACHA20_KEY_SIZE +}; + +enum chacha20_constants { /* expand 32-byte k */ + CHACHA20_CONSTANT_EXPA = 0x61707865U, + CHACHA20_CONSTANT_ND_3 = 0x3320646eU, + CHACHA20_CONSTANT_2_BY = 0x79622d32U, + CHACHA20_CONSTANT_TE_K = 0x6b206574U +}; + +struct chacha20_ctx { + union { + u32 state[16]; + struct { + u32 constant[4]; + u32 key[8]; + u32 counter[4]; + }; + }; +}; + +static inline void chacha20_init(struct chacha20_ctx *ctx, + const u8 key[CHACHA20_KEY_SIZE], + const u64 nonce) +{ + ctx->constant[0] = CHACHA20_CONSTANT_EXPA; + ctx->constant[1] = CHACHA20_CONSTANT_ND_3; + ctx->constant[2] = CHACHA20_CONSTANT_2_BY; + ctx->constant[3] = CHACHA20_CONSTANT_TE_K; + ctx->key[0] = get_unaligned_le32(key + 0); + ctx->key[1] = get_unaligned_le32(key + 4); + ctx->key[2] = get_unaligned_le32(key + 8); + ctx->key[3] = get_unaligned_le32(key + 12); + ctx->key[4] = get_unaligned_le32(key + 16); + ctx->key[5] = get_unaligned_le32(key + 20); + ctx->key[6] = get_unaligned_le32(key + 24); + ctx->key[7] = get_unaligned_le32(key + 28); + ctx->counter[0] = 0; + ctx->counter[1] = 0; + ctx->counter[2] = nonce & U32_MAX; + ctx->counter[3] = nonce >> 32; +} +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context); + +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context); + +#endif /* _ZINC_CHACHA20_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20poly1305.h b/net/wireguard/crypto/include/zinc/chacha20poly1305.h new file mode 100644 index 0000000000000..e3339f02996e2 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20poly1305.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20POLY1305_H +#define _ZINC_CHACHA20POLY1305_H + +#include +#include + +struct scatterlist; + +enum chacha20poly1305_lengths { + XCHACHA20POLY1305_NONCE_SIZE = 24, + CHACHA20POLY1305_KEY_SIZE = 32, + CHACHA20POLY1305_AUTHTAG_SIZE = 16 +}; + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_encrypt_sg_inplace( + struct scatterlist *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +bool __must_check +chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_decrypt_sg_inplace( + struct scatterlist *src, size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check xchacha20poly1305_decrypt( + u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +#endif /* _ZINC_CHACHA20POLY1305_H */ diff --git a/net/wireguard/crypto/include/zinc/curve25519.h b/net/wireguard/crypto/include/zinc/curve25519.h new file mode 100644 index 0000000000000..127d8a3a1c828 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/curve25519.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CURVE25519_H +#define _ZINC_CURVE25519_H + +#include + +enum curve25519_lengths { + CURVE25519_KEY_SIZE = 32 +}; + +bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]); +bool __must_check curve25519_generate_public( + u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]); + +static inline void curve25519_clamp_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + secret[0] &= 248; + secret[31] = (secret[31] & 127) | 64; +} + +#endif /* _ZINC_CURVE25519_H */ diff --git a/net/wireguard/crypto/include/zinc/poly1305.h b/net/wireguard/crypto/include/zinc/poly1305.h new file mode 100644 index 0000000000000..8a16d19f81776 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/poly1305.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_POLY1305_H +#define _ZINC_POLY1305_H + +#include +#include + +enum poly1305_lengths { + POLY1305_BLOCK_SIZE = 16, + POLY1305_KEY_SIZE = 32, + POLY1305_MAC_SIZE = 16 +}; + +struct poly1305_ctx { + u8 opaque[24 * sizeof(u64)]; + u32 nonce[4]; + u8 data[POLY1305_BLOCK_SIZE]; + size_t num; +} __aligned(8); + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context); +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context); + +#endif /* _ZINC_POLY1305_H */ diff --git a/net/wireguard/crypto/zinc.h b/net/wireguard/crypto/zinc.h new file mode 100644 index 0000000000000..9aa1e8d59bf5e --- /dev/null +++ b/net/wireguard/crypto/zinc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ZINC_H +#define _WG_ZINC_H + +int chacha20_mod_init(void); +int poly1305_mod_init(void); +int chacha20poly1305_mod_init(void); +int blake2s_mod_init(void); +int curve25519_mod_init(void); + +#endif diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c new file mode 100644 index 0000000000000..f8cda59bf297b --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); + +static bool blake2s_use_ssse3 __ro_after_init; +static bool blake2s_use_avx512 __ro_after_init; +static bool *const blake2s_nobs[] __initconst = { &blake2s_use_ssse3, + &blake2s_use_avx512 }; + +static void __init blake2s_fpu_init(void) +{ + blake2s_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); +#ifndef COMPAT_CANNOT_USE_AVX512 + blake2s_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + simd_context_t simd_context; + bool used_arch = false; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE / BLAKE2S_BLOCK_SIZE < 8); + + simd_get(&simd_context); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !blake2s_use_ssse3 || + !simd_use(&simd_context)) + goto out; + used_arch = true; + + for (;;) { + const size_t blocks = min_t(size_t, nblocks, + PAGE_SIZE / BLAKE2S_BLOCK_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && blake2s_use_avx512) + blake2s_compress_avx512(state, block, blocks, inc); + else + blake2s_compress_ssse3(state, block, blocks, inc); + + nblocks -= blocks; + if (!nblocks) + break; + block += blocks * BLAKE2S_BLOCK_SIZE; + simd_relax(&simd_context); + } +out: + simd_put(&simd_context); + return used_arch; +} diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S new file mode 100644 index 0000000000000..24910b766bdda --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. + */ + +#include + +.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32 +.align 32 +IV: .octa 0xA54FF53A3C6EF372BB67AE856A09E667 + .octa 0x5BE0CD191F83D9AB9B05688C510E527F +.section .rodata.cst16.ROT16, "aM", @progbits, 16 +.align 16 +ROT16: .octa 0x0D0C0F0E09080B0A0504070601000302 +.section .rodata.cst16.ROR328, "aM", @progbits, 16 +.align 16 +ROR328: .octa 0x0C0F0E0D080B0A090407060500030201 +.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160 +.align 64 +SIGMA: +.byte 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.byte 14, 4, 9, 13, 10, 8, 15, 6, 5, 1, 0, 11, 3, 12, 2, 7 +.byte 11, 12, 5, 15, 8, 0, 2, 13, 9, 10, 3, 7, 4, 14, 6, 1 +.byte 7, 3, 13, 11, 9, 1, 12, 14, 15, 2, 5, 4, 8, 6, 10, 0 +.byte 9, 5, 2, 10, 0, 7, 4, 15, 3, 14, 11, 6, 13, 1, 12, 8 +.byte 2, 6, 0, 8, 12, 10, 11, 3, 1, 4, 7, 15, 9, 13, 5, 14 +.byte 12, 1, 14, 4, 5, 15, 13, 10, 8, 0, 6, 9, 11, 7, 3, 2 +.byte 13, 7, 12, 3, 11, 14, 1, 9, 2, 5, 15, 8, 10, 0, 4, 6 +.byte 6, 14, 11, 0, 15, 9, 3, 8, 10, 12, 13, 1, 5, 2, 7, 4 +.byte 10, 8, 7, 1, 2, 4, 6, 5, 13, 15, 9, 3, 0, 11, 14, 12 +#ifdef CONFIG_AS_AVX512 +.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640 +.align 64 +SIGMA2: +.long 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.long 8, 2, 13, 15, 10, 9, 12, 3, 6, 4, 0, 14, 5, 11, 1, 7 +.long 11, 13, 8, 6, 5, 10, 14, 3, 2, 4, 12, 15, 1, 0, 7, 9 +.long 11, 10, 7, 0, 8, 15, 1, 13, 3, 6, 2, 12, 4, 14, 9, 5 +.long 4, 10, 9, 14, 15, 0, 11, 8, 1, 7, 3, 13, 2, 5, 6, 12 +.long 2, 11, 4, 15, 14, 3, 10, 8, 13, 6, 5, 7, 0, 12, 1, 9 +.long 4, 8, 15, 9, 14, 11, 13, 5, 3, 2, 1, 12, 6, 10, 7, 0 +.long 6, 13, 0, 14, 12, 2, 1, 11, 15, 4, 5, 8, 7, 9, 3, 10 +.long 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 +.long 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 +#endif /* CONFIG_AS_AVX512 */ + +.text +#ifdef CONFIG_AS_SSSE3 +SYM_FUNC_START(blake2s_compress_ssse3) + testq %rdx,%rdx + je .Lendofloop + movdqu (%rdi),%xmm0 + movdqu 0x10(%rdi),%xmm1 + movdqa ROT16(%rip),%xmm12 + movdqa ROR328(%rip),%xmm13 + movdqu 0x20(%rdi),%xmm14 + movq %rcx,%xmm15 + leaq SIGMA+0xa0(%rip),%r8 + jmp .Lbeginofloop + .align 32 +.Lbeginofloop: + movdqa %xmm0,%xmm10 + movdqa %xmm1,%xmm11 + paddq %xmm15,%xmm14 + movdqa IV(%rip),%xmm2 + movdqa %xmm14,%xmm3 + pxor IV+0x10(%rip),%xmm3 + leaq SIGMA(%rip),%rcx +.Lroundloop: + movzbl (%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0x1(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x2(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x3(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + punpckldq %xmm5,%xmm4 + punpckldq %xmm7,%xmm6 + punpcklqdq %xmm6,%xmm4 + paddd %xmm4,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0x4(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x5(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x6(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0x7(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + punpckldq %xmm6,%xmm5 + punpckldq %xmm4,%xmm7 + punpcklqdq %xmm7,%xmm5 + paddd %xmm5,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x93,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x39,%xmm2,%xmm2 + movzbl 0x8(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x9(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xa(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xb(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + punpckldq %xmm7,%xmm6 + punpckldq %xmm5,%xmm4 + punpcklqdq %xmm4,%xmm6 + paddd %xmm6,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0xc(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xd(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xe(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0xf(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + punpckldq %xmm4,%xmm7 + punpckldq %xmm6,%xmm5 + punpcklqdq %xmm5,%xmm7 + paddd %xmm7,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x39,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x93,%xmm2,%xmm2 + addq $0x10,%rcx + cmpq %r8,%rcx + jnz .Lroundloop + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm1 + pxor %xmm10,%xmm0 + pxor %xmm11,%xmm1 + addq $0x40,%rsi + decq %rdx + jnz .Lbeginofloop + movdqu %xmm0,(%rdi) + movdqu %xmm1,0x10(%rdi) + movdqu %xmm14,0x20(%rdi) +.Lendofloop: + ret +SYM_FUNC_END(blake2s_compress_ssse3) +#endif /* CONFIG_AS_SSSE3 */ + +#ifdef CONFIG_AS_AVX512 +SYM_FUNC_START(blake2s_compress_avx512) + vmovdqu (%rdi),%xmm0 + vmovdqu 0x10(%rdi),%xmm1 + vmovdqu 0x20(%rdi),%xmm4 + vmovq %rcx,%xmm5 + vmovdqa IV(%rip),%xmm14 + vmovdqa IV+16(%rip),%xmm15 + jmp .Lblake2s_compress_avx512_mainloop +.align 32 +.Lblake2s_compress_avx512_mainloop: + vmovdqa %xmm0,%xmm10 + vmovdqa %xmm1,%xmm11 + vpaddq %xmm5,%xmm4,%xmm4 + vmovdqa %xmm14,%xmm2 + vpxor %xmm15,%xmm4,%xmm3 + vmovdqu (%rsi),%ymm6 + vmovdqu 0x20(%rsi),%ymm7 + addq $0x40,%rsi + leaq SIGMA2(%rip),%rax + movb $0xa,%cl +.Lblake2s_compress_avx512_roundloop: + addq $0x40,%rax + vmovdqa -0x40(%rax),%ymm8 + vmovdqa -0x20(%rax),%ymm9 + vpermi2d %ymm7,%ymm6,%ymm8 + vpermi2d %ymm7,%ymm6,%ymm9 + vmovdqa %ymm8,%ymm6 + vmovdqa %ymm9,%ymm7 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm8,%xmm8 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x93,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x39,%xmm2,%xmm2 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm9,%xmm9 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x39,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x93,%xmm2,%xmm2 + decb %cl + jne .Lblake2s_compress_avx512_roundloop + vpxor %xmm10,%xmm0,%xmm0 + vpxor %xmm11,%xmm1,%xmm1 + vpxor %xmm2,%xmm0,%xmm0 + vpxor %xmm3,%xmm1,%xmm1 + decq %rdx + jne .Lblake2s_compress_avx512_mainloop + vmovdqu %xmm0,(%rdi) + vmovdqu %xmm1,0x10(%rdi) + vmovdqu %xmm4,0x20(%rdi) + vzeroupper + retq +SYM_FUNC_END(blake2s_compress_avx512) +#endif /* CONFIG_AS_AVX512 */ diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s.c b/net/wireguard/crypto/zinc/blake2s/blake2s.c new file mode 100644 index 0000000000000..e9e0bdaffc82a --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + * + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include + +static const u32 blake2s_iv[8] = { + 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL +}; + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_set_lastblock(struct blake2s_state *state) +{ + state->f[0] = -1; +} + +static inline void blake2s_increment_counter(struct blake2s_state *state, + const u32 inc) +{ + state->t[0] += inc; + state->t[1] += (state->t[0] < inc); +} + +static inline void blake2s_init_param(struct blake2s_state *state, + const u32 param) +{ + int i; + + memset(state, 0, sizeof(*state)); + for (i = 0; i < 8; ++i) + state->h[i] = blake2s_iv[i]; + state->h[0] ^= param; +} + +void blake2s_init(struct blake2s_state *state, const size_t outlen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE)); + blake2s_init_param(state, 0x01010000 | outlen); + state->outlen = outlen; +} +EXPORT_SYMBOL(blake2s_init); + +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen) +{ + u8 block[BLAKE2S_BLOCK_SIZE] = { 0 }; + + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE || + !key || !keylen || keylen > BLAKE2S_KEY_SIZE)); + blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen); + state->outlen = outlen; + memcpy(block, key, keylen); + blake2s_update(state, block, BLAKE2S_BLOCK_SIZE); + memzero_explicit(block, BLAKE2S_BLOCK_SIZE); +} +EXPORT_SYMBOL(blake2s_init_key); + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "blake2s-x86_64-glue.c" +#else +static bool *const blake2s_nobs[] __initconst = { }; +static void __init blake2s_fpu_init(void) +{ +} +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + return false; +} +#endif + +static inline void blake2s_compress(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + WARN_ON(IS_ENABLED(DEBUG) && + (nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE)); + + if (blake2s_compress_arch(state, block, nblocks, inc)) + return; + + while (nblocks > 0) { + blake2s_increment_counter(state, inc); + memcpy(m, block, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, state->h, 32); + v[ 8] = blake2s_iv[0]; + v[ 9] = blake2s_iv[1]; + v[10] = blake2s_iv[2]; + v[11] = blake2s_iv[3]; + v[12] = blake2s_iv[4] ^ state->t[0]; + v[13] = blake2s_iv[5] ^ state->t[1]; + v[14] = blake2s_iv[6] ^ state->f[0]; + v[15] = blake2s_iv[7] ^ state->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + state->h[i] ^= v[i] ^ v[i + 8]; + + block += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(state->buf + state->buflen, in, fill); + blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); + state->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + /* Hash one less (full) block than strictly possible */ + blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(state->buf + state->buflen, in, inlen); + state->buflen += inlen; +} +EXPORT_SYMBOL(blake2s_update); + +void blake2s_final(struct blake2s_state *state, u8 *out) +{ + WARN_ON(IS_ENABLED(DEBUG) && !out); + blake2s_set_lastblock(state); + memset(state->buf + state->buflen, 0, + BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ + blake2s_compress(state, state->buf, 1, state->buflen); + cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); + memcpy(out, state->h, state->outlen); + memzero_explicit(state, sizeof(*state)); +} +EXPORT_SYMBOL(blake2s_final); + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen) +{ + struct blake2s_state state; + u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; + u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); + int i; + + if (keylen > BLAKE2S_BLOCK_SIZE) { + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, key, keylen); + blake2s_final(&state, x_key); + } else + memcpy(x_key, key, keylen); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, in, inlen); + blake2s_final(&state, i_hash); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x5c ^ 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); + blake2s_final(&state, i_hash); + + memcpy(out, i_hash, outlen); + memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); + memzero_explicit(i_hash, BLAKE2S_HASH_SIZE); +} +EXPORT_SYMBOL(blake2s_hmac); + +#include "../selftest/blake2s.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init blake2s_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + blake2s_fpu_init(); + if (!selftest_run("blake2s", blake2s_selftest, blake2s_nobs, + ARRAY_SIZE(blake2s_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c new file mode 100644 index 0000000000000..41e2e79abb2bc --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#if defined(CONFIG_ZINC_ARCH_ARM) +#include +#include +#endif + +asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]); +asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_neon __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { &chacha20_use_neon }; +static void __init chacha20_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + chacha20_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A7: + case ARM_CPU_PART_CORTEX_A5: + /* The Cortex-A7 and Cortex-A5 do not perform well with the NEON + * implementation but do incredibly with the scalar one and use + * less power. + */ + break; + default: + chacha20_use_neon = elf_hwcap & HWCAP_NEON; + } +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + for (;;) { + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon && + len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + chacha20_neon(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } else { + chacha20_arm(dst, src, len, ctx->key, ctx->counter); + ctx->counter[0] += (len + 63) / 64; + break; + } + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM)) { + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + hchacha20_arm(x, derived_key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl new file mode 100644 index 0000000000000..6785383ab7bba --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl @@ -0,0 +1,1227 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# December 2014 +# +# ChaCha20 for ARMv4. +# +# September 2018 +# +# Improve scalar performance per Eric Biggers' suggestion to eliminate +# separate rotates. This requires b[0..3] and d[0..3] to be maintained +# pre-rotated, hence odd twists prior inner loop and when accumulating +# key material. Since amount of instructions is reduced as result, even +# NEON performance is improved somewhat, most notably by ~9% on low-end +# Cortex-A5/A7. Full unroll was shown to provide even better scalar +# performance on Cortex-A5/A7, naturally at the cost of manyfold size +# increase. We let it be. Oversized code works in benchmarks, but is not +# necessarily optimal in real life, when it's likely to be out-of-cache +# upon entry and evict significant part of cache upon completion. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.4 1xNEON 3xNEON+1xIALU +# +# Cortex-A5 14.2(*)/+160% 21.8 12.9(**) +# Cortex-A8 10.2(*)/+190% 13.9 6.10 +# Cortex-A9 10.8(*)/+150% 14.3 6.50 +# Cortex-A15 11.0/+40% 16.0 4.90 +# Snapdragon S4 13.9(***)/+90% 13.6 4.90 +# +# (*) most "favourable" result for aligned data on little-endian +# processor, result for misaligned data is 10-15% lower; +# (**) pure 4xNEON [with "vertical" layout] was shown to provide ~8% +# better performance on Cortex-A5/A7, but not on others; +# (***) it's 17% slower than original, trade-off is considered +# acceptable, because of improvement on others, specifically +# +36% on Cortex-A5/A7 and +20% on Cortex-A9; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x")); +my @t=map("r$_",(8..11)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my $odd = $d0&1; +my ($xc,$xc_) = (@t[0..1]); +my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]); +my @ret; + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' are permanently allocated in registers, @x[0..7], + # while 'c's and pair of 'd's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. If you observe 'd' column, you'll + # notice that 15 and 13 are reused in next pair of rounds. + # This is why these two are chosen for offloading to memory, + # to make loads count more. + push @ret,( + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#13')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#13')", + "&eor ($xd,@x[$a0],$xd,'ror#24')", + "&eor ($xd_,@x[$a1],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b0],$xc, @x[$b0],'ror#13')", + "&eor (@x[$b1],$xc_,@x[$b1],'ror#13')", + + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#20')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#20')", + "&eor ($xd,@x[$a0],$xd,'ror#16')", + "&eor ($xd_,@x[$a1],$xd_,'ror#16')" ); + push @ret,( + "&str ($xd,'[sp,#4*(16+$d0)]')" ) if ($odd); + push @ret,( + "&add ($xc,$xc,$xd,'ror#24')" ); + push @ret,( + "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd); + push @ret,( + "&str ($xd_,'[sp,#4*(16+$d1)]')" ) if (!$odd); + push @ret,( + "&add ($xc_,$xc_,$xd_,'ror#24')" ); + push @ret,( + "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd); + push @ret,( + "&str ($xc,'[sp,#4*(16+$c0)]')", + "&eor (@x[$b0],@x[$b0],$xc,'ror#12')", + "&str ($xc_,'[sp,#4*(16+$c1)]')", + "&eor (@x[$b1],@x[$b1],$xc_,'ror#12')" ); + + $xd=@x[$d2] if (!$odd); + $xd_=@x[$d3] if ($odd); + push @ret,( + "&ldr ($xc,'[sp,#4*(16+$c2)]')", + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#13')", + "&ldr ($xc_,'[sp,#4*(16+$c3)]')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#13')", + "&eor ($xd,@x[$a2],$xd,'ror#24')", + "&eor ($xd_,@x[$a3],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b2],$xc, @x[$b2],'ror#13')", + "&eor (@x[$b3],$xc_,@x[$b3],'ror#13')", + + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#20')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#20')", + "&eor ($xd,@x[$a2],$xd,'ror#16')", + "&eor ($xd_,@x[$a3],$xd_,'ror#16')", + + "&add ($xc,$xc,$xd,'ror#24')", + "&add ($xc_,$xc_,$xd_,'ror#24')", + "&eor (@x[$b2],@x[$b2],$xc,'ror#12')", + "&eor (@x[$b3],@x[$b3],$xc_,'ror#12')" ); + + @ret; +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define ChaCha20_ctr32 chacha20_arm_cryptogams +# define ChaCha20_neon chacha20_neon +#endif + +.text +#if defined(__thumb2__) || defined(__clang__) +.syntax unified +# define ldrhsb ldrbhs +#endif +#if defined(__thumb2__) +.thumb +#else +.code 32 +#endif + +.align 5 +.Lsigma: +.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral +.Lone: +.long 1,0,0,0 +.Lrot8: +.long 0x02010003,0x06050407 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.LChaCha20_ctr32 +#else +.word -1 +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: +.LChaCha20_ctr32: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +#if __ARM_ARCH__<7 && !defined(__thumb2__) + sub r14,pc,#16 @ ChaCha20_ctr32 +#else + adr r14,.LChaCha20_ctr32 +#endif + cmp r2,#0 @ len==0? +#ifdef __thumb2__ + itt eq +#endif + addeq sp,sp,#4*3 + beq .Lno_data +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + cmp r2,#192 @ test len + bls .Lshort + ldr r4,[r14,#-24] + ldr r4,[r14,r4] +# ifdef __APPLE__ + ldr r4,[r4] +# endif + tst r4,#ARMV7_NEON + bne .LChaCha20_neon +.Lshort: +#endif + ldmia r12,{r4-r7} @ load counter and nonce + sub sp,sp,#4*(16) @ off-load area + sub r14,r14,#64 @ .Lsigma + stmdb sp!,{r4-r7} @ copy counter and nonce + ldmia r3,{r4-r11} @ load key + ldmia r14,{r0-r3} @ load sigma + stmdb sp!,{r4-r11} @ copy key + stmdb sp!,{r0-r3} @ copy sigma + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + b .Loop_outer_enter + +.align 4 +.Loop_outer: + ldmia sp,{r0-r9} @ load key material + str @t[3],[sp,#4*(32+2)] @ save len + str r12, [sp,#4*(32+1)] @ save inp + str r14, [sp,#4*(32+0)] @ save out +.Loop_outer_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop + +.align 4 +.Loop: + subs @t[3],@t[3],#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + bne .Loop + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + cmp @t[3],#64 @ done yet? +#ifdef __thumb2__ + itete lo +#endif + addlo r12,sp,#4*(0) @ shortcut or ... + ldrhs r12,[sp,#4*(32+1)] @ ... load inp + addlo r14,sp,#4*(0) @ shortcut or ... + ldrhs r14,[sp,#4*(32+0)] @ ... load out + + ldr @t[0],[sp,#4*(0)] @ load key material + ldr @t[1],[sp,#4*(1)] + +#if __ARM_ARCH__>=6 || !defined(__ARMEB__) +# if __ARM_ARCH__<7 + orr @t[2],r12,r14 + tst @t[2],#3 @ are input and output aligned? + ldr @t[2],[sp,#4*(2)] + bne .Lunaligned + cmp @t[3],#64 @ restore flags +# else + ldr @t[2],[sp,#4*(2)] +# endif + ldr @t[3],[sp,#4*(3)] + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] @ xor with input + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(4) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[1],[r14,#-12] + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @x[5],@t[1],@x[5],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] + add @t[0],sp,#4*(8) + str @x[4],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(12) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + str @x[1],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @x[5],@t[1],@x[5],ror#24 +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[4],[r14],#16 @ store output + str @x[5],[r14,#-12] +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_outer + + beq .Ldone +# if __ARM_ARCH__<7 + b .Ltail + +.align 4 +.Lunaligned: @ unaligned endian-neutral path + cmp @t[3],#64 @ restore flags +# endif +#endif +#if __ARM_ARCH__<7 + ldr @t[3],[sp,#4*(3)] +___ +for ($i=0;$i<16;$i+=4) { +my $j=$i&0x7; +my $twist=""; +if ($i==4) { $twist = ",ror#13"; } +elsif ($i==12) { $twist = ",ror#24"; } + +$code.=<<___ if ($i==4); + add @x[0],sp,#4*(16+8) +___ +$code.=<<___ if ($i==8); + ldmia @x[0],{@x[0]-@x[7]} @ load second half +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" +___ +$code.=<<___; + add @x[$j+0],@t[0],@x[$j+0]$twist @ accumulate key material +___ +$code.=<<___ if ($i==12); +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +___ +$code.=<<___; + add @x[$j+1],@t[1],@x[$j+1]$twist + add @x[$j+2],@t[2],@x[$j+2]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[0],@t[0],@t[0] @ zero or ... + ldrhsb @t[0],[r12],#16 @ ... load input + eorlo @t[1],@t[1],@t[1] + ldrhsb @t[1],[r12,#-12] + + add @x[$j+3],@t[3],@x[$j+3]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[2],@t[2],@t[2] + ldrhsb @t[2],[r12,#-8] + eorlo @t[3],@t[3],@t[3] + ldrhsb @t[3],[r12,#-4] + + eor @x[$j+0],@t[0],@x[$j+0] @ xor with input (or zero) + eor @x[$j+1],@t[1],@x[$j+1] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-15] @ load more input + ldrhsb @t[1],[r12,#-11] + eor @x[$j+2],@t[2],@x[$j+2] + strb @x[$j+0],[r14],#16 @ store output + eor @x[$j+3],@t[3],@x[$j+3] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-7] + ldrhsb @t[3],[r12,#-3] + strb @x[$j+1],[r14,#-12] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-8] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-14] @ load more input + ldrhsb @t[1],[r12,#-10] + strb @x[$j+3],[r14,#-4] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-15] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-6] + ldrhsb @t[3],[r12,#-2] + strb @x[$j+1],[r14,#-11] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-7] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-13] @ load more input + ldrhsb @t[1],[r12,#-9] + strb @x[$j+3],[r14,#-3] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-14] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-5] + ldrhsb @t[3],[r12,#-1] + strb @x[$j+1],[r14,#-10] + strb @x[$j+2],[r14,#-6] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+3],[r14,#-2] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 + strb @x[$j+0],[r14,#-13] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+1],[r14,#-9] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 + strb @x[$j+2],[r14,#-5] + strb @x[$j+3],[r14,#-1] +___ +$code.=<<___ if ($i<12); + add @t[0],sp,#4*(4+$i) + ldmia @t[0],{@t[0]-@t[3]} @ load key material +___ +} +$code.=<<___; +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + bhi .Loop_outer + + beq .Ldone +#endif + +.Ltail: + ldr r12,[sp,#4*(32+1)] @ load inp + add @t[1],sp,#4*(0) + ldr r14,[sp,#4*(32+0)] @ load out + +.Loop_tail: + ldrb @t[2],[@t[1]],#1 @ read buffer on stack + ldrb @t[3],[r12],#1 @ read input + subs @t[0],@t[0],#1 + eor @t[3],@t[3],@t[2] + strb @t[3],[r14],#1 @ store output + bne .Loop_tail + +.Ldone: + add sp,sp,#4*(32+3) +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .long 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) = + map("q$_",(0..15)); + +# This can replace vshr-by-24+vsli-by-8. It gives ~3% improvement on +# Cortex-A5/A7, but hurts Cortex-A9 by 5% and Snapdragon S4 by 14%! +sub vperm() +{ my ($dst,$src,$tbl) = @_; + $code .= " vtbl.8 $dst#lo,{$src#lo},$tbl#lo\n"; + $code .= " vtbl.8 $dst#hi,{$src#hi},$tbl#lo\n"; +} + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&vadd_i32 ($a,$a,$b)", + "&veor ($d,$d,$a)", + "&vrev32_16 ($d,$d)", # vrot ($d,16) + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,20)", + "&vsli_32 ($b,$t,12)", + + "&vadd_i32 ($a,$a,$b)", + "&veor ($t,$d,$a)", + "&vshr_u32 ($d,$t,24)", + "&vsli_32 ($d,$t,8)", + #"&vperm ($d,$t,$t3)", + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,25)", + "&vsli_32 ($b,$t,7)", + + "&vext_8 ($a,$a,$a,$odd?4:12)", + "&vext_8 ($d,$d,$d,8)", + "&vext_8 ($c,$c,$c,$odd?12:4)" + ); +} + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.arch armv7-a +.fpu neon + +# ifdef __KERNEL__ +.globl ChaCha20_neon +@ For optimal performance it's appropriate for caller to enforce +@ minimum input length, 193 bytes is suggested. +# endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +.LChaCha20_neon: + adr r14,.Lsigma + vstmdb sp!,{d8-d15} @ ABI spec says so + stmdb sp!,{r0-r3} + + vld1.32 {$b0-$c0},[r3] @ load key + ldmia r3,{r4-r11} @ load key + + sub sp,sp,#4*(16+16) + vld1.32 {$d0},[r12] @ load counter and nonce + add r12,sp,#4*8 + ldmia r14,{r0-r3} @ load sigma + vld1.32 {$a0},[r14]! @ load sigma + vld1.32 {$t0},[r14]! @ one + @ vld1.32 {$t3#lo},[r14] @ rot8 + vst1.32 {$c0-$d0},[r12] @ copy 1/2key|counter|nonce + vst1.32 {$a0-$b0},[sp] @ copy sigma|1/2key + + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + vshl.i32 $t1#lo,$t0#lo,#1 @ two + vstr $t0#lo,[sp,#4*(16+0)] + vshl.i32 $t2#lo,$t0#lo,#2 @ four + vstr $t1#lo,[sp,#4*(16+2)] + vmov $a1,$a0 + vstr $t2#lo,[sp,#4*(16+4)] + vmov $a2,$a0 + @ vstr $t3#lo,[sp,#4*(16+6)] + vmov $b1,$b0 + vmov $b2,$b0 + b .Loop_neon_enter + +.align 4 +.Loop_neon_outer: + ldmia sp,{r0-r9} @ load key material + cmp @t[3],#64*2 @ if len<=64*2 + bls .Lbreak_neon @ switch to integer-only + @ vldr $t3#lo,[sp,#4*(16+6)] @ rot8 + vmov $a1,$a0 + str @t[3],[sp,#4*(32+2)] @ save len + vmov $a2,$a0 + str r12, [sp,#4*(32+1)] @ save inp + vmov $b1,$b0 + str r14, [sp,#4*(32+0)] @ save out + vmov $b2,$b0 +.Loop_neon_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + vadd.i32 $d1,$d0,$t0 @ counter+1 + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + vmov $c1,$c0 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + vmov $c2,$c0 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + vadd.i32 $d2,$d1,$t0 @ counter+2 + add @x[12],@x[12],#3 @ counter+3 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop_neon + +.align 4 +.Loop_neon: + subs @t[3],@t[3],#1 +___ + my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0); + my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0); + my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1); + @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1); + @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + bne .Loop_neon + + add @t[3],sp,#32 + vld1.32 {$t0-$t1},[sp] @ load key material + vld1.32 {$t2-$t3},[@t[3]] + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + ldr r12,[sp,#4*(32+1)] @ load inp + ldr r14,[sp,#4*(32+0)] @ load out + + vadd.i32 $a0,$a0,$t0 @ accumulate key material + vadd.i32 $a1,$a1,$t0 + vadd.i32 $a2,$a2,$t0 + vldr $t0#lo,[sp,#4*(16+0)] @ one + + vadd.i32 $b0,$b0,$t1 + vadd.i32 $b1,$b1,$t1 + vadd.i32 $b2,$b2,$t1 + vldr $t1#lo,[sp,#4*(16+2)] @ two + + vadd.i32 $c0,$c0,$t2 + vadd.i32 $c1,$c1,$t2 + vadd.i32 $c2,$c2,$t2 + vadd.i32 $d1#lo,$d1#lo,$t0#lo @ counter+1 + vadd.i32 $d2#lo,$d2#lo,$t1#lo @ counter+2 + + vadd.i32 $d0,$d0,$t3 + vadd.i32 $d1,$d1,$t3 + vadd.i32 $d2,$d2,$t3 + + cmp @t[3],#64*4 + blo .Ltail_neon + + vld1.8 {$t0-$t1},[r12]! @ load input + mov @t[3],sp + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 @ xor with input + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + vst1.8 {$a0-$b0},[r14]! @ store output + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vld1.32 {$a0-$b0},[@t[3]]! @ load for next iteration + veor $t0#hi,$t0#hi,$t0#hi + vldr $t0#lo,[sp,#4*(16+4)] @ four + veor $b2,$b2,$t1 + vld1.32 {$c0-$d0},[@t[3]] + veor $c2,$c2,$t2 + vst1.8 {$a1-$b1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$c1-$d1},[r14]! + + vadd.i32 $d0#lo,$d0#lo,$t0#lo @ next counter value + vldr $t0#lo,[sp,#4*(16+0)] @ one + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + vst1.8 {$a2-$b2},[r14]! + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] + vst1.8 {$c2-$d2},[r14]! + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] @ xor with input + add @t[0],sp,#4*(4) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[5],@t[1],@x[5],ror#13 + ldr @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + ldr @t[2],[r12,#-8] + add @x[7],@t[3],@x[7],ror#13 + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] + add @t[0],sp,#4*(8) + eor @x[5],@x[5],@t[1] + str @x[4],[r14],#16 @ store output + eor @x[6],@x[6],@t[2] + str @x[5],[r14,#-12] + eor @x[7],@x[7],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] +# ifdef __thumb2__ + it hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] +# ifdef __thumb2__ + it hi +# endif + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] + add @t[0],sp,#4*(12) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],@t[0],#4 @ next counter value + add @x[5],@t[1],@x[5],ror#24 + str @t[0],[sp,#4*(12)] @ save next counter value + ldr @t[0],[r12],#16 @ load input + add @x[6],@t[2],@x[6],ror#24 + add @x[4],@x[4],#3 @ counter+3 + ldr @t[1],[r12,#-12] + add @x[7],@t[3],@x[7],ror#24 + ldr @t[2],[r12,#-8] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] +# ifdef __thumb2__ + it hi +# endif + ldrhi @t[0],[sp,#4*(32+2)] @ re-load len + eor @x[5],@x[5],@t[1] + eor @x[6],@x[6],@t[2] + str @x[4],[r14],#16 @ store output + eor @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + sub @t[3],@t[0],#64*4 @ len-=64*4 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_neon_outer + + b .Ldone_neon + +.align 4 +.Lbreak_neon: + @ harmonize NEON and integer-only stack frames: load data + @ from NEON frame, but save to integer-only one; distance + @ between the two is 4*(32+4+16-32)=4*(20). + + str @t[3], [sp,#4*(20+32+2)] @ save len + add @t[3],sp,#4*(32+4) + str r12, [sp,#4*(20+32+1)] @ save inp + str r14, [sp,#4*(20+32+0)] @ save out + + ldr @x[12],[sp,#4*(16+10)] + ldr @x[14],[sp,#4*(16+11)] + vldmia @t[3],{d8-d15} @ fulfill ABI requirement + str @x[12],[sp,#4*(20+16+10)] @ copy "@x[10]" + str @x[14],[sp,#4*(20+16+11)] @ copy "@x[11]" + + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(20+16+15)] + add @t[3],sp,#4*(20) + vst1.32 {$a0-$b0},[@t[3]]! @ copy key + add sp,sp,#4*(20) @ switch frame + vst1.32 {$c0-$d0},[@t[3]] + mov @t[3],#10 + b .Loop @ go integer-only + +.align 4 +.Ltail_neon: + cmp @t[3],#64*3 + bhs .L192_or_more_neon + cmp @t[3],#64*2 + bhs .L128_or_more_neon + cmp @t[3],#64*1 + bhs .L64_or_more_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a0-$b0},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c0-$d0},[@t[0]] + b .Loop_tail_neon + +.align 4 +.L64_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vst1.8 {$a0-$b0},[r14]! + vst1.8 {$c0-$d0},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a1-$b1},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c1-$d1},[@t[0]] + sub @t[3],@t[3],#64*1 @ len-=64*1 + b .Loop_tail_neon + +.align 4 +.L128_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vst1.8 {$a0-$b0},[r14]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vst1.8 {$a1-$b1},[r14]! + vst1.8 {$c1-$d1},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a2-$b2},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c2-$d2},[@t[0]] + sub @t[3],@t[3],#64*2 @ len-=64*2 + b .Loop_tail_neon + +.align 4 +.L192_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$a0-$b0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vst1.8 {$c0-$d0},[r14]! + veor $b2,$b2,$t1 + vst1.8 {$a1-$b1},[r14]! + veor $c2,$c2,$t2 + vst1.8 {$c1-$d1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$a2-$b2},[r14]! + vst1.8 {$c2-$d2},[r14]! + + beq .Ldone_neon + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(4) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#13 + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 + ldmia @t[0],{@t[0]-@t[3]} @ load key material +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia sp,{@x[0]-@x[7]} + add @x[0],sp,#4*(16+8) + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(12) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#24 + add @x[4],@x[4],#3 @ counter+3 + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 + ldr @t[3],[sp,#4*(32+2)] @ re-load len +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia @t[0],{@x[0]-@x[7]} + add @t[2],sp,#4*(0) + sub @t[3],@t[3],#64*3 @ len-=64*3 + +.Loop_tail_neon: + ldrb @t[0],[@t[2]],#1 @ read buffer on stack + ldrb @t[1],[r12],#1 @ read input + subs @t[3],@t[3],#1 + eor @t[0],@t[0],@t[1] + strb @t[0],[r14],#1 @ store output + bne .Loop_tail_neon + +.Ldone_neon: + add sp,sp,#4*(32+4) + vldmia sp,{d8-d15} + add sp,sp,#4*(16+3) + ldmia sp!,{r4-r11,pc} +.size ChaCha20_neon,.-ChaCha20_neon +# ifndef __KERNEL__ +.comm OPENSSL_armcap_P,4,4 +# endif +#endif +___ +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl new file mode 100644 index 0000000000000..ac14a99241652 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl @@ -0,0 +1,1163 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# June 2015 +# +# ChaCha20 for ARMv8. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.9 3xNEON+1xIALU 6xNEON+2xIALU(*) +# +# Apple A7 5.50/+49% 3.33 1.70 +# Cortex-A53 8.40/+80% 4.72 4.72(**) +# Cortex-A57 8.06/+43% 4.90 4.43(***) +# Denver 4.50/+82% 2.63 2.67(**) +# X-Gene 9.50/+46% 8.82 8.89(**) +# Mongoose 8.00/+44% 3.64 3.25(***) +# Kryo 8.17/+50% 4.83 4.65(***) +# +# (*) since no non-Apple processor exhibits significantly better +# performance, the code path is #ifdef __APPLE__-ed; +# (**) it's expected that doubling interleave factor doesn't help +# all processors, only those with higher NEON latency and +# higher instruction issue rate; +# (***) expected improvement was actually higher; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4)); + +my @x=map("x$_",(5..17,19..21)); +my @d=map("x$_",(22..28,30)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); + + ( + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],16)", + "&ror_32 (@x[$d1],@x[$d1],16)", + "&ror_32 (@x[$d2],@x[$d2],16)", + "&ror_32 (@x[$d3],@x[$d3],16)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],20)", + "&ror_32 (@x[$b1],@x[$b1],20)", + "&ror_32 (@x[$b2],@x[$b2],20)", + "&ror_32 (@x[$b3],@x[$b3],20)", + + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],24)", + "&ror_32 (@x[$d1],@x[$d1],24)", + "&ror_32 (@x[$d2],@x[$d2],24)", + "&ror_32 (@x[$d3],@x[$d3],24)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],25)", + "&ror_32 (@x[$b1],@x[$b1],25)", + "&ror_32 (@x[$b2],@x[$b2],25)", + "&ror_32 (@x[$b3],@x[$b3],25)" + ); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define ChaCha20_ctr32 chacha20_arm +# define ChaCha20_neon chacha20_neon +#endif + +.text + +.align 5 +.Lsigma: +.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral +.Lone: +.long 1,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +# ifdef __ILP32__ +.long OPENSSL_armcap_P-. +# else +.quad OPENSSL_armcap_P-. +# endif +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + cbz $len,.Labort +#ifndef __KERNEL__ + adr @x[0],.LOPENSSL_armcap_P + cmp $len,#192 + b.lo .Lshort +# ifdef __ILP32__ + ldrsw @x[1],[@x[0]] +# else + ldr @x[1],[@x[0]] +# endif + ldr w17,[@x[1],@x[0]] + tst w17,#ARMV7_NEON + b.ne ChaCha20_neon + +.Lshort: +#endif + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ldp @d[6],@d[7],[$ctr] // load counter +#ifdef __AARCH64EB__ + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + +.Loop_outer: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#64 +.Loop: + sub $ctr,$ctr,#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + cbnz $ctr,.Loop + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + b.lo .Ltail + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + + b.hi .Loop_outer + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 +.Labort: + ret + +.align 4 +.Ltail: + add $len,$len,#64 +.Less_than_64: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + stp @x[0],@x[2],[sp,#0] + stp @x[4],@x[6],[sp,#16] + stp @x[8],@x[10],[sp,#32] + stp @x[12],@x[14],[sp,#48] + +.Loop_tail: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) = + map("v$_.4s",(0..7,16..23)); +my (@K)=map("v$_.4s",(24..30)); +my $ONE="v31.4s"; + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&add ('$a','$a','$b')", + "&eor ('$d','$d','$a')", + "&rev32_16 ('$d','$d')", # vrot ($d,16) + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',20)", + "&sli ('$b','$t',12)", + + "&add ('$a','$a','$b')", + "&eor ('$t','$d','$a')", + "&ushr ('$d','$t',24)", + "&sli ('$d','$t',8)", + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',25)", + "&sli ('$b','$t',7)", + + "&ext ('$a','$a','$a',$odd?4:12)", + "&ext ('$d','$d','$d',8)", + "&ext ('$c','$c','$c',$odd?12:4)" + ); +} + +$code.=<<___; +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl ChaCha20_neon +.type ChaCha20_neon,%function +#endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] +#ifdef __APPLE__ + cmp $len,#512 + b.hs .L512_or_more_neon +#endif + + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +#ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + add @K[3],@K[3],$ONE // += 1 + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + +.Loop_outer_neon: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov $A0,@K[0] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov $A1,@K[0] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov $A2,@K[0] + mov.32 @x[6],@d[3] + mov $B0,@K[1] + lsr @x[7],@d[3],#32 + mov $B1,@K[1] + mov.32 @x[8],@d[4] + mov $B2,@K[1] + lsr @x[9],@d[4],#32 + mov $D0,@K[3] + mov.32 @x[10],@d[5] + mov $D1,@K[4] + lsr @x[11],@d[5],#32 + mov $D2,@K[5] + mov.32 @x[12],@d[6] + mov $C0,@K[2] + lsr @x[13],@d[6],#32 + mov $C1,@K[2] + mov.32 @x[14],@d[7] + mov $C2,@K[2] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#256 +.Loop_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + cbnz $ctr,.Loop_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add $A0,$A0,@K[0] + add @x[1],@x[1],@d[0],lsr#32 + add $A1,$A1,@K[0] + add.32 @x[2],@x[2],@d[1] + add $A2,$A2,@K[0] + add @x[3],@x[3],@d[1],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[4],@x[4],@d[2] + add $C1,$C1,@K[2] + add @x[5],@x[5],@d[2],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[6],@x[6],@d[3] + add $D0,$D0,@K[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add $D1,$D1,@K[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add $D2,$D2,@K[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add $B0,$B0,@K[1] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add $B1,$B1,@K[1] + add @x[15],@x[15],@d[7],lsr#32 + add $B2,$B2,@K[1] + + b.lo .Ltail_neon + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + add @K[3],@K[3],$ONE // += 4 + stp @x[8],@x[10],[$out,#32] + add @K[4],@K[4],$ONE + stp @x[12],@x[14],[$out,#48] + add @K[5],@K[5],$ONE + add $out,$out,#64 + + st1.8 {$A0-$D0},[$out],#64 + ld1.8 {$A0-$D0},[$inp],#64 + + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + eor $A2,$A2,$A0 + eor $B2,$B2,$B0 + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + b.hi .Loop_outer_neon + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret + +.Ltail_neon: + add $len,$len,#256 + cmp $len,#64 + b.lo .Less_than_64 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_128 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A0,$A0,$T0 + eor $B0,$B0,$T1 + eor $C0,$C0,$T2 + eor $D0,$D0,$T3 + st1.8 {$A0-$D0},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_192 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + + st1.8 {$A2-$D2},[sp] + b .Last_neon + +.Less_than_128: + st1.8 {$A0-$D0},[sp] + b .Last_neon +.Less_than_192: + st1.8 {$A1-$D1},[sp] + b .Last_neon + +.align 4 +.Last_neon: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + +.Loop_tail_neon: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail_neon + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + +.Ldone_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_neon,.-ChaCha20_neon +___ +{ +my ($T0,$T1,$T2,$T3,$T4,$T5)=@K; +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2, + $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23)); + +$code.=<<___; +#ifdef __APPLE__ +.type ChaCha20_512_neon,%function +.align 5 +ChaCha20_512_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + +.L512_or_more_neon: + sub sp,sp,#128+64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +# ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +# endif + add @K[3],@K[3],$ONE // += 1 + stp @K[0],@K[1],[sp,#0] // off-load key block, invariant part + add @K[3],@K[3],$ONE // not typo + str @K[2],[sp,#32] + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + add @K[6],@K[5],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + + stp d8,d9,[sp,#128+0] // meet ABI requirements + stp d10,d11,[sp,#128+16] + stp d12,d13,[sp,#128+32] + stp d14,d15,[sp,#128+48] + + sub $len,$len,#512 // not typo + +.Loop_outer_512_neon: + mov $A0,@K[0] + mov $A1,@K[0] + mov $A2,@K[0] + mov $A3,@K[0] + mov $A4,@K[0] + mov $A5,@K[0] + mov $B0,@K[1] + mov.32 @x[0],@d[0] // unpack key block + mov $B1,@K[1] + lsr @x[1],@d[0],#32 + mov $B2,@K[1] + mov.32 @x[2],@d[1] + mov $B3,@K[1] + lsr @x[3],@d[1],#32 + mov $B4,@K[1] + mov.32 @x[4],@d[2] + mov $B5,@K[1] + lsr @x[5],@d[2],#32 + mov $D0,@K[3] + mov.32 @x[6],@d[3] + mov $D1,@K[4] + lsr @x[7],@d[3],#32 + mov $D2,@K[5] + mov.32 @x[8],@d[4] + mov $D3,@K[6] + lsr @x[9],@d[4],#32 + mov $C0,@K[2] + mov.32 @x[10],@d[5] + mov $C1,@K[2] + lsr @x[11],@d[5],#32 + add $D4,$D0,$ONE // +4 + mov.32 @x[12],@d[6] + add $D5,$D1,$ONE // +4 + lsr @x[13],@d[6],#32 + mov $C2,@K[2] + mov.32 @x[14],@d[7] + mov $C3,@K[2] + lsr @x[15],@d[7],#32 + mov $C4,@K[2] + stp @K[3],@K[4],[sp,#48] // off-load key block, variable part + mov $C5,@K[2] + str @K[5],[sp,#80] + + mov $ctr,#5 + subs $len,$len,#512 +.Loop_upper_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + my $diff = ($#thread0+1)*6 - $#thread67 - 1; + my $i = 0; + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_upper_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + stp @x[4],@x[6],[$out,#16] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + stp @x[8],@x[10],[$out,#32] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#5 +.Loop_lower_neon: + sub $ctr,$ctr,#1 +___ + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_lower_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + ldp @K[0],@K[1],[sp,#0] + add @x[1],@x[1],@d[0],lsr#32 + ldp @K[2],@K[3],[sp,#32] + add.32 @x[2],@x[2],@d[1] + ldp @K[4],@K[5],[sp,#64] + add @x[3],@x[3],@d[1],lsr#32 + add $A0,$A0,@K[0] + add.32 @x[4],@x[4],@d[2] + add $A1,$A1,@K[0] + add @x[5],@x[5],@d[2],lsr#32 + add $A2,$A2,@K[0] + add.32 @x[6],@x[6],@d[3] + add $A3,$A3,@K[0] + add @x[7],@x[7],@d[3],lsr#32 + add $A4,$A4,@K[0] + add.32 @x[8],@x[8],@d[4] + add $A5,$A5,@K[0] + add @x[9],@x[9],@d[4],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[10],@x[10],@d[5] + add $C1,$C1,@K[2] + add @x[11],@x[11],@d[5],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[12],@x[12],@d[6] + add $C3,$C3,@K[2] + add @x[13],@x[13],@d[6],lsr#32 + add $C4,$C4,@K[2] + add.32 @x[14],@x[14],@d[7] + add $C5,$C5,@K[2] + add @x[15],@x[15],@d[7],lsr#32 + add $D4,$D4,$ONE // +4 + add @x[0],@x[0],@x[1],lsl#32 // pack + add $D5,$D5,$ONE // +4 + add @x[2],@x[2],@x[3],lsl#32 + add $D0,$D0,@K[3] + ldp @x[1],@x[3],[$inp,#0] // load input + add $D1,$D1,@K[4] + add @x[4],@x[4],@x[5],lsl#32 + add $D2,$D2,@K[5] + add @x[6],@x[6],@x[7],lsl#32 + add $D3,$D3,@K[6] + ldp @x[5],@x[7],[$inp,#16] + add $D4,$D4,@K[3] + add @x[8],@x[8],@x[9],lsl#32 + add $D5,$D5,@K[4] + add @x[10],@x[10],@x[11],lsl#32 + add $B0,$B0,@K[1] + ldp @x[9],@x[11],[$inp,#32] + add $B1,$B1,@K[1] + add @x[12],@x[12],@x[13],lsl#32 + add $B2,$B2,@K[1] + add @x[14],@x[14],@x[15],lsl#32 + add $B3,$B3,@K[1] + ldp @x[13],@x[15],[$inp,#48] + add $B4,$B4,@K[1] + add $inp,$inp,#64 + add $B5,$B5,@K[1] + +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#7 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + st1.8 {$A0-$D0},[$out],#64 + + ld1.8 {$A0-$D0},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + ld1.8 {$A1-$D1},[$inp],#64 + eor $A2,$A2,$A0 + ldp @K[0],@K[1],[sp,#0] + eor $B2,$B2,$B0 + ldp @K[2],@K[3],[sp,#32] + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + ld1.8 {$A2-$D2},[$inp],#64 + eor $A3,$A3,$A1 + eor $B3,$B3,$B1 + eor $C3,$C3,$C1 + eor $D3,$D3,$D1 + st1.8 {$A3-$D3},[$out],#64 + + ld1.8 {$A3-$D3},[$inp],#64 + eor $A4,$A4,$A2 + eor $B4,$B4,$B2 + eor $C4,$C4,$C2 + eor $D4,$D4,$D2 + st1.8 {$A4-$D4},[$out],#64 + + shl $A0,$ONE,#1 // 4 -> 8 + eor $A5,$A5,$A3 + eor $B5,$B5,$B3 + eor $C5,$C5,$C3 + eor $D5,$D5,$D3 + st1.8 {$A5-$D5},[$out],#64 + + add @K[3],@K[3],$A0 // += 8 + add @K[4],@K[4],$A0 + add @K[5],@K[5],$A0 + add @K[6],@K[6],$A0 + + b.hs .Loop_outer_512_neon + + adds $len,$len,#512 + ushr $A0,$ONE,#2 // 4 -> 1 + + ldp d8,d9,[sp,#128+0] // meet ABI requirements + ldp d10,d11,[sp,#128+16] + ldp d12,d13,[sp,#128+32] + ldp d14,d15,[sp,#128+48] + + stp @K[0],$ONE,[sp,#0] // wipe off-load area + stp @K[0],$ONE,[sp,#32] + stp @K[0],$ONE,[sp,#64] + + b.eq .Ldone_512_neon + + cmp $len,#192 + sub @K[3],@K[3],$A0 // -= 1 + sub @K[4],@K[4],$A0 + sub @K[5],@K[5],$A0 + add sp,sp,#128 + b.hs .Loop_outer_neon + + eor @K[1],@K[1],@K[1] + eor @K[2],@K[2],@K[2] + eor @K[3],@K[3],@K[3] + eor @K[4],@K[4],@K[4] + eor @K[5],@K[5],@K[5] + eor @K[6],@K[6],@K[6] + b .Loop_outer + +.Ldone_512_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#128+64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_512_neon,.-ChaCha20_512_neon +#endif +#endif +___ +} +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or + (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1)) or + (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or + (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or + (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1)); + + print $_,"\n"; +} +close STDOUT; # flush diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c new file mode 100644 index 0000000000000..96ce01e2c1333 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void chacha20_mips(u32 state[16], u8 *out, const u8 *in, + const size_t len); +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + chacha20_mips(ctx->state, dst, src, len); + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S new file mode 100644 index 0000000000000..a81e02db95e73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst . All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#define MASK_U32 0x3c +#define CHACHA20_BLOCK_SIZE 64 +#define STACK_SIZE 32 + +#define X0 $t0 +#define X1 $t1 +#define X2 $t2 +#define X3 $t3 +#define X4 $t4 +#define X5 $t5 +#define X6 $t6 +#define X7 $t7 +#define X8 $t8 +#define X9 $t9 +#define X10 $v1 +#define X11 $s6 +#define X12 $s5 +#define X13 $s4 +#define X14 $s3 +#define X15 $s2 +/* Use regs which are overwritten on exit for Tx so we don't leak clear data. */ +#define T0 $s1 +#define T1 $s0 +#define T(n) T ## n +#define X(n) X ## n + +/* Input arguments */ +#define STATE $a0 +#define OUT $a1 +#define IN $a2 +#define BYTES $a3 + +/* Output argument */ +/* NONCE[0] is kept in a register and not in memory. + * We don't want to touch original value in memory. + * Must be incremented every loop iteration. + */ +#define NONCE_0 $v0 + +/* SAVED_X and SAVED_CA are set in the jump table. + * Use regs which are overwritten on exit else we don't leak clear data. + * They are used to handling the last bytes which are not multiple of 4. + */ +#define SAVED_X X15 +#define SAVED_CA $s7 + +#define IS_UNALIGNED $s7 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#define ROTx rotl +#define ROTR(n) rotr n, 24 +#define CPU_TO_LE32(n) \ + wsbh n; \ + rotr n, 16; +#else +#define MSB 3 +#define LSB 0 +#define ROTx rotr +#define CPU_TO_LE32(n) +#define ROTR(n) +#endif + +#define FOR_EACH_WORD(x) \ + x( 0); \ + x( 1); \ + x( 2); \ + x( 3); \ + x( 4); \ + x( 5); \ + x( 6); \ + x( 7); \ + x( 8); \ + x( 9); \ + x(10); \ + x(11); \ + x(12); \ + x(13); \ + x(14); \ + x(15); + +#define FOR_EACH_WORD_REV(x) \ + x(15); \ + x(14); \ + x(13); \ + x(12); \ + x(11); \ + x(10); \ + x( 9); \ + x( 8); \ + x( 7); \ + x( 6); \ + x( 5); \ + x( 4); \ + x( 3); \ + x( 2); \ + x( 1); \ + x( 0); + +#define PLUS_ONE_0 1 +#define PLUS_ONE_1 2 +#define PLUS_ONE_2 3 +#define PLUS_ONE_3 4 +#define PLUS_ONE_4 5 +#define PLUS_ONE_5 6 +#define PLUS_ONE_6 7 +#define PLUS_ONE_7 8 +#define PLUS_ONE_8 9 +#define PLUS_ONE_9 10 +#define PLUS_ONE_10 11 +#define PLUS_ONE_11 12 +#define PLUS_ONE_12 13 +#define PLUS_ONE_13 14 +#define PLUS_ONE_14 15 +#define PLUS_ONE_15 16 +#define PLUS_ONE(x) PLUS_ONE_ ## x +#define _CONCAT3(a,b,c) a ## b ## c +#define CONCAT3(a,b,c) _CONCAT3(a,b,c) + +#define STORE_UNALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lwl T1, (x*4)+MSB ## (IN); \ + lwr T1, (x*4)+LSB ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + swl X ## x, (x*4)+MSB ## (OUT); \ + swr X ## x, (x*4)+LSB ## (OUT); + +#define STORE_ALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lw T1, (x*4) ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + sw X ## x, (x*4) ## (OUT); + +/* Jump table macro. + * Used for setup and handling the last bytes, which are not multiple of 4. + * X15 is free to store Xn + * Every jumptable entry must be equal in size. + */ +#define JMPTBL_ALIGNED(x) \ +.Lchacha20_mips_jmptbl_aligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_aligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define JMPTBL_UNALIGNED(x) \ +.Lchacha20_mips_jmptbl_unaligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_unaligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define AXR(A, B, C, D, K, L, M, N, V, W, Y, Z, S) \ + addu X(A), X(K); \ + addu X(B), X(L); \ + addu X(C), X(M); \ + addu X(D), X(N); \ + xor X(V), X(A); \ + xor X(W), X(B); \ + xor X(Y), X(C); \ + xor X(Z), X(D); \ + rotl X(V), S; \ + rotl X(W), S; \ + rotl X(Y), S; \ + rotl X(Z), S; + +.text +.set reorder +.set noat +.globl chacha20_mips +.ent chacha20_mips +chacha20_mips: + .frame $sp, STACK_SIZE, $ra + + addiu $sp, -STACK_SIZE + + /* Return bytes = 0. */ + beqz BYTES, .Lchacha20_mips_end + + lw NONCE_0, 48(STATE) + + /* Save s0-s7 */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + sw $s6, 24($sp) + sw $s7, 28($sp) + + /* Test IN or OUT is unaligned. + * IS_UNALIGNED = ( IN | OUT ) & 0x00000003 + */ + or IS_UNALIGNED, IN, OUT + andi IS_UNALIGNED, 0x3 + + /* Set number of rounds */ + li $at, 20 + + b .Lchacha20_rounds_start + +.align 4 +.Loop_chacha20_rounds: + addiu IN, CHACHA20_BLOCK_SIZE + addiu OUT, CHACHA20_BLOCK_SIZE + addiu NONCE_0, 1 + +.Lchacha20_rounds_start: + lw X0, 0(STATE) + lw X1, 4(STATE) + lw X2, 8(STATE) + lw X3, 12(STATE) + + lw X4, 16(STATE) + lw X5, 20(STATE) + lw X6, 24(STATE) + lw X7, 28(STATE) + lw X8, 32(STATE) + lw X9, 36(STATE) + lw X10, 40(STATE) + lw X11, 44(STATE) + + move X12, NONCE_0 + lw X13, 52(STATE) + lw X14, 56(STATE) + lw X15, 60(STATE) + +.Loop_chacha20_xor_rounds: + addiu $at, -2 + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 16); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 12); + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 8); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 7); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 16); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 12); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 8); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 7); + bnez $at, .Loop_chacha20_xor_rounds + + addiu BYTES, -(CHACHA20_BLOCK_SIZE) + + /* Is data src/dst unaligned? Jump */ + bnez IS_UNALIGNED, .Loop_chacha20_unaligned + + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES < 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_aligned + + FOR_EACH_WORD_REV(STORE_ALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + + /* BYTES < 0? Handle last bytes */ + bltz BYTES, .Lchacha20_mips_xor_bytes + +.Lchacha20_mips_xor_done: + /* Restore used registers */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + lw $s6, 24($sp) + lw $s7, 28($sp) + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + +.Lchacha20_mips_end: + addiu $sp, STACK_SIZE + jr $ra + +.Lchacha20_mips_no_full_block_aligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_aligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_aligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_ALIGNED) + + +.Loop_chacha20_unaligned: + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES > 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_unaligned + + FOR_EACH_WORD_REV(STORE_UNALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + + .set noreorder + /* Fall through to byte handling */ + bgez BYTES, .Lchacha20_mips_xor_done +.Lchacha20_mips_xor_unaligned_0_b: +.Lchacha20_mips_xor_aligned_0_b: + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + .set reorder + +.Lchacha20_mips_xor_bytes: + addu IN, $at + addu OUT, $at + /* First byte */ + lbu T1, 0(IN) + addiu $at, BYTES, 1 + CPU_TO_LE32(SAVED_X) + ROTR(SAVED_X) + xor T1, SAVED_X + sb T1, 0(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Second byte */ + lbu T1, 1(IN) + addiu $at, BYTES, 2 + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 1(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Third byte */ + lbu T1, 2(IN) + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 2(OUT) + b .Lchacha20_mips_xor_done + +.Lchacha20_mips_no_full_block_unaligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_UNALIGNED) +.end chacha20_mips +.set at diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S new file mode 100644 index 0000000000000..8fb4bc2e7b5b2 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S @@ -0,0 +1,461 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Google, Inc. + */ + +#include +#include + +/* + * Design notes: + * + * 16 registers would be needed to hold the state matrix, but only 14 are + * available because 'sp' and 'pc' cannot be used. So we spill the elements + * (x8, x9) to the stack and swap them out with (x10, x11). This adds one + * 'ldrd' and one 'strd' instruction per round. + * + * All rotates are performed using the implicit rotate operand accepted by the + * 'add' and 'eor' instructions. This is faster than using explicit rotate + * instructions. To make this work, we allow the values in the second and last + * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the + * wrong rotation amount. The rotation amount is then fixed up just in time + * when the values are used. 'brot' is the number of bits the values in row 'b' + * need to be rotated right to arrive at the correct values, and 'drot' + * similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such + * that they end up as (25, 24) after every round. + */ + + // ChaCha state registers + X0 .req r0 + X1 .req r1 + X2 .req r2 + X3 .req r3 + X4 .req r4 + X5 .req r5 + X6 .req r6 + X7 .req r7 + X8_X10 .req r8 // shared by x8 and x10 + X9_X11 .req r9 // shared by x9 and x11 + X12 .req r10 + X13 .req r11 + X14 .req r12 + X15 .req r14 + +.Lexpand_32byte_k: + // "expand 32-byte k" + .word 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + +#ifdef __thumb2__ +# define adrl adr +#endif + +.macro __rev out, in, t0, t1, t2 +.if __LINUX_ARM_ARCH__ >= 6 + rev \out, \in +.else + lsl \t0, \in, #24 + and \t1, \in, #0xff00 + and \t2, \in, #0xff0000 + orr \out, \t0, \in, lsr #24 + orr \out, \out, \t1, lsl #8 + orr \out, \out, \t2, lsr #8 +.endif +.endm + +.macro _le32_bswap x, t0, t1, t2 +#ifdef __ARMEB__ + __rev \x, \x, \t0, \t1, \t2 +#endif +.endm + +.macro _le32_bswap_4x a, b, c, d, t0, t1, t2 + _le32_bswap \a, \t0, \t1, \t2 + _le32_bswap \b, \t0, \t1, \t2 + _le32_bswap \c, \t0, \t1, \t2 + _le32_bswap \d, \t0, \t1, \t2 +.endm + +.macro __ldrd a, b, src, offset +#if __LINUX_ARM_ARCH__ >= 6 + ldrd \a, \b, [\src, #\offset] +#else + ldr \a, [\src, #\offset] + ldr \b, [\src, #\offset + 4] +#endif +.endm + +.macro __strd a, b, dst, offset +#if __LINUX_ARM_ARCH__ >= 6 + strd \a, \b, [\dst, #\offset] +#else + str \a, [\dst, #\offset] + str \b, [\dst, #\offset + 4] +#endif +.endm + +.macro _halfround a1, b1, c1, d1, a2, b2, c2, d2 + + // a += b; d ^= a; d = rol(d, 16); + add \a1, \a1, \b1, ror #brot + add \a2, \a2, \b2, ror #brot + eor \d1, \a1, \d1, ror #drot + eor \d2, \a2, \d2, ror #drot + // drot == 32 - 16 == 16 + + // c += d; b ^= c; b = rol(b, 12); + add \c1, \c1, \d1, ror #16 + add \c2, \c2, \d2, ror #16 + eor \b1, \c1, \b1, ror #brot + eor \b2, \c2, \b2, ror #brot + // brot == 32 - 12 == 20 + + // a += b; d ^= a; d = rol(d, 8); + add \a1, \a1, \b1, ror #20 + add \a2, \a2, \b2, ror #20 + eor \d1, \a1, \d1, ror #16 + eor \d2, \a2, \d2, ror #16 + // drot == 32 - 8 == 24 + + // c += d; b ^= c; b = rol(b, 7); + add \c1, \c1, \d1, ror #24 + add \c2, \c2, \d2, ror #24 + eor \b1, \c1, \b1, ror #20 + eor \b2, \c2, \b2, ror #20 + // brot == 32 - 7 == 25 +.endm + +.macro _doubleround + + // column round + + // quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13) + _halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13 + + // save (x8, x9); restore (x10, x11) + __strd X8_X10, X9_X11, sp, 0 + __ldrd X8_X10, X9_X11, sp, 8 + + // quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15) + _halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15 + + .set brot, 25 + .set drot, 24 + + // diagonal round + + // quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12) + _halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12 + + // save (x10, x11); restore (x8, x9) + __strd X8_X10, X9_X11, sp, 8 + __ldrd X8_X10, X9_X11, sp, 0 + + // quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14) + _halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14 +.endm + +.macro _chacha_permute nrounds + .set brot, 0 + .set drot, 0 + .rept \nrounds / 2 + _doubleround + .endr +.endm + +.macro _chacha nrounds + +.Lnext_block\@: + // Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + + // Do the core ChaCha permutation to update x0-x15. + _chacha_permute \nrounds + + add sp, #8 + // Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15). + push {X8_X10, X9_X11, X12, X13, X14, X15} + + // Load (OUT, IN, LEN). + ldr r14, [sp, #96] + ldr r12, [sp, #100] + ldr r11, [sp, #104] + + orr r10, r14, r12 + + // Use slow path if fewer than 64 bytes remain. + cmp r11, #64 + blt .Lxor_slowpath\@ + + // Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on + // ARMv6+, since ldmia and stmia (used below) still require alignment. + tst r10, #3 + bne .Lxor_slowpath\@ + + // Fast path: XOR 64 bytes of aligned data. + + // Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // x0-x3 + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + ldmia r12!, {r8-r11} + eor X0, X0, r8 + eor X1, X1, r9 + eor X2, X2, r10 + eor X3, X3, r11 + stmia r14!, {X0-X3} + + // x4-x7 + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + ldmia r12!, {X0-X3} + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + eor X4, X4, X0 + eor X5, X5, X1 + eor X6, X6, X2 + eor X7, X7, X3 + stmia r14!, {X4-X7} + + // x8-x15 + pop {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + ldmia r12!, {r8-r11} + eor r0, r0, r8 // x8 + eor r1, r1, r9 // x9 + eor r6, r6, r10 // x10 + eor r7, r7, r11 // x11 + stmia r14!, {r0,r1,r6,r7} + ldmia r12!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + ldr r9, [sp, #72] // load LEN + eor r2, r2, r0 // x12 + eor r3, r3, r1 // x13 + eor r4, r4, r6 // x14 + eor r5, r5, r7 // x15 + subs r9, #64 // decrement and check LEN + stmia r14!, {r2-r5} + + beq .Ldone\@ + +.Lprepare_for_next_block\@: + + // Stack: x0-x15 OUT IN LEN + + // Increment block counter (x12) + add r8, #1 + + // Store updated (OUT, IN, LEN) + str r14, [sp, #64] + str r12, [sp, #68] + str r9, [sp, #72] + + mov r14, sp + + // Store updated block counter (x12) + str r8, [sp, #48] + + sub sp, #16 + + // Reload state and do next block + ldmia r14!, {r0-r11} // load x0-x11 + __strd r10, r11, sp, 8 // store x10-x11 before state + ldmia r14, {r10-r12,r14} // load x12-x15 + b .Lnext_block\@ + +.Lxor_slowpath\@: + // Slow path: < 64 bytes remaining, or unaligned input or output buffer. + // We handle it by storing the 64 bytes of keystream to the stack, then + // XOR-ing the needed portion with the data. + + // Allocate keystream buffer + sub sp, #64 + mov r14, sp + + // Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Save keystream for x0-x3 + __ldrd r8, r9, sp, 96 + __ldrd r10, r11, sp, 104 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + stmia r14!, {X0-X3} + + // Save keystream for x4-x7 + __ldrd r8, r9, sp, 112 + __ldrd r10, r11, sp, 120 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + add r8, sp, #64 + stmia r14!, {X4-X7} + + // Save keystream for x8-x15 + ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 128 + __ldrd r10, r11, sp, 136 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + stmia r14!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 144 + __ldrd r10, r11, sp, 152 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + stmia r14, {r2-r5} + + // Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN + // Registers: r8 is block counter, r12 is IN. + + ldr r9, [sp, #168] // LEN + ldr r14, [sp, #160] // OUT + cmp r9, #64 + mov r0, sp + movle r1, r9 + movgt r1, #64 + // r1 is number of bytes to XOR, in range [1, 64] + +.if __LINUX_ARM_ARCH__ < 6 + orr r2, r12, r14 + tst r2, #3 // IN or OUT misaligned? + bne .Lxor_next_byte\@ +.endif + + // XOR a word at a time +.rept 16 + subs r1, #4 + blt .Lxor_words_done\@ + ldr r2, [r12], #4 + ldr r3, [r0], #4 + eor r2, r2, r3 + str r2, [r14], #4 +.endr + b .Lxor_slowpath_done\@ +.Lxor_words_done\@: + ands r1, r1, #3 + beq .Lxor_slowpath_done\@ + + // XOR a byte at a time +.Lxor_next_byte\@: + ldrb r2, [r12], #1 + ldrb r3, [r0], #1 + eor r2, r2, r3 + strb r2, [r14], #1 + subs r1, #1 + bne .Lxor_next_byte\@ + +.Lxor_slowpath_done\@: + subs r9, #64 + add sp, #96 + bgt .Lprepare_for_next_block\@ + +.Ldone\@: +.endm // _chacha + +/* + * void chacha20_arm(u8 *out, const u8 *in, size_t len, const u32 key[8], + * const u32 iv[4]); + */ +SYM_FUNC_START(chacha20_arm) + cmp r2, #0 // len == 0? + reteq lr + + push {r0-r2,r4-r11,lr} + + // Push state x0-x15 onto stack. + // Also store an extra copy of x10-x11 just before the state. + + ldr r4, [sp, #48] // iv + mov r0, sp + sub sp, #80 + + // iv: x12-x15 + ldm r4, {X12,X13,X14,X15} + stmdb r0!, {X12,X13,X14,X15} + + // key: x4-x11 + __ldrd X8_X10, X9_X11, r3, 24 + __strd X8_X10, X9_X11, sp, 8 + stmdb r0!, {X8_X10, X9_X11} + ldm r3, {X4-X9_X11} + stmdb r0!, {X4-X9_X11} + + // constants: x0-x3 + adrl X3, .Lexpand_32byte_k + ldm X3, {X0-X3} + __strd X0, X1, sp, 16 + __strd X2, X3, sp, 24 + + _chacha 20 + + add sp, #76 + pop {r4-r11, pc} +SYM_FUNC_END(chacha20_arm) + +/* + * void hchacha20_arm(const u32 state[16], u32 out[8]); + */ +SYM_FUNC_START(hchacha20_arm) + push {r1,r4-r11,lr} + + mov r14, r0 + ldmia r14!, {r0-r11} // load x0-x11 + push {r10-r11} // store x10-x11 to stack + ldm r14, {r10-r12,r14} // load x12-x15 + sub sp, #8 + + _chacha_permute 20 + + // Skip over (unused0-unused1, x10-x11) + add sp, #16 + + // Fix up rotations of x12-x15 + ror X12, X12, #drot + ror X13, X13, #drot + pop {r4} // load 'out' + ror X14, X14, #drot + ror X15, X15, #drot + + // Store (x0-x3,x12-x15) to 'out' + stm r4, {X0,X1,X2,X3,X12,X13,X14,X15} + + pop {r4-r11,pc} +SYM_FUNC_END(hchacha20_arm) diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c new file mode 100644 index 0000000000000..5ac5f686a641a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void hchacha20_ssse3(u32 *derived_key, const u8 *nonce, + const u8 *key); +asmlinkage void chacha20_ssse3(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx2(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512vl(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_ssse3 __ro_after_init; +static bool chacha20_use_avx2 __ro_after_init; +static bool chacha20_use_avx512 __ro_after_init; +static bool chacha20_use_avx512vl __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { + &chacha20_use_ssse3, &chacha20_use_avx2, &chacha20_use_avx512, + &chacha20_use_avx512vl }; + +static void __init chacha20_fpu_init(void) +{ + chacha20_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); + chacha20_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + chacha20_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; + chacha20_use_avx512vl = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !chacha20_use_ssse3 || + len <= CHACHA20_BLOCK_SIZE || !simd_use(simd_context)) + return false; + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 && + len >= CHACHA20_BLOCK_SIZE * 8) + chacha20_avx512(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx512vl(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx2(dst, src, bytes, ctx->key, ctx->counter); + else + chacha20_ssse3(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_AS_SSSE3) && chacha20_use_ssse3 && + simd_use(simd_context)) { + hchacha20_ssse3(derived_key, nonce, key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl new file mode 100644 index 0000000000000..29906a66b8b73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl @@ -0,0 +1,4106 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# November 2014 +# +# ChaCha20 for x86_64. +# +# December 2016 +# +# Add AVX512F code path. +# +# December 2017 +# +# Add AVX512VL code path. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc 4.8(i) 1x/2xSSSE3(ii) 4xSSSE3 NxAVX(v) +# +# P4 9.48/+99% - - +# Core2 7.83/+55% 7.90/5.76 4.35 +# Westmere 7.19/+50% 5.60/4.50 3.00 +# Sandy Bridge 8.31/+42% 5.45/4.00 2.72 +# Ivy Bridge 6.71/+46% 5.40/? 2.41 +# Haswell 5.92/+43% 5.20/3.45 2.42 1.23 +# Skylake[-X] 5.87/+39% 4.70/3.22 2.31 1.19[0.80(vi)] +# Silvermont 12.0/+33% 7.75/6.90 7.03(iii) +# Knights L 11.7/- ? 9.60(iii) 0.80 +# Goldmont 10.6/+17% 5.10/3.52 3.28 +# Sledgehammer 7.28/+52% - - +# Bulldozer 9.66/+28% 9.85/5.35(iv) 3.06(iv) +# Ryzen 5.96/+50% 5.19/3.00 2.40 2.09 +# VIA Nano 10.5/+46% 6.72/6.88 6.05 +# +# (i) compared to older gcc 3.x one can observe >2x improvement on +# most platforms; +# (ii) 2xSSSE3 is code path optimized specifically for 128 bytes used +# by chacha20_poly1305_tls_cipher, results are EVP-free; +# (iii) this is not optimal result for Atom because of MSROM +# limitations, SSE2 can do better, but gain is considered too +# low to justify the [maintenance] effort; +# (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20 +# and 4.85 for 128-byte inputs; +# (v) 8xAVX2, 8xAVX512VL or 16xAVX512F, whichever best applicable; +# (vi) even though Skylake-X can execute AVX512F code and deliver 0.57 +# cpb in single thread, the corresponding capability is suppressed; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +# input parameter block +($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8"); + +$code.=<<___ if $kernel; +#include +___ + +sub declare_variable() { + my ($name, $size, $type, $payload) = @_; + if($kernel) { + $code.=".section .rodata.cst$size.L$name, \"aM\", \@progbits, $size\n"; + $code.=".align $size\n"; + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } else { + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +if(!$kernel) { + $code .= ".text\n"; +} +&declare_variable('zero', 16, 'long', '0,0,0,0'); +&declare_variable('one', 16, 'long', '1,0,0,0'); +&declare_variable('inc', 16, 'long', '0,1,2,3'); +&declare_variable('four', 16, 'long', '4,4,4,4'); +&declare_variable('incy', 32, 'long', '0,2,4,6,1,3,5,7'); +&declare_variable('eight', 32, 'long', '8,8,8,8,8,8,8,8'); +&declare_variable('rot16', 16, 'byte', '0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd'); +&declare_variable('rot24', 16, 'byte', '0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe'); +&declare_variable('twoy', 32, 'long', '2,0,0,0, 2,0,0,0'); +&declare_variable('zeroz', 64, 'long', '0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0'); +&declare_variable('fourz', 64, 'long', '4,0,0,0, 4,0,0,0, 4,0,0,0, 4,0,0,0'); +&declare_variable('incz', 64, 'long', '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15'); +&declare_variable('sixteen', 64, 'long', '16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16'); +&declare_variable('sigma', 16, 'ascii', '"expand 32-byte k"'); + +$code.=<<___ if !$kernel; +.asciz "ChaCha20 for x86_64, CRYPTOGAMS by " +___ +$code.=".text\n"; + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +@x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)), + "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15))); +@t=("%esi","%edi"); + +sub ROUND { # critical path is 24 cycles per round +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_)=map("\"$_\"",@t); +my @x=map("\"$_\"",@x); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + # Normally instructions would be interleaved to favour in-order + # execution. Generally out-of-order cores manage it gracefully, + # but not this time for some reason. As in-order execution + # cores are dying breed, old Atom is the only one around, + # instructions are left uninterleaved. Besides, Atom is better + # off executing 1xSSSE3 code anyway... + + ( + "&add (@x[$a0],@x[$b0])", # Q1 + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],16)", + "&add (@x[$a1],@x[$b1])", # Q2 + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],16)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],12)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],12)", + + "&add (@x[$a0],@x[$b0])", + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],8)", + "&add (@x[$a1],@x[$b1])", + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],8)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],7)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],7)", + + "&mov (\"4*$c0(%rsp)\",$xc)", # reload pair of 'c's + "&mov (\"4*$c1(%rsp)\",$xc_)", + "&mov ($xc,\"4*$c2(%rsp)\")", + "&mov ($xc_,\"4*$c3(%rsp)\")", + + "&add (@x[$a2],@x[$b2])", # Q3 + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],16)", + "&add (@x[$a3],@x[$b3])", # Q4 + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],16)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],12)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],12)", + + "&add (@x[$a2],@x[$b2])", + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],8)", + "&add (@x[$a3],@x[$b3])", + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],8)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],7)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],7)" + ); +} + +######################################################################## +# Generic code path that handles all lengths on pre-SSSE3 processors. +if(!$kernel) { +&declare_function("chacha20_ctr32", 64, 5); +$code.=<<___; +.cfi_startproc + cmp \$0,$len + je .Lno_data + mov OPENSSL_ia32cap_P+4(%rip),%r9 +___ +$code.=<<___ if ($avx>2); + bt \$48,%r9 # check for AVX512F + jc .Lchacha20_avx512 + test %r9,%r9 # check for AVX512VL + js .Lchacha20_avx512vl +___ +$code.=<<___; + test \$`1<<(41-32)`,%r9d + jnz .Lchacha20_ssse3 +___ +$code.=<<___; + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$64+24,%rsp +.cfi_adjust_cfa_offset 64+24 +.Lctr32_body: + + #movdqa .Lsigma(%rip),%xmm0 + movdqu ($key),%xmm1 + movdqu 16($key),%xmm2 + movdqu ($counter),%xmm3 + movdqa .Lone(%rip),%xmm4 + + #movdqa %xmm0,4*0(%rsp) # key[0] + movdqa %xmm1,4*4(%rsp) # key[1] + movdqa %xmm2,4*8(%rsp) # key[2] + movdqa %xmm3,4*12(%rsp) # key[3] + mov $len,%rbp # reassign $len + jmp .Loop_outer + +.align 32 +.Loop_outer: + mov \$0x61707865,@x[0] # 'expa' + mov \$0x3320646e,@x[1] # 'nd 3' + mov \$0x79622d32,@x[2] # '2-by' + mov \$0x6b206574,@x[3] # 'te k' + mov 4*4(%rsp),@x[4] + mov 4*5(%rsp),@x[5] + mov 4*6(%rsp),@x[6] + mov 4*7(%rsp),@x[7] + movd %xmm3,@x[12] + mov 4*13(%rsp),@x[13] + mov 4*14(%rsp),@x[14] + mov 4*15(%rsp),@x[15] + + mov %rbp,64+0(%rsp) # save len + mov \$10,%ebp + mov $inp,64+8(%rsp) # save inp + movq %xmm2,%rsi # "@x[8]" + mov $out,64+16(%rsp) # save out + mov %rsi,%rdi + shr \$32,%rdi # "@x[9]" + jmp .Loop + +.align 32 +.Loop: +___ + foreach (&ROUND (0, 4, 8,12)) { eval; } + foreach (&ROUND (0, 5,10,15)) { eval; } + &dec ("%ebp"); + &jnz (".Loop"); + +$code.=<<___; + mov @t[1],4*9(%rsp) # modulo-scheduled + mov @t[0],4*8(%rsp) + mov 64(%rsp),%rbp # load len + movdqa %xmm2,%xmm1 + mov 64+8(%rsp),$inp # load inp + paddd %xmm4,%xmm3 # increment counter + mov 64+16(%rsp),$out # load out + + add \$0x61707865,@x[0] # 'expa' + add \$0x3320646e,@x[1] # 'nd 3' + add \$0x79622d32,@x[2] # '2-by' + add \$0x6b206574,@x[3] # 'te k' + add 4*4(%rsp),@x[4] + add 4*5(%rsp),@x[5] + add 4*6(%rsp),@x[6] + add 4*7(%rsp),@x[7] + add 4*12(%rsp),@x[12] + add 4*13(%rsp),@x[13] + add 4*14(%rsp),@x[14] + add 4*15(%rsp),@x[15] + paddd 4*8(%rsp),%xmm1 + + cmp \$64,%rbp + jb .Ltail + + xor 4*0($inp),@x[0] # xor with input + xor 4*1($inp),@x[1] + xor 4*2($inp),@x[2] + xor 4*3($inp),@x[3] + xor 4*4($inp),@x[4] + xor 4*5($inp),@x[5] + xor 4*6($inp),@x[6] + xor 4*7($inp),@x[7] + movdqu 4*8($inp),%xmm0 + xor 4*12($inp),@x[12] + xor 4*13($inp),@x[13] + xor 4*14($inp),@x[14] + xor 4*15($inp),@x[15] + lea 4*16($inp),$inp # inp+=64 + pxor %xmm1,%xmm0 + + movdqa %xmm2,4*8(%rsp) + movd %xmm3,4*12(%rsp) + + mov @x[0],4*0($out) # write output + mov @x[1],4*1($out) + mov @x[2],4*2($out) + mov @x[3],4*3($out) + mov @x[4],4*4($out) + mov @x[5],4*5($out) + mov @x[6],4*6($out) + mov @x[7],4*7($out) + movdqu %xmm0,4*8($out) + mov @x[12],4*12($out) + mov @x[13],4*13($out) + mov @x[14],4*14($out) + mov @x[15],4*15($out) + lea 4*16($out),$out # out+=64 + + sub \$64,%rbp + jnz .Loop_outer + + jmp .Ldone + +.align 16 +.Ltail: + mov @x[0],4*0(%rsp) + mov @x[1],4*1(%rsp) + xor %rbx,%rbx + mov @x[2],4*2(%rsp) + mov @x[3],4*3(%rsp) + mov @x[4],4*4(%rsp) + mov @x[5],4*5(%rsp) + mov @x[6],4*6(%rsp) + mov @x[7],4*7(%rsp) + movdqa %xmm1,4*8(%rsp) + mov @x[12],4*12(%rsp) + mov @x[13],4*13(%rsp) + mov @x[14],4*14(%rsp) + mov @x[15],4*15(%rsp) + +.Loop_tail: + movzb ($inp,%rbx),%eax + movzb (%rsp,%rbx),%edx + lea 1(%rbx),%rbx + xor %edx,%eax + mov %al,-1($out,%rbx) + dec %rbp + jnz .Loop_tail + +.Ldone: + add \$64+24,%rsp +.cfi_adjust_cfa_offset -64-24 + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbp +.cfi_restore %rbp + pop %rbx +.cfi_restore %rbx +.Lno_data: + ret +.cfi_endproc +___ +&end_function("chacha20_ctr32"); +} + +######################################################################## +# SSSE3 code path that handles shorter lengths +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7)); + +sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot16); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,20); + &pslld ($t,12); + &por ($b,$t); + + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot24); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,25); + &pslld ($t,7); + &por ($b,$t); +} + +my $xframe = $win64 ? 32+8 : 8; + +if($kernel) { + $code .= "#ifdef CONFIG_AS_SSSE3\n"; +} + +if($kernel) { +&declare_function("hchacha20_ssse3", 32, 5); +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($len),$b + movdqu 16($len),$c + movdqu ($inp),$d + # This code is only used when targeting kernel. + # If targeting win64, xmm{6,7} preserving needs to be added. + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + mov \$10,$counter # reuse $counter + jmp 1f +.align 32 +1: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz ("1b"); + +$code.=<<___; + movdqu $a, ($out) + movdqu $d, 16($out) + ret +___ +&end_function("hchacha20_ssse3"); +} + +&declare_function("chacha20_ssse3", 32, 5); +$code.=<<___; +.cfi_startproc + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if ($avx && !$kernel); + test \$`1<<(43-32)`,%r10d + jnz .Lchacha20_4xop # XOP is fastest even if we use 1/4 +___ +$code.=<<___; + cmp \$128,$len # we might throw away some data, + je .Lchacha20_128 + ja .Lchacha20_4x # but overall it won't be slower + +.Ldo_ssse3_after_all: + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lssse3_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_ssse3 + +.align 32 +.Loop_outer_ssse3: + movdqa .Lone(%rip),$d + movdqa 0x00(%rsp),$a + movdqa 0x10(%rsp),$b + movdqa 0x20(%rsp),$c + paddd 0x30(%rsp),$d + mov \$10,$counter + movdqa $d,0x30(%rsp) + jmp .Loop_ssse3 + +.align 32 +.Loop_ssse3: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz (".Loop_ssse3"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + + cmp \$64,$len + jb .Ltail_ssse3 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + lea 0x40($inp),$inp # inp+=64 + pxor $t,$c + pxor $t1,$d + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + lea 0x40($out),$out # out+=64 + + sub \$64,$len + jnz .Loop_outer_ssse3 + + jmp .Ldone_ssse3 + +.align 16 +.Ltail_ssse3: + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + xor $counter,$counter + +.Loop_tail_ssse3: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_ssse3 + +.Ldone_ssse3: +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lssse3_epilogue: + ret +.cfi_endproc +___ +} +&end_function("chacha20_ssse3"); + +######################################################################## +# SSSE3 code path that handles 128-byte inputs +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(8,9,2..7)); +my ($a1,$b1,$c1,$d1)=map("%xmm$_",(10,11,0,1)); + +sub SSSE3ROUND_2x { + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot16); + &pshufb($d1,$rot16); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,20); + &movdqa($t1,$b1); + &pslld ($t,12); + &psrld ($b1,20); + &por ($b,$t); + &pslld ($t1,12); + &por ($b1,$t1); + + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot24); + &pshufb($d1,$rot24); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,25); + &movdqa($t1,$b1); + &pslld ($t,7); + &psrld ($b1,25); + &por ($b,$t); + &pslld ($t1,7); + &por ($b1,$t1); +} + +my $xframe = $win64 ? 0x68 : 8; + +$code.=<<___; +.type chacha20_128,\@function,5 +.align 32 +chacha20_128: +.cfi_startproc +.Lchacha20_128: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x70(%r10) + movaps %xmm7,-0x60(%r10) + movaps %xmm8,-0x50(%r10) + movaps %xmm9,-0x40(%r10) + movaps %xmm10,-0x30(%r10) + movaps %xmm11,-0x20(%r10) +.L128_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lone(%rip),$d1 + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,$a1 + movdqa $a,0x00(%rsp) + movdqa $b,$b1 + movdqa $b,0x10(%rsp) + movdqa $c,$c1 + movdqa $c,0x20(%rsp) + paddd $d,$d1 + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_128 + +.align 32 +.Loop_128: +___ + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &pshufd ($a1,$a1,0b10010011); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b00111001); + + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + &pshufd ($a1,$a1,0b00111001); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b10010011); + + &dec ($counter); + &jnz (".Loop_128"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + paddd .Lone(%rip),$d1 + paddd 0x00(%rsp),$a1 + paddd 0x10(%rsp),$b1 + paddd 0x20(%rsp),$c1 + paddd 0x30(%rsp),$d1 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + pxor $t,$c + movdqu 0x40($inp),$t + pxor $t1,$d + movdqu 0x50($inp),$t1 + pxor $t,$a1 + movdqu 0x60($inp),$t + pxor $t1,$b1 + movdqu 0x70($inp),$t1 + pxor $t,$c1 + pxor $t1,$d1 + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + movdqu $a1,0x40($out) + movdqu $b1,0x50($out) + movdqu $c1,0x60($out) + movdqu $d1,0x70($out) +___ +$code.=<<___ if ($win64); + movaps -0x70(%r10),%xmm6 + movaps -0x60(%r10),%xmm7 + movaps -0x50(%r10),%xmm8 + movaps -0x40(%r10),%xmm9 + movaps -0x30(%r10),%xmm10 + movaps -0x20(%r10),%xmm11 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L128_epilogue: + ret +.cfi_endproc +.size chacha20_128,.-chacha20_128 +___ +} + +######################################################################## +# SSSE3 code path that handles longer messages. +{ +# assign variables to favor Atom front-end +my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3, + $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub SSSE3_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&paddd (@x[$a0],@x[$b0])", # Q1 + "&paddd (@x[$a1],@x[$b1])", # Q2 + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t1)", + "&pshufb (@x[$d1],$t1)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t0,@x[$b0])", + "&pslld (@x[$b0],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b1])", + "&pslld (@x[$b1],12)", + "&por (@x[$b0],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b1],$t1)", + + "&paddd (@x[$a0],@x[$b0])", + "&paddd (@x[$a1],@x[$b1])", + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t0)", + "&pshufb (@x[$d1],$t0)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t1,@x[$b0])", + "&pslld (@x[$b0],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b1])", + "&pslld (@x[$b1],7)", + "&por (@x[$b0],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b1],$t0)", + + "&movdqa (\"`16*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&movdqa (\"`16*($c1-8)`(%rsp)\",$xc_)", + "&movdqa ($xc,\"`16*($c2-8)`(%rsp)\")", + "&movdqa ($xc_,\"`16*($c3-8)`(%rsp)\")", + + "&paddd (@x[$a2],@x[$b2])", # Q3 + "&paddd (@x[$a3],@x[$b3])", # Q4 + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t1)", + "&pshufb (@x[$d3],$t1)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t0,@x[$b2])", + "&pslld (@x[$b2],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b3])", + "&pslld (@x[$b3],12)", + "&por (@x[$b2],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b3],$t1)", + + "&paddd (@x[$a2],@x[$b2])", + "&paddd (@x[$a3],@x[$b3])", + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t0)", + "&pshufb (@x[$d3],$t0)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t1,@x[$b2])", + "&pslld (@x[$b2],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b3])", + "&pslld (@x[$b3],7)", + "&por (@x[$b2],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b3],$t0)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_4x,\@function,5 +.align 32 +chacha20_4x: +.cfi_startproc +.Lchacha20_4x: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if (!$kernel); + mov %r9,%r11 +___ +$code.=<<___ if ($avx>1 && !$kernel); + shr \$32,%r9 # OPENSSL_ia32cap_P+8 + test \$`1<<5`,%r9 # test AVX2 + jnz .Lchacha20_8x +___ +$code.=<<___; + cmp \$192,$len + ja .Lproceed4x +___ +$code.=<<___ if (!$kernel); + and \$`1<<26|1<<22`,%r11 # isolate XSAVE+MOVBE + cmp \$`1<<22`,%r11 # check for MOVBE without XSAVE + je .Ldo_ssse3_after_all # to detect Atom +___ +$code.=<<___; +.Lproceed4x: + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4x_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$xa3 # key[0] + movdqu ($key),$xb3 # key[1] + movdqu 16($key),$xt3 # key[2] + movdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + pshufd \$0x00,$xa3,$xa0 # smash key by lanes... + pshufd \$0x55,$xa3,$xa1 + movdqa $xa0,0x40(%rsp) # ... and offload + pshufd \$0xaa,$xa3,$xa2 + movdqa $xa1,0x50(%rsp) + pshufd \$0xff,$xa3,$xa3 + movdqa $xa2,0x60(%rsp) + movdqa $xa3,0x70(%rsp) + + pshufd \$0x00,$xb3,$xb0 + pshufd \$0x55,$xb3,$xb1 + movdqa $xb0,0x80-0x100(%rcx) + pshufd \$0xaa,$xb3,$xb2 + movdqa $xb1,0x90-0x100(%rcx) + pshufd \$0xff,$xb3,$xb3 + movdqa $xb2,0xa0-0x100(%rcx) + movdqa $xb3,0xb0-0x100(%rcx) + + pshufd \$0x00,$xt3,$xt0 # "$xc0" + pshufd \$0x55,$xt3,$xt1 # "$xc1" + movdqa $xt0,0xc0-0x100(%rcx) + pshufd \$0xaa,$xt3,$xt2 # "$xc2" + movdqa $xt1,0xd0-0x100(%rcx) + pshufd \$0xff,$xt3,$xt3 # "$xc3" + movdqa $xt2,0xe0-0x100(%rcx) + movdqa $xt3,0xf0-0x100(%rcx) + + pshufd \$0x00,$xd3,$xd0 + pshufd \$0x55,$xd3,$xd1 + paddd .Linc(%rip),$xd0 # don't save counters yet + pshufd \$0xaa,$xd3,$xd2 + movdqa $xd1,0x110-0x100(%rcx) + pshufd \$0xff,$xd3,$xd3 + movdqa $xd2,0x120-0x100(%rcx) + movdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4x + +.align 32 +.Loop_outer4x: + movdqa 0x40(%rsp),$xa0 # re-load smashed key + movdqa 0x50(%rsp),$xa1 + movdqa 0x60(%rsp),$xa2 + movdqa 0x70(%rsp),$xa3 + movdqa 0x80-0x100(%rcx),$xb0 + movdqa 0x90-0x100(%rcx),$xb1 + movdqa 0xa0-0x100(%rcx),$xb2 + movdqa 0xb0-0x100(%rcx),$xb3 + movdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + movdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + movdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + movdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + movdqa 0x100-0x100(%rcx),$xd0 + movdqa 0x110-0x100(%rcx),$xd1 + movdqa 0x120-0x100(%rcx),$xd2 + movdqa 0x130-0x100(%rcx),$xd3 + paddd .Lfour(%rip),$xd0 # next SIMD counters + +.Loop_enter4x: + movdqa $xt2,0x20(%rsp) # SIMD equivalent of "@x[10]" + movdqa $xt3,0x30(%rsp) # SIMD equivalent of "@x[11]" + movdqa (%r9),$xt3 # .Lrot16(%rip) + mov \$10,%eax + movdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4x + +.align 32 +.Loop4x: +___ + foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4x + + paddd 0x40(%rsp),$xa0 # accumulate key material + paddd 0x50(%rsp),$xa1 + paddd 0x60(%rsp),$xa2 + paddd 0x70(%rsp),$xa3 + + movdqa $xa0,$xt2 # "de-interlace" data + punpckldq $xa1,$xa0 + movdqa $xa2,$xt3 + punpckldq $xa3,$xa2 + punpckhdq $xa1,$xt2 + punpckhdq $xa3,$xt3 + movdqa $xa0,$xa1 + punpcklqdq $xa2,$xa0 # "a0" + movdqa $xt2,$xa3 + punpcklqdq $xt3,$xt2 # "a2" + punpckhqdq $xa2,$xa1 # "a1" + punpckhqdq $xt3,$xa3 # "a3" +___ + ($xa2,$xt2)=($xt2,$xa2); +$code.=<<___; + paddd 0x80-0x100(%rcx),$xb0 + paddd 0x90-0x100(%rcx),$xb1 + paddd 0xa0-0x100(%rcx),$xb2 + paddd 0xb0-0x100(%rcx),$xb3 + + movdqa $xa0,0x00(%rsp) # offload $xaN + movdqa $xa1,0x10(%rsp) + movdqa 0x20(%rsp),$xa0 # "xc2" + movdqa 0x30(%rsp),$xa1 # "xc3" + + movdqa $xb0,$xt2 + punpckldq $xb1,$xb0 + movdqa $xb2,$xt3 + punpckldq $xb3,$xb2 + punpckhdq $xb1,$xt2 + punpckhdq $xb3,$xt3 + movdqa $xb0,$xb1 + punpcklqdq $xb2,$xb0 # "b0" + movdqa $xt2,$xb3 + punpcklqdq $xt3,$xt2 # "b2" + punpckhqdq $xb2,$xb1 # "b1" + punpckhqdq $xt3,$xb3 # "b3" +___ + ($xb2,$xt2)=($xt2,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + paddd 0xc0-0x100(%rcx),$xc0 + paddd 0xd0-0x100(%rcx),$xc1 + paddd 0xe0-0x100(%rcx),$xc2 + paddd 0xf0-0x100(%rcx),$xc3 + + movdqa $xa2,0x20(%rsp) # keep offloading $xaN + movdqa $xa3,0x30(%rsp) + + movdqa $xc0,$xt2 + punpckldq $xc1,$xc0 + movdqa $xc2,$xt3 + punpckldq $xc3,$xc2 + punpckhdq $xc1,$xt2 + punpckhdq $xc3,$xt3 + movdqa $xc0,$xc1 + punpcklqdq $xc2,$xc0 # "c0" + movdqa $xt2,$xc3 + punpcklqdq $xt3,$xt2 # "c2" + punpckhqdq $xc2,$xc1 # "c1" + punpckhqdq $xt3,$xc3 # "c3" +___ + ($xc2,$xt2)=($xt2,$xc2); + ($xt0,$xt1)=($xa2,$xa3); # use $xaN as temporary +$code.=<<___; + paddd 0x100-0x100(%rcx),$xd0 + paddd 0x110-0x100(%rcx),$xd1 + paddd 0x120-0x100(%rcx),$xd2 + paddd 0x130-0x100(%rcx),$xd3 + + movdqa $xd0,$xt2 + punpckldq $xd1,$xd0 + movdqa $xd2,$xt3 + punpckldq $xd3,$xd2 + punpckhdq $xd1,$xt2 + punpckhdq $xd3,$xt3 + movdqa $xd0,$xd1 + punpcklqdq $xd2,$xd0 # "d0" + movdqa $xt2,$xd3 + punpcklqdq $xt3,$xt2 # "d2" + punpckhqdq $xd2,$xd1 # "d1" + punpckhqdq $xt3,$xd3 # "d3" +___ + ($xd2,$xt2)=($xt2,$xd2); +$code.=<<___; + cmp \$64*4,$len + jb .Ltail4x + + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # inp+=64*4 + pxor 0x30(%rsp),$xt0 + pxor $xb3,$xt1 + pxor $xc3,$xt2 + pxor $xd3,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4x + + jmp .Ldone4x + +.Ltail4x: + cmp \$192,$len + jae .L192_or_more4x + cmp \$128,$len + jae .L128_or_more4x + cmp \$64,$len + jae .L64_or_more4x + + #movdqa 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + xor %r9,%r9 + #movdqa $xt0,0x00(%rsp) + movdqa $xb0,0x10(%rsp) + movdqa $xc0,0x20(%rsp) + movdqa $xd0,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L64_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaxN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x10(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + movdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + movdqa $xd1,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L128_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + je .Ldone4x + + movdqa 0x20(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + movdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + movdqa $xd2,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L192_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x30(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*3 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + movdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + movdqa $xd3,0x30(%rsp) + +.Loop_tail4x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4x + +.Ldone4x: +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4x_epilogue: + ret +.cfi_endproc +.size chacha20_4x,.-chacha20_4x +___ +} +if($kernel) { + $code .= "#endif\n"; +} + +######################################################################## +# XOP code path that handles all lengths. +if ($avx && !$kernel) { +# There is some "anomaly" observed depending on instructions' size or +# alignment. If you look closely at below code you'll notice that +# sometimes argument order varies. The order affects instruction +# encoding by making it larger, and such fiddling gives 5% performance +# improvement. This is on FX-4100... + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3); + +sub XOP_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],16)", + "&vprotd (@x[$d1],@x[$d1],16)", + "&vprotd (@x[$d2],@x[$d2],16)", + "&vprotd (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],12)", + "&vprotd (@x[$b1],@x[$b1],12)", + "&vprotd (@x[$b2],@x[$b2],12)", + "&vprotd (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$b0],@x[$a0])", # flip + "&vpaddd (@x[$a1],@x[$b1],@x[$a1])", # flip + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],8)", + "&vprotd (@x[$d1],@x[$d1],8)", + "&vprotd (@x[$d2],@x[$d2],8)", + "&vprotd (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],7)", + "&vprotd (@x[$b1],@x[$b1],7)", + "&vprotd (@x[$b2],@x[$b2],7)", + "&vprotd (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_xop", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_4xop: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4xop_body: +___ +$code.=<<___; + vzeroupper + + vmovdqa .Lsigma(%rip),$xa3 # key[0] + vmovdqu ($key),$xb3 # key[1] + vmovdqu 16($key),$xt3 # key[2] + vmovdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x40(%rsp) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0x50(%rsp) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0x60(%rsp) + vmovdqa $xa3,0x70(%rsp) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x80-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x90-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0xa0-0x100(%rcx) + vmovdqa $xb3,0xb0-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "$xc0" + vpshufd \$0x55,$xt3,$xt1 # "$xc1" + vmovdqa $xt0,0xc0-0x100(%rcx) + vpshufd \$0xaa,$xt3,$xt2 # "$xc2" + vmovdqa $xt1,0xd0-0x100(%rcx) + vpshufd \$0xff,$xt3,$xt3 # "$xc3" + vmovdqa $xt2,0xe0-0x100(%rcx) + vmovdqa $xt3,0xf0-0x100(%rcx) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Linc(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x110-0x100(%rcx) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x120-0x100(%rcx) + vmovdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4xop + +.align 32 +.Loop_outer4xop: + vmovdqa 0x40(%rsp),$xa0 # re-load smashed key + vmovdqa 0x50(%rsp),$xa1 + vmovdqa 0x60(%rsp),$xa2 + vmovdqa 0x70(%rsp),$xa3 + vmovdqa 0x80-0x100(%rcx),$xb0 + vmovdqa 0x90-0x100(%rcx),$xb1 + vmovdqa 0xa0-0x100(%rcx),$xb2 + vmovdqa 0xb0-0x100(%rcx),$xb3 + vmovdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + vmovdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + vmovdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + vmovdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + vmovdqa 0x100-0x100(%rcx),$xd0 + vmovdqa 0x110-0x100(%rcx),$xd1 + vmovdqa 0x120-0x100(%rcx),$xd2 + vmovdqa 0x130-0x100(%rcx),$xd3 + vpaddd .Lfour(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter4xop: + mov \$10,%eax + vmovdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4xop + +.align 32 +.Loop4xop: +___ + foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4xop + + vpaddd 0x40(%rsp),$xa0,$xa0 # accumulate key material + vpaddd 0x50(%rsp),$xa1,$xa1 + vpaddd 0x60(%rsp),$xa2,$xa2 + vpaddd 0x70(%rsp),$xa3,$xa3 + + vmovdqa $xt2,0x20(%rsp) # offload $xc2,3 + vmovdqa $xt3,0x30(%rsp) + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x80-0x100(%rcx),$xb0,$xb0 + vpaddd 0x90-0x100(%rcx),$xb1,$xb1 + vpaddd 0xa0-0x100(%rcx),$xb2,$xb2 + vpaddd 0xb0-0x100(%rcx),$xb3,$xb3 + + vmovdqa $xa0,0x00(%rsp) # offload $xa0,1 + vmovdqa $xa1,0x10(%rsp) + vmovdqa 0x20(%rsp),$xa0 # "xc2" + vmovdqa 0x30(%rsp),$xa1 # "xc3" + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vpaddd 0xc0-0x100(%rcx),$xc0,$xc0 + vpaddd 0xd0-0x100(%rcx),$xc1,$xc1 + vpaddd 0xe0-0x100(%rcx),$xc2,$xc2 + vpaddd 0xf0-0x100(%rcx),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xd0,$xd0 + vpaddd 0x110-0x100(%rcx),$xd1,$xd1 + vpaddd 0x120-0x100(%rcx),$xd2,$xd2 + vpaddd 0x130-0x100(%rcx),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # restore $xa0,1 + vmovdqa 0x10(%rsp),$xa1 + + cmp \$64*4,$len + jb .Ltail4xop + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + vpxor 0x40($inp),$xa3,$xa3 + vpxor 0x50($inp),$xb3,$xb3 + vpxor 0x60($inp),$xc3,$xc3 + vpxor 0x70($inp),$xd3,$xd3 + lea 0x80($inp),$inp # inp+=64*4 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + vmovdqu $xa3,0x40($out) + vmovdqu $xb3,0x50($out) + vmovdqu $xc3,0x60($out) + vmovdqu $xd3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4xop + + jmp .Ldone4xop + +.align 32 +.Ltail4xop: + cmp \$192,$len + jae .L192_or_more4xop + cmp \$128,$len + jae .L128_or_more4xop + cmp \$64,$len + jae .L64_or_more4xop + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x10(%rsp) + vmovdqa $xc0,0x20(%rsp) + vmovdqa $xd0,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L64_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*1 + vmovdqa $xa1,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + vmovdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + vmovdqa $xd1,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L128_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + je .Ldone4xop + + lea 0x80($inp),$inp # inp+=64*2 + vmovdqa $xa2,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + vmovdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + vmovdqa $xd2,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L192_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*3 + vmovdqa $xa3,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + vmovdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + vmovdqa $xd3,0x30(%rsp) + +.Loop_tail4xop: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4xop + +.Ldone4xop: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4xop_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_xop"); +} + +######################################################################## +# AVX2 code path +if ($avx>1) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub AVX2_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t1)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t1)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t0,@x[$b0],12)", + "&vpsrld (@x[$b0],@x[$b0],20)", + "&vpor (@x[$b0],$t0,@x[$b0])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t1,@x[$b1],12)", + "&vpsrld (@x[$b1],@x[$b1],20)", + "&vpor (@x[$b1],$t1,@x[$b1])", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t0)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t0)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t1,@x[$b0],7)", + "&vpsrld (@x[$b0],@x[$b0],25)", + "&vpor (@x[$b0],$t1,@x[$b0])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t0,@x[$b1],7)", + "&vpsrld (@x[$b1],@x[$b1],25)", + "&vpor (@x[$b1],$t0,@x[$b1])", + + "&vmovdqa (\"`32*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&vmovdqa (\"`32*($c1-8)`(%rsp)\",$xc_)", + "&vmovdqa ($xc,\"`32*($c2-8)`(%rsp)\")", + "&vmovdqa ($xc_,\"`32*($c3-8)`(%rsp)\")", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t1)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t1)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t0,@x[$b2],12)", + "&vpsrld (@x[$b2],@x[$b2],20)", + "&vpor (@x[$b2],$t0,@x[$b2])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t1,@x[$b3],12)", + "&vpsrld (@x[$b3],@x[$b3],20)", + "&vpor (@x[$b3],$t1,@x[$b3])", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t0)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t0)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t1,@x[$b2],7)", + "&vpsrld (@x[$b2],@x[$b2],25)", + "&vpor (@x[$b2],$t1,@x[$b2])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t0,@x[$b3],7)", + "&vpsrld (@x[$b3],@x[$b3],25)", + "&vpor (@x[$b3],$t0,@x[$b3])" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_avx2", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_8x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$0x280+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8x_body: +___ +$code.=<<___; + vzeroupper + + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x80 constant copy of key[0-2] smashed by lanes + # ... + # +0x200 SIMD counters (with nonce smashed by lanes) + # ... + # +0x280 + + vbroadcasti128 .Lsigma(%rip),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xt3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea 0x200(%rsp),%rax # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x80-0x100(%rcx) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0xa0-0x100(%rcx) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0xc0-0x100(%rcx) + vmovdqa $xa3,0xe0-0x100(%rcx) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x100-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x120-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0x140-0x100(%rcx) + vmovdqa $xb3,0x160-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "xc0" + vpshufd \$0x55,$xt3,$xt1 # "xc1" + vmovdqa $xt0,0x180-0x200(%rax) + vpshufd \$0xaa,$xt3,$xt2 # "xc2" + vmovdqa $xt1,0x1a0-0x200(%rax) + vpshufd \$0xff,$xt3,$xt3 # "xc3" + vmovdqa $xt2,0x1c0-0x200(%rax) + vmovdqa $xt3,0x1e0-0x200(%rax) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x220-0x200(%rax) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x240-0x200(%rax) + vmovdqa $xd3,0x260-0x200(%rax) + + jmp .Loop_enter8x + +.align 32 +.Loop_outer8x: + vmovdqa 0x80-0x100(%rcx),$xa0 # re-load smashed key + vmovdqa 0xa0-0x100(%rcx),$xa1 + vmovdqa 0xc0-0x100(%rcx),$xa2 + vmovdqa 0xe0-0x100(%rcx),$xa3 + vmovdqa 0x100-0x100(%rcx),$xb0 + vmovdqa 0x120-0x100(%rcx),$xb1 + vmovdqa 0x140-0x100(%rcx),$xb2 + vmovdqa 0x160-0x100(%rcx),$xb3 + vmovdqa 0x180-0x200(%rax),$xt0 # "xc0" + vmovdqa 0x1a0-0x200(%rax),$xt1 # "xc1" + vmovdqa 0x1c0-0x200(%rax),$xt2 # "xc2" + vmovdqa 0x1e0-0x200(%rax),$xt3 # "xc3" + vmovdqa 0x200-0x200(%rax),$xd0 + vmovdqa 0x220-0x200(%rax),$xd1 + vmovdqa 0x240-0x200(%rax),$xd2 + vmovdqa 0x260-0x200(%rax),$xd3 + vpaddd .Leight(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter8x: + vmovdqa $xt2,0x40(%rsp) # SIMD equivalent of "@x[10]" + vmovdqa $xt3,0x60(%rsp) # SIMD equivalent of "@x[11]" + vbroadcasti128 (%r9),$xt3 + vmovdqa $xd0,0x200-0x200(%rax) # save SIMD counters + mov \$10,%eax + jmp .Loop8x + +.align 32 +.Loop8x: +___ + foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8x + + lea 0x200(%rsp),%rax # size optimization + vpaddd 0x80-0x100(%rcx),$xa0,$xa0 # accumulate key + vpaddd 0xa0-0x100(%rcx),$xa1,$xa1 + vpaddd 0xc0-0x100(%rcx),$xa2,$xa2 + vpaddd 0xe0-0x100(%rcx),$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xb0,$xb0 + vpaddd 0x120-0x100(%rcx),$xb1,$xb1 + vpaddd 0x140-0x100(%rcx),$xb2,$xb2 + vpaddd 0x160-0x100(%rcx),$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vperm2i128 \$0x20,$xb0,$xa0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xb0,$xa0,$xb0 + vperm2i128 \$0x20,$xb1,$xa1,$xa0 + vperm2i128 \$0x31,$xb1,$xa1,$xb1 + vperm2i128 \$0x20,$xb2,$xa2,$xa1 + vperm2i128 \$0x31,$xb2,$xa2,$xb2 + vperm2i128 \$0x20,$xb3,$xa3,$xa2 + vperm2i128 \$0x31,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vmovdqa $xa0,0x00(%rsp) # offload $xaN + vmovdqa $xa1,0x20(%rsp) + vmovdqa 0x40(%rsp),$xc2 # $xa0 + vmovdqa 0x60(%rsp),$xc3 # $xa1 + + vpaddd 0x180-0x200(%rax),$xc0,$xc0 + vpaddd 0x1a0-0x200(%rax),$xc1,$xc1 + vpaddd 0x1c0-0x200(%rax),$xc2,$xc2 + vpaddd 0x1e0-0x200(%rax),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x200-0x200(%rax),$xd0,$xd0 + vpaddd 0x220-0x200(%rax),$xd1,$xd1 + vpaddd 0x240-0x200(%rax),$xd2,$xd2 + vpaddd 0x260-0x200(%rax),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # $xaN was offloaded, remember? + vmovdqa 0x20(%rsp),$xa1 + + cmp \$64*8,$len + jb .Ltail8x + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea 0x80($out),$out # size optimization + + sub \$64*8,$len + jnz .Loop_outer8x + + jmp .Ldone8x + +.Ltail8x: + cmp \$448,$len + jae .L448_or_more8x + cmp \$384,$len + jae .L384_or_more8x + cmp \$320,$len + jae .L320_or_more8x + cmp \$256,$len + jae .L256_or_more8x + cmp \$192,$len + jae .L192_or_more8x + cmp \$128,$len + jae .L128_or_more8x + cmp \$64,$len + jae .L64_or_more8x + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L64_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + je .Ldone8x + + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + vmovdqa $xc0,0x00(%rsp) + lea 0x40($out),$out # out+=64*1 + sub \$64,$len # len-=64*1 + vmovdqa $xd0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L128_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + je .Ldone8x + + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + vmovdqa $xa1,0x00(%rsp) + lea 0x80($out),$out # out+=64*2 + sub \$128,$len # len-=64*2 + vmovdqa $xb1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L192_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + je .Ldone8x + + lea 0xc0($inp),$inp # inp+=64*3 + xor %r9,%r9 + vmovdqa $xc1,0x00(%rsp) + lea 0xc0($out),$out # out+=64*3 + sub \$192,$len # len-=64*3 + vmovdqa $xd1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L256_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + je .Ldone8x + + lea 0x100($inp),$inp # inp+=64*4 + xor %r9,%r9 + vmovdqa $xa2,0x00(%rsp) + lea 0x100($out),$out # out+=64*4 + sub \$256,$len # len-=64*4 + vmovdqa $xb2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L320_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + je .Ldone8x + + lea 0x140($inp),$inp # inp+=64*5 + xor %r9,%r9 + vmovdqa $xc2,0x00(%rsp) + lea 0x140($out),$out # out+=64*5 + sub \$320,$len # len-=64*5 + vmovdqa $xd2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L384_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + je .Ldone8x + + lea 0x180($inp),$inp # inp+=64*6 + xor %r9,%r9 + vmovdqa $xa3,0x00(%rsp) + lea 0x180($out),$out # out+=64*6 + sub \$384,$len # len-=64*6 + vmovdqa $xb3,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L448_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vpxor 0x180($inp),$xa3,$xa3 + vpxor 0x1a0($inp),$xb3,$xb3 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + vmovdqu $xa3,0x180($out) + vmovdqu $xb3,0x1a0($out) + je .Ldone8x + + lea 0x1c0($inp),$inp # inp+=64*7 + xor %r9,%r9 + vmovdqa $xc3,0x00(%rsp) + lea 0x1c0($out),$out # out+=64*7 + sub \$448,$len # len-=64*7 + vmovdqa $xd3,0x20(%rsp) + +.Loop_tail8x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8x + +.Ldone8x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8x_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx2"); +if($kernel) { + $code .= "#endif\n"; +} +} + +######################################################################## +# AVX512 code paths +if ($avx>2) { +# This one handles shorter inputs... +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20)); +my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +sub vpxord() # size optimization +{ my $opcode = "vpxor"; # adhere to vpxor when possible + + foreach (@_) { + if (/%([zy])mm([0-9]+)/ && ($1 eq "z" || $2>=16)) { + $opcode = "vpxord"; + last; + } + } + + $code .= "\t$opcode\t".join(',',reverse @_)."\n"; +} + +sub AVX512ROUND { # critical path is 14 "SIMD ticks" per round + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,16); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,12); + + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,8); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,7); +} + +my $xframe = $win64 ? 32+8 : 8; + +&declare_function("chacha20_avx512", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$512,$len + ja .Lchacha20_16x + + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512_body: +___ +$code.=<<___; + vbroadcasti32x4 .Lsigma(%rip),$a + vbroadcasti32x4 ($key),$b + vbroadcasti32x4 16($key),$c + vbroadcasti32x4 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Lfourz(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 16 +.Loop_outer_avx512: + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512 + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$1,$a,$t0 + vextracti32x4 \$1,$b,$t1 + vextracti32x4 \$1,$c,$t2 + vextracti32x4 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$2,$a,$t0 + vextracti32x4 \$2,$b,$t1 + vextracti32x4 \$2,$c,$t2 + vextracti32x4 \$2,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$3,$a,$t0 + vextracti32x4 \$3,$b,$t1 + vextracti32x4 \$3,$c,$t2 + vextracti32x4 \$3,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jnz .Loop_outer_avx512 + + jmp .Ldone_avx512 + +.align 16 +.Ltail64_avx512: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512 + +.align 16 +.Ltail_avx512: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512 + + vmovdqu32 $a_,0x00(%rsp) + +.Ldone_avx512: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512"); + +map(s/%z/%y/, $a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz); + +&declare_function("chacha20_avx512vl", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512vl: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$128,$len + ja .Lchacha20_8xvl + + sub \$64+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512vl_body: +___ +$code.=<<___; + vbroadcasti128 .Lsigma(%rip),$a + vbroadcasti128 ($key),$b + vbroadcasti128 16($key),$c + vbroadcasti128 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Ltwoy(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 16 +.Loop_outer_avx512vl: + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 32 +.Loop_avx512vl: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512vl"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512vl + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512vl + + vextracti128 \$1,$a,$t0 + vextracti128 \$1,$b,$t1 + vextracti128 \$1,$c,$t2 + vextracti128 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512vl + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + jnz .Loop_outer_avx512vl + + jmp .Ldone_avx512vl + +.align 16 +.Ltail64_avx512vl: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512vl + +.align 16 +.Ltail_avx512vl: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512vl: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512vl + + vmovdqu32 $a_,0x00(%rsp) + vmovdqu32 $a_,0x20(%rsp) + +.Ldone_avx512vl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512vl_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512vl"); + +# This one handles longer inputs... + +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%zmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +my @key=map("%zmm$_",(16..31)); +my ($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +sub AVX512_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],16)", + "&vprold (@x[$d1],@x[$d1],16)", + "&vprold (@x[$d2],@x[$d2],16)", + "&vprold (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],12)", + "&vprold (@x[$b1],@x[$b1],12)", + "&vprold (@x[$b2],@x[$b2],12)", + "&vprold (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],8)", + "&vprold (@x[$d1],@x[$d1],8)", + "&vprold (@x[$d2],@x[$d2],8)", + "&vprold (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],7)", + "&vprold (@x[$b1],@x[$b1],7)", + "&vprold (@x[$b2],@x[$b2],7)", + "&vprold (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_16x,\@function,5 +.align 32 +chacha20_16x: +.cfi_startproc +.Lchacha20_16x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L16x_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti32x4 (%r9),$xa3 # key[0] + vbroadcasti32x4 ($key),$xb3 # key[1] + vbroadcasti32x4 16($key),$xc3 # key[2] + vbroadcasti32x4 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincz(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop_outer16x: + vpbroadcastd 0(%r9),$xa0 # reload key + vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Lsixteen(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop16x: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop16x + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0x44,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xb0,$xa0,$xb0 + vshufi32x4 \$0x44,$xb1,$xa1,$xa0 + vshufi32x4 \$0xee,$xb1,$xa1,$xb1 + vshufi32x4 \$0x44,$xb2,$xa2,$xa1 + vshufi32x4 \$0xee,$xb2,$xa2,$xb2 + vshufi32x4 \$0x44,$xb3,$xa3,$xa2 + vshufi32x4 \$0xee,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vshufi32x4 \$0x44,$xd0,$xc0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xd0,$xc0,$xd0 + vshufi32x4 \$0x44,$xd1,$xc1,$xc0 + vshufi32x4 \$0xee,$xd1,$xc1,$xd1 + vshufi32x4 \$0x44,$xd2,$xc2,$xc1 + vshufi32x4 \$0xee,$xd2,$xc2,$xd2 + vshufi32x4 \$0x44,$xd3,$xc3,$xc2 + vshufi32x4 \$0xee,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); +$code.=<<___; + vshufi32x4 \$0x88,$xc0,$xa0,$xt0 # "de-interlace" further + vshufi32x4 \$0xdd,$xc0,$xa0,$xa0 + vshufi32x4 \$0x88,$xd0,$xb0,$xc0 + vshufi32x4 \$0xdd,$xd0,$xb0,$xd0 + vshufi32x4 \$0x88,$xc1,$xa1,$xt1 + vshufi32x4 \$0xdd,$xc1,$xa1,$xa1 + vshufi32x4 \$0x88,$xd1,$xb1,$xc1 + vshufi32x4 \$0xdd,$xd1,$xb1,$xd1 + vshufi32x4 \$0x88,$xc2,$xa2,$xt2 + vshufi32x4 \$0xdd,$xc2,$xa2,$xa2 + vshufi32x4 \$0x88,$xd2,$xb2,$xc2 + vshufi32x4 \$0xdd,$xd2,$xb2,$xd2 + vshufi32x4 \$0x88,$xc3,$xa3,$xt3 + vshufi32x4 \$0xdd,$xc3,$xa3,$xa3 + vshufi32x4 \$0x88,$xd3,$xb3,$xc3 + vshufi32x4 \$0xdd,$xd3,$xb3,$xd3 +___ + ($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3)= + ($xt0,$xt1,$xt2,$xt3,$xa0,$xa1,$xa2,$xa3); + + ($xa0,$xb0,$xc0,$xd0, $xa1,$xb1,$xc1,$xd1, + $xa2,$xb2,$xc2,$xd2, $xa3,$xb3,$xc3,$xd3) = + ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +$code.=<<___; + cmp \$64*16,$len + jb .Ltail16x + + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxord 0x40($inp),$xb0,$xb0 + vpxord 0x80($inp),$xc0,$xc0 + vpxord 0xc0($inp),$xd0,$xd0 + vmovdqu32 $xa0,0x00($out) + vmovdqu32 $xb0,0x40($out) + vmovdqu32 $xc0,0x80($out) + vmovdqu32 $xd0,0xc0($out) + + vpxord 0x100($inp),$xa1,$xa1 + vpxord 0x140($inp),$xb1,$xb1 + vpxord 0x180($inp),$xc1,$xc1 + vpxord 0x1c0($inp),$xd1,$xd1 + vmovdqu32 $xa1,0x100($out) + vmovdqu32 $xb1,0x140($out) + vmovdqu32 $xc1,0x180($out) + vmovdqu32 $xd1,0x1c0($out) + + vpxord 0x200($inp),$xa2,$xa2 + vpxord 0x240($inp),$xb2,$xb2 + vpxord 0x280($inp),$xc2,$xc2 + vpxord 0x2c0($inp),$xd2,$xd2 + vmovdqu32 $xa2,0x200($out) + vmovdqu32 $xb2,0x240($out) + vmovdqu32 $xc2,0x280($out) + vmovdqu32 $xd2,0x2c0($out) + + vpxord 0x300($inp),$xa3,$xa3 + vpxord 0x340($inp),$xb3,$xb3 + vpxord 0x380($inp),$xc3,$xc3 + vpxord 0x3c0($inp),$xd3,$xd3 + lea 0x400($inp),$inp + vmovdqu32 $xa3,0x300($out) + vmovdqu32 $xb3,0x340($out) + vmovdqu32 $xc3,0x380($out) + vmovdqu32 $xd3,0x3c0($out) + lea 0x400($out),$out + + sub \$64*16,$len + jnz .Loop_outer16x + + jmp .Ldone16x + +.align 32 +.Ltail16x: + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_16x + vpxord ($inp),$xa0,$xa0 # xor with input + vmovdqu32 $xa0,($out,$inp) + je .Ldone16x + vmovdqa32 $xb0,$xa0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_16x + vpxord ($inp),$xb0,$xb0 + vmovdqu32 $xb0,($out,$inp) + je .Ldone16x + vmovdqa32 $xc0,$xa0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_16x + vpxord ($inp),$xc0,$xc0 + vmovdqu32 $xc0,($out,$inp) + je .Ldone16x + vmovdqa32 $xd0,$xa0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_16x + vpxord ($inp),$xd0,$xd0 + vmovdqu32 $xd0,($out,$inp) + je .Ldone16x + vmovdqa32 $xa1,$xa0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_16x + vpxord ($inp),$xa1,$xa1 + vmovdqu32 $xa1,($out,$inp) + je .Ldone16x + vmovdqa32 $xb1,$xa0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_16x + vpxord ($inp),$xb1,$xb1 + vmovdqu32 $xb1,($out,$inp) + je .Ldone16x + vmovdqa32 $xc1,$xa0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_16x + vpxord ($inp),$xc1,$xc1 + vmovdqu32 $xc1,($out,$inp) + je .Ldone16x + vmovdqa32 $xd1,$xa0 + lea 64($inp),$inp + + cmp \$64*8,$len + jb .Less_than_64_16x + vpxord ($inp),$xd1,$xd1 + vmovdqu32 $xd1,($out,$inp) + je .Ldone16x + vmovdqa32 $xa2,$xa0 + lea 64($inp),$inp + + cmp \$64*9,$len + jb .Less_than_64_16x + vpxord ($inp),$xa2,$xa2 + vmovdqu32 $xa2,($out,$inp) + je .Ldone16x + vmovdqa32 $xb2,$xa0 + lea 64($inp),$inp + + cmp \$64*10,$len + jb .Less_than_64_16x + vpxord ($inp),$xb2,$xb2 + vmovdqu32 $xb2,($out,$inp) + je .Ldone16x + vmovdqa32 $xc2,$xa0 + lea 64($inp),$inp + + cmp \$64*11,$len + jb .Less_than_64_16x + vpxord ($inp),$xc2,$xc2 + vmovdqu32 $xc2,($out,$inp) + je .Ldone16x + vmovdqa32 $xd2,$xa0 + lea 64($inp),$inp + + cmp \$64*12,$len + jb .Less_than_64_16x + vpxord ($inp),$xd2,$xd2 + vmovdqu32 $xd2,($out,$inp) + je .Ldone16x + vmovdqa32 $xa3,$xa0 + lea 64($inp),$inp + + cmp \$64*13,$len + jb .Less_than_64_16x + vpxord ($inp),$xa3,$xa3 + vmovdqu32 $xa3,($out,$inp) + je .Ldone16x + vmovdqa32 $xb3,$xa0 + lea 64($inp),$inp + + cmp \$64*14,$len + jb .Less_than_64_16x + vpxord ($inp),$xb3,$xb3 + vmovdqu32 $xb3,($out,$inp) + je .Ldone16x + vmovdqa32 $xc3,$xa0 + lea 64($inp),$inp + + cmp \$64*15,$len + jb .Less_than_64_16x + vpxord ($inp),$xc3,$xc3 + vmovdqu32 $xc3,($out,$inp) + je .Ldone16x + vmovdqa32 $xd3,$xa0 + lea 64($inp),$inp + +.Less_than_64_16x: + vmovdqa32 $xa0,0x00(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail16x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail16x + + vpxord $xa0,$xa0,$xa0 + vmovdqa32 $xa0,0(%rsp) + +.Ldone16x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L16x_epilogue: + ret +.cfi_endproc +.size chacha20_16x,.-chacha20_16x +___ + +# switch to %ymm domain +($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%ymm$_",(0..15)); +@xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +@key=map("%ymm$_",(16..31)); +($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +$code.=<<___; +.type chacha20_8xvl,\@function,5 +.align 32 +chacha20_8xvl: +.cfi_startproc +.Lchacha20_8xvl: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8xvl_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti128 (%r9),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xc3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop_outer8xvl: + #vpbroadcastd 0(%r9),$xa0 # reload key + #vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Leight(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop8xvl: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8xvl + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$3,$xb0,$xa0,$xb0 + vshufi32x4 \$0,$xb1,$xa1,$xa0 + vshufi32x4 \$3,$xb1,$xa1,$xb1 + vshufi32x4 \$0,$xb2,$xa2,$xa1 + vshufi32x4 \$3,$xb2,$xa2,$xb2 + vshufi32x4 \$0,$xb3,$xa3,$xa2 + vshufi32x4 \$3,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); +$code.=<<___; + cmp \$64*8,$len + jb .Ltail8xvl + + mov \$0x80,%eax # size optimization + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea ($out,%rax),$out # size optimization + + vpbroadcastd 0(%r9),%ymm0 # reload key + vpbroadcastd 4(%r9),%ymm1 + + sub \$64*8,$len + jnz .Loop_outer8xvl + + jmp .Ldone8xvl + +.align 32 +.Ltail8xvl: + vmovdqa64 $xa0,%ymm8 # size optimization +___ +$xa0 = "%ymm8"; +$code.=<<___; + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out,$inp) + vmovdqu $xb0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc0,$xa0 + vmovdqa $xd0,$xb0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc0,$xc0 + vpxor 0x20($inp),$xd0,$xd0 + vmovdqu $xc0,0x00($out,$inp) + vmovdqu $xd0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa1,$xa0 + vmovdqa $xb1,$xb0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vmovdqu $xa1,0x00($out,$inp) + vmovdqu $xb1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc1,$xa0 + vmovdqa $xd1,$xb0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc1,$xc1 + vpxor 0x20($inp),$xd1,$xd1 + vmovdqu $xc1,0x00($out,$inp) + vmovdqu $xd1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa32 $xa2,$xa0 + vmovdqa $xb2,$xb0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_8xvl + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vmovdqu32 $xa2,0x00($out,$inp) + vmovdqu $xb2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc2,$xa0 + vmovdqa $xd2,$xb0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc2,$xc2 + vpxor 0x20($inp),$xd2,$xd2 + vmovdqu $xc2,0x00($out,$inp) + vmovdqu $xd2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa3,$xa0 + vmovdqa $xb3,$xb0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vmovdqu $xa3,0x00($out,$inp) + vmovdqu $xb3,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc3,$xa0 + vmovdqa $xd3,$xb0 + lea 64($inp),$inp + +.Less_than_64_8xvl: + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail8xvl: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8xvl + + vpxor $xa0,$xa0,$xa0 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xa0,0x20(%rsp) + +.Ldone8xvl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8xvl_epilogue: + ret +.cfi_endproc +.size chacha20_8xvl,.-chacha20_8xvl +___ +if($kernel) { + $code .= "#endif\n"; +} +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + lea .Lctr32_body(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea .Lno_data(%rip),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 64+24+48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.type simd_handler,\@abi-omnipotent +.align 16 +simd_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipR9 + + mov 4(%r11),%r10d # HandlerData[1] + mov 8(%r11),%ecx # HandlerData[2] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + neg %rcx + lea -8(%rax,%rcx),%rsi + lea 512($context),%rdi # &context.Xmm6 + neg %ecx + shr \$3,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lcommon_seh_tail +.size simd_handler,.-simd_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_chacha20_ctr32 + .rva .LSEH_end_chacha20_ctr32 + .rva .LSEH_info_chacha20_ctr32 + + .rva .LSEH_begin_chacha20_ssse3 + .rva .LSEH_end_chacha20_ssse3 + .rva .LSEH_info_chacha20_ssse3 + + .rva .LSEH_begin_chacha20_128 + .rva .LSEH_end_chacha20_128 + .rva .LSEH_info_chacha20_128 + + .rva .LSEH_begin_chacha20_4x + .rva .LSEH_end_chacha20_4x + .rva .LSEH_info_chacha20_4x +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_chacha20_xop + .rva .LSEH_end_chacha20_xop + .rva .LSEH_info_chacha20_xop +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_chacha20_avx2 + .rva .LSEH_end_chacha20_avx2 + .rva .LSEH_info_chacha20_avx2 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_chacha20_avx512 + .rva .LSEH_end_chacha20_avx512 + .rva .LSEH_info_chacha20_avx512 + + .rva .LSEH_begin_chacha20_avx512vl + .rva .LSEH_end_chacha20_avx512vl + .rva .LSEH_info_chacha20_avx512vl + + .rva .LSEH_begin_chacha20_16x + .rva .LSEH_end_chacha20_16x + .rva .LSEH_info_chacha20_16x + + .rva .LSEH_begin_chacha20_8xvl + .rva .LSEH_end_chacha20_8xvl + .rva .LSEH_info_chacha20_8xvl +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_chacha20_ctr32: + .byte 9,0,0,0 + .rva se_handler + +.LSEH_info_chacha20_ssse3: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lssse3_body,.Lssse3_epilogue + .long 0x20,0 + +.LSEH_info_chacha20_128: + .byte 9,0,0,0 + .rva simd_handler + .rva .L128_body,.L128_epilogue + .long 0x60,0 + +.LSEH_info_chacha20_4x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4x_body,.L4x_epilogue + .long 0xa0,0 +___ +$code.=<<___ if ($avx); +.LSEH_info_chacha20_xop: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4xop_body,.L4xop_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>1); +.LSEH_info_chacha20_avx2: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8x_body,.L8x_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>2); +.LSEH_info_chacha20_avx512: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512_body,.Lavx512_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_avx512vl: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512vl_body,.Lavx512vl_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_16x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L16x_body,.L16x_epilogue # HandlerData[] + .long 0xa0,0 + +.LSEH_info_chacha20_8xvl: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8xvl_body,.L8xvl_epilogue # HandlerData[] + .long 0xa0,0 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/%x#%[yz]/%x/g; # "down-shift" + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} + +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20.c b/net/wireguard/crypto/zinc/chacha20/chacha20.c new file mode 100644 index 0000000000000..b4763c81e0826 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the ChaCha20 stream cipher. + * + * Information: https://cr.yp.to/chacha.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include // For crypto_xor_cpy. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "chacha20-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "chacha20-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) +#include "chacha20-mips-glue.c" +#else +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + return false; +} +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} +#endif + +#define QUARTER_ROUND(x, a, b, c, d) ( \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 16), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 12), \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 8), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 7) \ +) + +#define C(i, j) (i * 4 + j) + +#define DOUBLE_ROUND(x) ( \ + /* Column Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 0), C(2, 0), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 1), C(2, 1), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 2), C(2, 2), C(3, 2)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 3), C(2, 3), C(3, 3)), \ + /* Diagonal Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 1), C(2, 2), C(3, 3)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 2), C(2, 3), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 3), C(2, 0), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 0), C(2, 1), C(3, 2)) \ +) + +#define TWENTY_ROUNDS(x) ( \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x) \ +) + +static void chacha20_block_generic(struct chacha20_ctx *ctx, __le32 *stream) +{ + u32 x[CHACHA20_BLOCK_WORDS]; + int i; + + for (i = 0; i < ARRAY_SIZE(x); ++i) + x[i] = ctx->state[i]; + + TWENTY_ROUNDS(x); + + for (i = 0; i < ARRAY_SIZE(x); ++i) + stream[i] = cpu_to_le32(x[i] + ctx->state[i]); + + ctx->counter[0] += 1; +} + +static void chacha20_generic(struct chacha20_ctx *ctx, u8 *out, const u8 *in, + u32 len) +{ + __le32 buf[CHACHA20_BLOCK_WORDS]; + + while (len >= CHACHA20_BLOCK_SIZE) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, CHACHA20_BLOCK_SIZE); + len -= CHACHA20_BLOCK_SIZE; + out += CHACHA20_BLOCK_SIZE; + in += CHACHA20_BLOCK_SIZE; + } + if (len) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, len); + } +} + +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context) +{ + if (!chacha20_arch(ctx, dst, src, len, simd_context)) + chacha20_generic(ctx, dst, src, len); +} +EXPORT_SYMBOL(chacha20); + +static void hchacha20_generic(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE]) +{ + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + + TWENTY_ROUNDS(x); + + memcpy(derived_key + 0, x + 0, sizeof(u32) * 4); + memcpy(derived_key + 4, x + 12, sizeof(u32) * 4); +} + +/* Derived key should be 32-bit aligned */ +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context) +{ + if (!hchacha20_arch(derived_key, nonce, key, simd_context)) + hchacha20_generic(derived_key, nonce, key); +} +EXPORT_SYMBOL(hchacha20); + +#include "../selftest/chacha20.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + chacha20_fpu_init(); + if (!selftest_run("chacha20", chacha20_selftest, chacha20_nobs, + ARRAY_SIZE(chacha20_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20poly1305.c b/net/wireguard/crypto/zinc/chacha20poly1305.c new file mode 100644 index 0000000000000..571a64e95a83a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20poly1305.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the ChaCha20Poly1305 AEAD construction. + * + * Information: https://tools.ietf.org/html/rfc8439 + */ + +#include +#include +#include +#include "selftest/run.h" + +#include +#include +#include +#include +#include // For blkcipher_walk. + +static const u8 pad0[CHACHA20_BLOCK_SIZE] = { 0 }; + +static inline void +__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + chacha20(&chacha20_state, dst, src, src_len, simd_context); + + poly1305_update(&poly1305_state, dst, src_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, dst + src_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + + simd_get(&simd_context); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key, + &simd_context); + simd_put(&simd_context); +} +EXPORT_SYMBOL(chacha20poly1305_encrypt); + +bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, + const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + poly1305_update(&poly1305_state, miter.addr, + min_t(size_t, sl, miter.length), simd_context); + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) + poly1305_final(&poly1305_state, miter.addr + miter.length + sl, + simd_context); + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.mac, simd_context); + scatterwalk_map_and_copy(b.mac, src, src_len, sizeof(b.mac), 1); + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return true; +} +EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); + +static inline bool +__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + int ret; + size_t dst_len; + union { + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return false; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + dst_len = src_len - POLY1305_MAC_SIZE; + poly1305_update(&poly1305_state, src, dst_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - dst_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(dst_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, b.mac, simd_context); + + ret = crypto_memneq(b.mac, src + dst_len, POLY1305_MAC_SIZE); + if (likely(!ret)) + chacha20(&chacha20_state, dst, src, dst_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + + return !ret; +} + +bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context, ret; + + simd_get(&simd_context); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce, + key, &simd_context); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt); + +bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, + size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + struct { + u8 read_mac[POLY1305_MAC_SIZE]; + u8 computed_mac[POLY1305_MAC_SIZE]; + }; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + bool ret = false; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return ret; + src_len -= POLY1305_MAC_SIZE; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + poly1305_update(&poly1305_state, addr, length, simd_context); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + ret = !crypto_memneq(b.computed_mac, + miter.addr + miter.length + sl, + POLY1305_MAC_SIZE); + } + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + scatterwalk_map_and_copy(b.read_mac, src, src_len, + sizeof(b.read_mac), 0); + ret = !crypto_memneq(b.read_mac, b.computed_mac, + POLY1305_MAC_SIZE); + + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); +} +EXPORT_SYMBOL(xchacha20poly1305_encrypt); + +bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + bool ret; + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(xchacha20poly1305_decrypt); + +#include "selftest/chacha20poly1305.c" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!selftest_run("chacha20poly1305", chacha20poly1305_selftest, + NULL, 0)) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c new file mode 100644 index 0000000000000..e0c5a5d297c01 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); + +static bool curve25519_use_neon __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { &curve25519_use_neon }; +static void __init curve25519_fpu_init(void) +{ + curve25519_use_neon = elf_hwcap & HWCAP_NEON; +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + simd_context_t simd_context; + bool used_arch = false; + + simd_get(&simd_context); + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && curve25519_use_neon && + simd_use(&simd_context)) { + curve25519_neon(mypublic, secret, basepoint); + used_arch = true; + } + simd_put(&simd_context); + return used_arch; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S new file mode 100644 index 0000000000000..8eca8a11ef28f --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S @@ -0,0 +1,2064 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This + * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been + * manually reworked for use in kernel space. + */ + +#if defined(CONFIG_KERNEL_MODE_NEON) && !defined(__ARMEB__) +#include + +.text +.fpu neon +.arch armv7-a +.align 4 + +SYM_FUNC_START(curve25519_neon) + push {r4-r11, lr} + mov ip, sp + sub r3, sp, #704 + and r3, r3, #0xfffffff0 + mov sp, r3 + movw r4, #0 + movw r5, #254 + vmov.i32 q0, #1 + vshr.u64 q1, q0, #7 + vshr.u64 q0, q0, #8 + vmov.i32 d4, #19 + vmov.i32 d5, #38 + add r6, sp, #480 + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128] + add r6, r3, #0 + vmov.i32 q2, #0 + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 d4, [r6, : 64] + add r6, r3, #0 + movw r7, #960 + sub r7, r7, #2 + neg r7, r7 + sub r7, r7, r7, LSL #7 + str r7, [r6] + add r6, sp, #672 + vld1.8 {d4-d5}, [r1]! + vld1.8 {d6-d7}, [r1] + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d6-d7}, [r6, : 128] + sub r1, r6, #16 + ldrb r6, [r1] + and r6, r6, #248 + strb r6, [r1] + ldrb r6, [r1, #31] + and r6, r6, #127 + orr r6, r6, #64 + strb r6, [r1, #31] + vmov.i64 q2, #0xffffffff + vshr.u64 q3, q2, #7 + vshr.u64 q2, q2, #6 + vld1.8 {d8}, [r2] + vld1.8 {d10}, [r2] + add r2, r2, #6 + vld1.8 {d12}, [r2] + vld1.8 {d14}, [r2] + add r2, r2, #6 + vld1.8 {d16}, [r2] + add r2, r2, #4 + vld1.8 {d18}, [r2] + vld1.8 {d20}, [r2] + add r2, r2, #6 + vld1.8 {d22}, [r2] + add r2, r2, #2 + vld1.8 {d24}, [r2] + vld1.8 {d26}, [r2] + vshr.u64 q5, q5, #26 + vshr.u64 q6, q6, #3 + vshr.u64 q7, q7, #29 + vshr.u64 q8, q8, #6 + vshr.u64 q10, q10, #25 + vshr.u64 q11, q11, #3 + vshr.u64 q12, q12, #12 + vshr.u64 q13, q13, #38 + vand q4, q4, q2 + vand q6, q6, q2 + vand q8, q8, q2 + vand q10, q10, q2 + vand q2, q12, q2 + vand q5, q5, q3 + vand q7, q7, q3 + vand q9, q9, q3 + vand q11, q11, q3 + vand q3, q13, q3 + add r2, r3, #48 + vadd.i64 q12, q4, q1 + vadd.i64 q13, q10, q1 + vshr.s64 q12, q12, #26 + vshr.s64 q13, q13, #26 + vadd.i64 q5, q5, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q14, q5, q0 + vadd.i64 q11, q11, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q11, q0 + vsub.i64 q4, q4, q12 + vshr.s64 q12, q14, #25 + vsub.i64 q10, q10, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q14, q6, q1 + vadd.i64 q2, q2, q13 + vsub.i64 q5, q5, q12 + vshr.s64 q12, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q1 + vadd.i64 q7, q7, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q15, q7, q0 + vsub.i64 q11, q11, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q12 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q3, q0 + vadd.i64 q8, q8, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q15, q8, q1 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q7, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q9, q9, q12 + vtrn.32 d12, d14 + vshl.i64 q12, q12, #26 + vtrn.32 d13, d15 + vadd.i64 q0, q9, q0 + vadd.i64 q4, q4, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q6, q13, #4 + vsub.i64 q7, q8, q12 + vshr.s64 q0, q0, #25 + vadd.i64 q4, q4, q6 + vadd.i64 q6, q10, q0 + vshl.i64 q0, q0, #25 + vadd.i64 q8, q6, q1 + vadd.i64 q4, q4, q13 + vshl.i64 q10, q13, #25 + vadd.i64 q1, q4, q1 + vsub.i64 q0, q9, q0 + vshr.s64 q8, q8, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d14, d0 + vshr.s64 q1, q1, #26 + vtrn.32 d15, d1 + vadd.i64 q0, q11, q8 + vst1.8 d14, [r2, : 64] + vshl.i64 q7, q8, #26 + vadd.i64 q5, q5, q1 + vtrn.32 d4, d6 + vshl.i64 q1, q1, #26 + vtrn.32 d5, d7 + vsub.i64 q3, q6, q7 + add r2, r2, #16 + vsub.i64 q1, q4, q1 + vst1.8 d4, [r2, : 64] + vtrn.32 d6, d0 + vtrn.32 d7, d1 + sub r2, r2, #8 + vtrn.32 d2, d10 + vtrn.32 d3, d11 + vst1.8 d6, [r2, : 64] + sub r2, r2, #24 + vst1.8 d2, [r2, : 64] + add r2, r3, #96 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #144 + vmov.i32 q0, #0 + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #240 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #48 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d4, [r6, : 64] +.Lmainloop: + mov r2, r5, LSR #3 + and r6, r5, #7 + ldrb r2, [r1, r2] + mov r2, r2, LSR r6 + and r2, r2, #1 + str r5, [sp, #456] + eor r4, r4, r2 + str r2, [sp, #460] + neg r2, r4 + add r4, r3, #96 + add r5, r3, #192 + add r6, r3, #144 + vld1.8 {d8-d9}, [r4, : 128]! + add r7, r3, #240 + vld1.8 {d10-d11}, [r5, : 128]! + veor q6, q4, q5 + vld1.8 {d14-d15}, [r6, : 128]! + vdup.i32 q8, r2 + vld1.8 {d18-d19}, [r7, : 128]! + veor q10, q7, q9 + vld1.8 {d22-d23}, [r4, : 128]! + vand q6, q6, q8 + vld1.8 {d24-d25}, [r5, : 128]! + vand q10, q10, q8 + vld1.8 {d26-d27}, [r6, : 128]! + veor q4, q4, q6 + vld1.8 {d28-d29}, [r7, : 128]! + veor q5, q5, q6 + vld1.8 {d0}, [r4, : 64] + veor q6, q7, q10 + vld1.8 {d2}, [r5, : 64] + veor q7, q9, q10 + vld1.8 {d4}, [r6, : 64] + veor q9, q11, q12 + vld1.8 {d6}, [r7, : 64] + veor q10, q0, q1 + sub r2, r4, #32 + vand q9, q9, q8 + sub r4, r5, #32 + vand q10, q10, q8 + sub r5, r6, #32 + veor q11, q11, q9 + sub r6, r7, #32 + veor q0, q0, q10 + veor q9, q12, q9 + veor q1, q1, q10 + veor q10, q13, q14 + veor q12, q2, q3 + vand q10, q10, q8 + vand q8, q12, q8 + veor q12, q13, q10 + veor q2, q2, q8 + veor q10, q14, q10 + veor q3, q3, q8 + vadd.i32 q8, q4, q6 + vsub.i32 q4, q4, q6 + vst1.8 {d16-d17}, [r2, : 128]! + vadd.i32 q6, q11, q12 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q4, q11, q12 + vst1.8 {d12-d13}, [r2, : 128]! + vadd.i32 q6, q0, q2 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q0, q0, q2 + vst1.8 d12, [r2, : 64] + vadd.i32 q2, q5, q7 + vst1.8 d0, [r5, : 64] + vsub.i32 q0, q5, q7 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q9, q10 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q9, q10 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q1, q3 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q1, q3 + vst1.8 d4, [r4, : 64] + vst1.8 d0, [r6, : 64] + add r2, sp, #512 + add r4, r3, #96 + add r5, r3, #144 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #288 + vshl.i64 q12, q12, #25 + add r4, r3, #336 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #240 + add r4, r3, #96 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #144 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #192 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128] + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #144 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, r3, #288 + add r4, r3, #336 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vsub.i32 q1, q1, q2 + add r5, r3, #240 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vsub.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #144 + add r4, r3, #96 + add r5, r3, #144 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q2, q0, q1 + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d6-d7}, [r4, : 128]! + vsub.i32 q4, q1, q3 + vadd.i32 q1, q1, q3 + vld1.8 {d6}, [r2, : 64] + vld1.8 {d10}, [r4, : 64] + vsub.i32 q6, q3, q5 + vadd.i32 q3, q3, q5 + vst1.8 {d4-d5}, [r5, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d8-d9}, [r5, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d12, [r5, : 64] + vst1.8 d6, [r6, : 64] + add r2, r3, #0 + add r4, r3, #240 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #336 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #288 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #288 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, sp, #512 + add r4, r3, #144 + add r5, r3, #192 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #144 + vshl.i64 q12, q12, #25 + add r4, r3, #192 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #336 + add r4, r3, #288 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q1, q1, q2 + add r5, r3, #288 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vadd.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #48 + add r4, r3, #144 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #288 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #240 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #240 + vshl.i64 q7, q7, #25 + add r4, r3, #144 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + ldr r2, [sp, #456] + ldr r4, [sp, #460] + subs r5, r2, #1 + bge .Lmainloop + add r1, r3, #144 + add r2, r3, #336 + vld1.8 {d0-d1}, [r1, : 128]! + vld1.8 {d2-d3}, [r1, : 128]! + vld1.8 {d4}, [r1, : 64] + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 d4, [r2, : 64] + movw r1, #0 +.Linvertloop: + add r2, r3, #144 + movw r4, #0 + movw r5, #2 + cmp r1, #1 + moveq r5, #1 + addeq r2, r3, #336 + addeq r4, r3, #48 + cmp r1, #2 + moveq r5, #1 + addeq r2, r3, #48 + cmp r1, #3 + moveq r5, #5 + addeq r4, r3, #336 + cmp r1, #4 + moveq r5, #10 + cmp r1, #5 + moveq r5, #20 + cmp r1, #6 + moveq r5, #10 + addeq r2, r3, #336 + addeq r4, r3, #336 + cmp r1, #7 + moveq r5, #50 + cmp r1, #8 + moveq r5, #100 + cmp r1, #9 + moveq r5, #50 + addeq r2, r3, #336 + cmp r1, #10 + moveq r5, #5 + addeq r2, r3, #48 + cmp r1, #11 + moveq r5, #0 + addeq r2, r3, #96 + add r6, r3, #144 + add r7, r3, #288 + vld1.8 {d0-d1}, [r6, : 128]! + vld1.8 {d2-d3}, [r6, : 128]! + vld1.8 {d4}, [r6, : 64] + vst1.8 {d0-d1}, [r7, : 128]! + vst1.8 {d2-d3}, [r7, : 128]! + vst1.8 d4, [r7, : 64] + cmp r5, #0 + beq .Lskipsquaringloop +.Lsquaringloop: + add r6, r3, #288 + add r7, r3, #288 + add r8, r3, #288 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r7, : 128]! + vld1.8 {d6-d7}, [r7, : 128]! + vld1.8 {d9}, [r7, : 64] + vld1.8 {d10-d11}, [r6, : 128]! + add r7, sp, #384 + vld1.8 {d12-d13}, [r6, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r6, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r7, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r7, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r7, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r7, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r7, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r6, r7, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r6, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r6, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r6, r6, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r6, : 64] + sub r6, r6, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r7, sp, #480 + vld1.8 {d14-d15}, [r7, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r6, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r6, : 64]! + add r6, sp, #496 + vld1.8 {d20-d21}, [r6, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r6, r8, #8 + vst1.8 d0, [r6, : 64] + vsub.i64 d19, d19, d9 + add r6, r6, #16 + vst1.8 d16, [r6, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r6, r6, #8 + vst1.8 d6, [r6, : 64] + add r6, r6, #16 + vst1.8 d18, [r6, : 64] + vzip.i32 q2, q5 + sub r6, r6, #32 + vst1.8 d4, [r6, : 64] + subs r5, r5, #1 + bhi .Lsquaringloop +.Lskipsquaringloop: + mov r2, r2 + add r5, r3, #288 + add r6, r3, #144 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r5, : 128]! + vld1.8 {d6-d7}, [r5, : 128]! + vld1.8 {d9}, [r5, : 64] + vld1.8 {d10-d11}, [r2, : 128]! + add r5, sp, #384 + vld1.8 {d12-d13}, [r2, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r2, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r5, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r5, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r5, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r5, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r5, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r2, r5, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r2, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r2, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r2, r2, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r2, : 64] + sub r2, r2, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r5, sp, #480 + vld1.8 {d14-d15}, [r5, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r2, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r2, : 64]! + add r2, sp, #496 + vld1.8 {d20-d21}, [r2, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r2, r6, #8 + vst1.8 d0, [r2, : 64] + vsub.i64 d19, d19, d9 + add r2, r2, #16 + vst1.8 d16, [r2, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r2, r2, #8 + vst1.8 d6, [r2, : 64] + add r2, r2, #16 + vst1.8 d18, [r2, : 64] + vzip.i32 q2, q5 + sub r2, r2, #32 + vst1.8 d4, [r2, : 64] + cmp r4, #0 + beq .Lskippostcopy + add r2, r3, #144 + mov r4, r4 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskippostcopy: + cmp r1, #1 + bne .Lskipfinalcopy + add r2, r3, #288 + add r4, r3, #144 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskipfinalcopy: + add r1, r1, #1 + cmp r1, #12 + blo .Linvertloop + add r1, r3, #144 + ldr r2, [r1], #4 + ldr r3, [r1], #4 + ldr r4, [r1], #4 + ldr r5, [r1], #4 + ldr r6, [r1], #4 + ldr r7, [r1], #4 + ldr r8, [r1], #4 + ldr r9, [r1], #4 + ldr r10, [r1], #4 + ldr r1, [r1] + add r11, r1, r1, LSL #4 + add r11, r11, r1, LSL #1 + add r11, r11, #16777216 + mov r11, r11, ASR #25 + add r11, r11, r2 + mov r11, r11, ASR #26 + add r11, r11, r3 + mov r11, r11, ASR #25 + add r11, r11, r4 + mov r11, r11, ASR #26 + add r11, r11, r5 + mov r11, r11, ASR #25 + add r11, r11, r6 + mov r11, r11, ASR #26 + add r11, r11, r7 + mov r11, r11, ASR #25 + add r11, r11, r8 + mov r11, r11, ASR #26 + add r11, r11, r9 + mov r11, r11, ASR #25 + add r11, r11, r10 + mov r11, r11, ASR #26 + add r11, r11, r1 + mov r11, r11, ASR #25 + add r2, r2, r11 + add r2, r2, r11, LSL #1 + add r2, r2, r11, LSL #4 + mov r11, r2, ASR #26 + add r3, r3, r11 + sub r2, r2, r11, LSL #26 + mov r11, r3, ASR #25 + add r4, r4, r11 + sub r3, r3, r11, LSL #25 + mov r11, r4, ASR #26 + add r5, r5, r11 + sub r4, r4, r11, LSL #26 + mov r11, r5, ASR #25 + add r6, r6, r11 + sub r5, r5, r11, LSL #25 + mov r11, r6, ASR #26 + add r7, r7, r11 + sub r6, r6, r11, LSL #26 + mov r11, r7, ASR #25 + add r8, r8, r11 + sub r7, r7, r11, LSL #25 + mov r11, r8, ASR #26 + add r9, r9, r11 + sub r8, r8, r11, LSL #26 + mov r11, r9, ASR #25 + add r10, r10, r11 + sub r9, r9, r11, LSL #25 + mov r11, r10, ASR #26 + add r1, r1, r11 + sub r10, r10, r11, LSL #26 + mov r11, r1, ASR #25 + sub r1, r1, r11, LSL #25 + add r2, r2, r3, LSL #26 + mov r3, r3, LSR #6 + add r3, r3, r4, LSL #19 + mov r4, r4, LSR #13 + add r4, r4, r5, LSL #13 + mov r5, r5, LSR #19 + add r5, r5, r6, LSL #6 + add r6, r7, r8, LSL #25 + mov r7, r8, LSR #7 + add r7, r7, r9, LSL #19 + mov r8, r9, LSR #13 + add r8, r8, r10, LSL #12 + mov r9, r10, LSR #20 + add r1, r9, r1, LSL #6 + str r2, [r0] + str r3, [r0, #4] + str r4, [r0, #8] + str r5, [r0, #12] + str r6, [r0, #16] + str r7, [r0, #20] + str r8, [r0, #24] + str r1, [r0, #28] + movw r0, #0 + mov sp, ip + pop {r4-r11, pc} +SYM_FUNC_END(curve25519_neon) +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c new file mode 100644 index 0000000000000..42cfb6c00f984 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2016 The fiat-crypto Authors. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally + * machine generated, it has been tweaked to be suitable for use in the kernel. + * It is optimized for 32-bit machines and machines that cannot work efficiently + * with 128-bit integer types. + */ + +/* fe means field element. Here the field is \Z/(2^255-19). An element t, + * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 + * t[3]+2^102 t[4]+...+2^230 t[9]. + * fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc. + * Multiplication and carrying produce fe from fe_loose. + */ +typedef struct fe { u32 v[10]; } fe; + +/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc + * Addition and subtraction produce fe_loose from (fe, fe). + */ +typedef struct fe_loose { u32 v[10]; } fe_loose; + +static __always_inline void fe_frombytes_impl(u32 h[10], const u8 *s) +{ + /* Ignores top bit of s. */ + u32 a0 = get_unaligned_le32(s); + u32 a1 = get_unaligned_le32(s+4); + u32 a2 = get_unaligned_le32(s+8); + u32 a3 = get_unaligned_le32(s+12); + u32 a4 = get_unaligned_le32(s+16); + u32 a5 = get_unaligned_le32(s+20); + u32 a6 = get_unaligned_le32(s+24); + u32 a7 = get_unaligned_le32(s+28); + h[0] = a0&((1<<26)-1); /* 26 used, 32-26 left. 26 */ + h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 = 6+19 = 25 */ + h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */ + h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) + 6 = 19+ 6 = 25 */ + h[4] = (a3>> 6); /* (32- 6) = 26 */ + h[5] = a4&((1<<25)-1); /* 25 */ + h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 = 7+19 = 26 */ + h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */ + h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) + 6 = 20+ 6 = 26 */ + h[9] = (a7>> 6)&((1<<25)-1); /* 25 */ +} + +static __always_inline void fe_frombytes(fe *h, const u8 *s) +{ + fe_frombytes_impl(h->v, s); +} + +static __always_inline u8 /*bool*/ +addcarryx_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of carry + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 25) - 1); + return (x >> 25) & 1; +} + +static __always_inline u8 /*bool*/ +addcarryx_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of carry + * (27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 26) - 1); + return (x >> 26) & 1; +} + +static __always_inline u8 /*bool*/ +subborrow_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of borrow + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 25) - 1); + return x >> 31; +} + +static __always_inline u8 /*bool*/ +subborrow_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of borrow + *(27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 26) - 1); + return x >> 31; +} + +static __always_inline u32 cmovznz32(u32 t, u32 z, u32 nz) +{ + t = -!!t; /* all set if nonzero, 0 if 0 */ + return (t&nz) | ((~t)&z); +} + +static __always_inline void fe_freeze(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u32 x20; u8/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20); + { u32 x23; u8/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23); + { u32 x26; u8/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26); + { u32 x29; u8/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29); + { u32 x32; u8/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32); + { u32 x35; u8/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35); + { u32 x38; u8/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38); + { u32 x41; u8/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41); + { u32 x44; u8/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44); + { u32 x47; u8/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47); + { u32 x49 = cmovznz32(x48, 0x0, 0xffffffff); + { u32 x50 = (x49 & 0x3ffffed); + { u32 x52; u8/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52); + { u32 x54 = (x49 & 0x1ffffff); + { u32 x56; u8/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56); + { u32 x58 = (x49 & 0x3ffffff); + { u32 x60; u8/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60); + { u32 x62 = (x49 & 0x1ffffff); + { u32 x64; u8/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64); + { u32 x66 = (x49 & 0x3ffffff); + { u32 x68; u8/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68); + { u32 x70 = (x49 & 0x1ffffff); + { u32 x72; u8/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72); + { u32 x74 = (x49 & 0x3ffffff); + { u32 x76; u8/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76); + { u32 x78 = (x49 & 0x1ffffff); + { u32 x80; u8/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80); + { u32 x82 = (x49 & 0x3ffffff); + { u32 x84; u8/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84); + { u32 x86 = (x49 & 0x1ffffff); + { u32 x88; addcarryx_u25(x85, x47, x86, &x88); + out[0] = x52; + out[1] = x56; + out[2] = x60; + out[3] = x64; + out[4] = x68; + out[5] = x72; + out[6] = x76; + out[7] = x80; + out[8] = x84; + out[9] = x88; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_tobytes(u8 s[32], const fe *f) +{ + u32 h[10]; + fe_freeze(h, f->v); + s[0] = h[0] >> 0; + s[1] = h[0] >> 8; + s[2] = h[0] >> 16; + s[3] = (h[0] >> 24) | (h[1] << 2); + s[4] = h[1] >> 6; + s[5] = h[1] >> 14; + s[6] = (h[1] >> 22) | (h[2] << 3); + s[7] = h[2] >> 5; + s[8] = h[2] >> 13; + s[9] = (h[2] >> 21) | (h[3] << 5); + s[10] = h[3] >> 3; + s[11] = h[3] >> 11; + s[12] = (h[3] >> 19) | (h[4] << 6); + s[13] = h[4] >> 2; + s[14] = h[4] >> 10; + s[15] = h[4] >> 18; + s[16] = h[5] >> 0; + s[17] = h[5] >> 8; + s[18] = h[5] >> 16; + s[19] = (h[5] >> 24) | (h[6] << 1); + s[20] = h[6] >> 7; + s[21] = h[6] >> 15; + s[22] = (h[6] >> 23) | (h[7] << 3); + s[23] = h[7] >> 5; + s[24] = h[7] >> 13; + s[25] = (h[7] >> 21) | (h[8] << 4); + s[26] = h[8] >> 4; + s[27] = h[8] >> 12; + s[28] = (h[8] >> 20) | (h[9] << 6); + s[29] = h[9] >> 2; + s[30] = h[9] >> 10; + s[31] = h[9] >> 18; +} + +/* h = f */ +static __always_inline void fe_copy(fe *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +static __always_inline void fe_copy_lt(fe_loose *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +/* h = 0 */ +static __always_inline void fe_0(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); +} + +/* h = 1 */ +static __always_inline void fe_1(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); + h->v[0] = 1; +} + +static void fe_add_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = (x5 + x23); + out[1] = (x7 + x25); + out[2] = (x9 + x27); + out[3] = (x11 + x29); + out[4] = (x13 + x31); + out[5] = (x15 + x33); + out[6] = (x17 + x35); + out[7] = (x19 + x37); + out[8] = (x21 + x39); + out[9] = (x20 + x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f + g + * Can overlap h with f or g. + */ +static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g) +{ + fe_add_impl(h->v, f->v, g->v); +} + +static void fe_sub_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = ((0x7ffffda + x5) - x23); + out[1] = ((0x3fffffe + x7) - x25); + out[2] = ((0x7fffffe + x9) - x27); + out[3] = ((0x3fffffe + x11) - x29); + out[4] = ((0x7fffffe + x13) - x31); + out[5] = ((0x3fffffe + x15) - x33); + out[6] = ((0x7fffffe + x17) - x35); + out[7] = ((0x3fffffe + x19) - x37); + out[8] = ((0x7fffffe + x21) - x39); + out[9] = ((0x3fffffe + x20) - x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f - g + * Can overlap h with f or g. + */ +static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g) +{ + fe_sub_impl(h->v, f->v, g->v); +} + +static void fe_mul_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void +fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static void fe_sqr_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u64 x19 = ((u64)x2 * x2); + { u64 x20 = ((u64)(0x2 * x2) * x4); + { u64 x21 = (0x2 * (((u64)x4 * x4) + ((u64)x2 * x6))); + { u64 x22 = (0x2 * (((u64)x4 * x6) + ((u64)x2 * x8))); + { u64 x23 = ((((u64)x6 * x6) + ((u64)(0x4 * x4) * x8)) + ((u64)(0x2 * x2) * x10)); + { u64 x24 = (0x2 * ((((u64)x6 * x8) + ((u64)x4 * x10)) + ((u64)x2 * x12))); + { u64 x25 = (0x2 * (((((u64)x8 * x8) + ((u64)x6 * x10)) + ((u64)x2 * x14)) + ((u64)(0x2 * x4) * x12))); + { u64 x26 = (0x2 * (((((u64)x8 * x10) + ((u64)x6 * x12)) + ((u64)x4 * x14)) + ((u64)x2 * x16))); + { u64 x27 = (((u64)x10 * x10) + (0x2 * ((((u64)x6 * x14) + ((u64)x2 * x18)) + (0x2 * (((u64)x4 * x16) + ((u64)x8 * x12)))))); + { u64 x28 = (0x2 * ((((((u64)x10 * x12) + ((u64)x8 * x14)) + ((u64)x6 * x16)) + ((u64)x4 * x18)) + ((u64)x2 * x17))); + { u64 x29 = (0x2 * (((((u64)x12 * x12) + ((u64)x10 * x14)) + ((u64)x6 * x18)) + (0x2 * (((u64)x8 * x16) + ((u64)x4 * x17))))); + { u64 x30 = (0x2 * (((((u64)x12 * x14) + ((u64)x10 * x16)) + ((u64)x8 * x18)) + ((u64)x6 * x17))); + { u64 x31 = (((u64)x14 * x14) + (0x2 * (((u64)x10 * x18) + (0x2 * (((u64)x12 * x16) + ((u64)x8 * x17)))))); + { u64 x32 = (0x2 * ((((u64)x14 * x16) + ((u64)x12 * x18)) + ((u64)x10 * x17))); + { u64 x33 = (0x2 * ((((u64)x16 * x16) + ((u64)x14 * x18)) + ((u64)(0x2 * x12) * x17))); + { u64 x34 = (0x2 * (((u64)x16 * x18) + ((u64)x14 * x17))); + { u64 x35 = (((u64)x18 * x18) + ((u64)(0x4 * x16) * x17)); + { u64 x36 = ((u64)(0x2 * x18) * x17); + { u64 x37 = ((u64)(0x2 * x17) * x17); + { u64 x38 = (x27 + (x37 << 0x4)); + { u64 x39 = (x38 + (x37 << 0x1)); + { u64 x40 = (x39 + x37); + { u64 x41 = (x26 + (x36 << 0x4)); + { u64 x42 = (x41 + (x36 << 0x1)); + { u64 x43 = (x42 + x36); + { u64 x44 = (x25 + (x35 << 0x4)); + { u64 x45 = (x44 + (x35 << 0x1)); + { u64 x46 = (x45 + x35); + { u64 x47 = (x24 + (x34 << 0x4)); + { u64 x48 = (x47 + (x34 << 0x1)); + { u64 x49 = (x48 + x34); + { u64 x50 = (x23 + (x33 << 0x4)); + { u64 x51 = (x50 + (x33 << 0x1)); + { u64 x52 = (x51 + x33); + { u64 x53 = (x22 + (x32 << 0x4)); + { u64 x54 = (x53 + (x32 << 0x1)); + { u64 x55 = (x54 + x32); + { u64 x56 = (x21 + (x31 << 0x4)); + { u64 x57 = (x56 + (x31 << 0x1)); + { u64 x58 = (x57 + x31); + { u64 x59 = (x20 + (x30 << 0x4)); + { u64 x60 = (x59 + (x30 << 0x1)); + { u64 x61 = (x60 + x30); + { u64 x62 = (x19 + (x29 << 0x4)); + { u64 x63 = (x62 + (x29 << 0x1)); + { u64 x64 = (x63 + x29); + { u64 x65 = (x64 >> 0x1a); + { u32 x66 = ((u32)x64 & 0x3ffffff); + { u64 x67 = (x65 + x61); + { u64 x68 = (x67 >> 0x19); + { u32 x69 = ((u32)x67 & 0x1ffffff); + { u64 x70 = (x68 + x58); + { u64 x71 = (x70 >> 0x1a); + { u32 x72 = ((u32)x70 & 0x3ffffff); + { u64 x73 = (x71 + x55); + { u64 x74 = (x73 >> 0x19); + { u32 x75 = ((u32)x73 & 0x1ffffff); + { u64 x76 = (x74 + x52); + { u64 x77 = (x76 >> 0x1a); + { u32 x78 = ((u32)x76 & 0x3ffffff); + { u64 x79 = (x77 + x49); + { u64 x80 = (x79 >> 0x19); + { u32 x81 = ((u32)x79 & 0x1ffffff); + { u64 x82 = (x80 + x46); + { u64 x83 = (x82 >> 0x1a); + { u32 x84 = ((u32)x82 & 0x3ffffff); + { u64 x85 = (x83 + x43); + { u64 x86 = (x85 >> 0x19); + { u32 x87 = ((u32)x85 & 0x1ffffff); + { u64 x88 = (x86 + x40); + { u64 x89 = (x88 >> 0x1a); + { u32 x90 = ((u32)x88 & 0x3ffffff); + { u64 x91 = (x89 + x28); + { u64 x92 = (x91 >> 0x19); + { u32 x93 = ((u32)x91 & 0x1ffffff); + { u64 x94 = (x66 + (0x13 * x92)); + { u32 x95 = (u32) (x94 >> 0x1a); + { u32 x96 = ((u32)x94 & 0x3ffffff); + { u32 x97 = (x95 + x69); + { u32 x98 = (x97 >> 0x19); + { u32 x99 = (x97 & 0x1ffffff); + out[0] = x96; + out[1] = x99; + out[2] = (x98 + x72); + out[3] = x75; + out[4] = x78; + out[5] = x81; + out[6] = x84; + out[7] = x87; + out[8] = x90; + out[9] = x93; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_sq_tl(fe *h, const fe_loose *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_sq_tt(fe *h, const fe *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_loose_invert(fe *out, const fe_loose *z) +{ + fe t0; + fe t1; + fe t2; + fe t3; + int i; + + fe_sq_tl(&t0, z); + fe_sq_tt(&t1, &t0); + for (i = 1; i < 2; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_tlt(&t1, z, &t1); + fe_mul_ttt(&t0, &t0, &t1); + fe_sq_tt(&t2, &t0); + fe_mul_ttt(&t1, &t1, &t2); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 20; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 100; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t1, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_ttt(out, &t1, &t0); +} + +static __always_inline void fe_invert(fe *out, const fe *z) +{ + fe_loose l; + fe_copy_lt(&l, z); + fe_loose_invert(out, &l); +} + +/* Replace (f,g) with (g,f) if b == 1; + * replace (f,g) with (f,g) if b == 0. + * + * Preconditions: b in {0,1} + */ +static __always_inline void fe_cswap(fe *f, fe *g, unsigned int b) +{ + unsigned i; + b = 0 - b; + for (i = 0; i < 10; i++) { + u32 x = f->v[i] ^ g->v[i]; + x &= b; + f->v[i] ^= x; + g->v[i] ^= x; + } +} + +/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/ +static __always_inline void fe_mul_121666_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = 0; + { const u32 x39 = 0; + { const u32 x37 = 0; + { const u32 x35 = 0; + { const u32 x33 = 0; + { const u32 x31 = 0; + { const u32 x29 = 0; + { const u32 x27 = 0; + { const u32 x25 = 0; + { const u32 x23 = 121666; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul121666(fe *h, const fe_loose *f) +{ + fe_mul_121666_impl(h->v, f->v); +} + +static void curve25519_generic(u8 out[CURVE25519_KEY_SIZE], + const u8 scalar[CURVE25519_KEY_SIZE], + const u8 point[CURVE25519_KEY_SIZE]) +{ + fe x1, x2, z2, x3, z3; + fe_loose x2l, z2l, x3l; + unsigned swap = 0; + int pos; + u8 e[32]; + + memcpy(e, scalar, 32); + curve25519_clamp_secret(e); + + /* The following implementation was transcribed to Coq and proven to + * correspond to unary scalar multiplication in affine coordinates given + * that x1 != 0 is the x coordinate of some point on the curve. It was + * also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives + * z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was + * quantified over the underlying field, so it applies to Curve25519 + * itself and the quadratic twist of Curve25519. It was not proven in + * Coq that prime-field arithmetic correctly simulates extension-field + * arithmetic on prime-field values. The decoding of the byte array + * representation of e was not considered. + * + * Specification of Montgomery curves in affine coordinates: + * + * + * Proof that these form a group that is isomorphic to a Weierstrass + * curve: + * + * + * Coq transcription and correctness proof of the loop + * (where scalarbits=255): + * + * + * preconditions: 0 <= e < 2^255 (not necessarily e < order), + * fe_invert(0) = 0 + */ + fe_frombytes(&x1, point); + fe_1(&x2); + fe_0(&z2); + fe_copy(&x3, &x1); + fe_1(&z3); + + for (pos = 254; pos >= 0; --pos) { + fe tmp0, tmp1; + fe_loose tmp0l, tmp1l; + /* loop invariant as of right before the test, for the case + * where x1 != 0: + * pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 + * is nonzero + * let r := e >> (pos+1) in the following equalities of + * projective points: + * to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + * to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + * x1 is the nonzero x coordinate of the nonzero + * point (r*P-(r+1)*P) + */ + unsigned b = 1 & (e[pos / 8] >> (pos & 7)); + swap ^= b; + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + swap = b; + /* Coq transcription of ladderstep formula (called from + * transcribed loop): + * + * + * x1 != 0 + * x1 = 0 + */ + fe_sub(&tmp0l, &x3, &z3); + fe_sub(&tmp1l, &x2, &z2); + fe_add(&x2l, &x2, &z2); + fe_add(&z2l, &x3, &z3); + fe_mul_tll(&z3, &tmp0l, &x2l); + fe_mul_tll(&z2, &z2l, &tmp1l); + fe_sq_tl(&tmp0, &tmp1l); + fe_sq_tl(&tmp1, &x2l); + fe_add(&x3l, &z3, &z2); + fe_sub(&z2l, &z3, &z2); + fe_mul_ttt(&x2, &tmp1, &tmp0); + fe_sub(&tmp1l, &tmp1, &tmp0); + fe_sq_tl(&z2, &z2l); + fe_mul121666(&z3, &tmp1l); + fe_sq_tl(&x3, &x3l); + fe_add(&tmp0l, &tmp0, &z3); + fe_mul_ttt(&z3, &x1, &z2); + fe_mul_tll(&z2, &tmp1l, &tmp0l); + } + /* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) + * else (x2, z2) + */ + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + + fe_invert(&z2, &z2); + fe_mul_ttt(&x2, &x2, &z2); + fe_tobytes(out, &x2); + + memzero_explicit(&x1, sizeof(x1)); + memzero_explicit(&x2, sizeof(x2)); + memzero_explicit(&z2, sizeof(z2)); + memzero_explicit(&x3, sizeof(x3)); + memzero_explicit(&z3, sizeof(z3)); + memzero_explicit(&x2l, sizeof(x2l)); + memzero_explicit(&z2l, sizeof(z2l)); + memzero_explicit(&x3l, sizeof(x3l)); + memzero_explicit(&e, sizeof(e)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c new file mode 100644 index 0000000000000..0f729ec74ece8 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2016-2017 INRIA and Microsoft Corporation. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally machine + * generated, it has been tweaked to be suitable for use in the kernel. It is + * optimized for 64-bit machines that can efficiently work with 128-bit + * integer types. + */ + +typedef __uint128_t u128; + +static __always_inline u64 u64_eq_mask(u64 a, u64 b) +{ + u64 x = a ^ b; + u64 minus_x = ~x + (u64)1U; + u64 x_or_minus_x = x | minus_x; + u64 xnx = x_or_minus_x >> (u32)63U; + u64 c = xnx - (u64)1U; + return c; +} + +static __always_inline u64 u64_gte_mask(u64 a, u64 b) +{ + u64 x = a; + u64 y = b; + u64 x_xor_y = x ^ y; + u64 x_sub_y = x - y; + u64 x_sub_y_xor_y = x_sub_y ^ y; + u64 q = x_xor_y | x_sub_y_xor_y; + u64 x_xor_q = x ^ q; + u64 x_xor_q_ = x_xor_q >> (u32)63U; + u64 c = x_xor_q_ - (u64)1U; + return c; +} + +static __always_inline void modulo_carry_top(u64 *b) +{ + u64 b4 = b[4]; + u64 b0 = b[0]; + u64 b4_ = b4 & 0x7ffffffffffffLLU; + u64 b0_ = b0 + 19 * (b4 >> 51); + b[4] = b4_; + b[0] = b0_; +} + +static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input) +{ + { + u128 xi = input[0]; + output[0] = ((u64)(xi)); + } + { + u128 xi = input[1]; + output[1] = ((u64)(xi)); + } + { + u128 xi = input[2]; + output[2] = ((u64)(xi)); + } + { + u128 xi = input[3]; + output[3] = ((u64)(xi)); + } + { + u128 xi = input[4]; + output[4] = ((u64)(xi)); + } +} + +static __always_inline void +fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s) +{ + output[0] += (u128)input[0] * s; + output[1] += (u128)input[1] * s; + output[2] += (u128)input[2] * s; + output[3] += (u128)input[3] * s; + output[4] += (u128)input[4] * s; +} + +static __always_inline void fproduct_carry_wide_(u128 *tmp) +{ + { + u32 ctr = 0; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 1; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + + { + u32 ctr = 2; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 3; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } +} + +static __always_inline void fmul_shift_reduce(u64 *output) +{ + u64 tmp = output[4]; + u64 b0; + { + u32 ctr = 5 - 0 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 1 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 2 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 3 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + output[0] = tmp; + b0 = output[0]; + output[0] = 19 * b0; +} + +static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input, + u64 *input21) +{ + u32 i; + u64 input2i; + { + u64 input2i = input21[0]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[1]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[2]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[3]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + i = 4; + input2i = input21[i]; + fproduct_sum_scalar_multiplication_(output, input, input2i); +} + +static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21) +{ + u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] }; + { + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + u128 t[5] = { 0 }; + fmul_mul_shift_reduce_(t, tmp, input21); + fproduct_carry_wide_(t); + b4 = t[4]; + b0 = t[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + t[4] = b4_; + t[0] = b0_; + fproduct_copy_from_wide_(output, t); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; + } +} + +static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output) +{ + u64 r0 = output[0]; + u64 r1 = output[1]; + u64 r2 = output[2]; + u64 r3 = output[3]; + u64 r4 = output[4]; + u64 d0 = r0 * 2; + u64 d1 = r1 * 2; + u64 d2 = r2 * 2 * 19; + u64 d419 = r4 * 19; + u64 d4 = d419 * 2; + u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) + + (((u128)(d2) * (r3)))); + u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) + + (((u128)(r3 * 19) * (r3)))); + u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) + + (((u128)(d4) * (r3)))); + u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) + + (((u128)(r4) * (d419)))); + u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) + + (((u128)(r2) * (r2)))); + tmp[0] = s0; + tmp[1] = s1; + tmp[2] = s2; + tmp[3] = s3; + tmp[4] = s4; +} + +static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output) +{ + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + fsquare_fsquare__(tmp, output); + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; +} + +static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp, + u32 count1) +{ + u32 i; + fsquare_fsquare_(tmp, output); + for (i = 1; i < count1; ++i) + fsquare_fsquare_(tmp, output); +} + +static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input, + u32 count1) +{ + u128 t[5]; + memcpy(output, input, 5 * sizeof(*input)); + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void fsquare_fsquare_times_inplace(u64 *output, + u32 count1) +{ + u128 t[5]; + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void crecip_crecip(u64 *out, u64 *z) +{ + u64 buf[20] = { 0 }; + u64 *a0 = buf; + u64 *t00 = buf + 5; + u64 *b0 = buf + 10; + u64 *t01; + u64 *b1; + u64 *c0; + u64 *a; + u64 *t0; + u64 *b; + u64 *c; + fsquare_fsquare_times(a0, z, 1); + fsquare_fsquare_times(t00, a0, 2); + fmul_fmul(b0, t00, z); + fmul_fmul(a0, b0, a0); + fsquare_fsquare_times(t00, a0, 1); + fmul_fmul(b0, t00, b0); + fsquare_fsquare_times(t00, b0, 5); + t01 = buf + 5; + b1 = buf + 10; + c0 = buf + 15; + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 10); + fmul_fmul(c0, t01, b1); + fsquare_fsquare_times(t01, c0, 20); + fmul_fmul(t01, t01, c0); + fsquare_fsquare_times_inplace(t01, 10); + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 50); + a = buf; + t0 = buf + 5; + b = buf + 10; + c = buf + 15; + fmul_fmul(c, t0, b); + fsquare_fsquare_times(t0, c, 100); + fmul_fmul(t0, t0, c); + fsquare_fsquare_times_inplace(t0, 50); + fmul_fmul(t0, t0, b); + fsquare_fsquare_times_inplace(t0, 5); + fmul_fmul(out, t0, a); +} + +static __always_inline void fsum(u64 *a, u64 *b) +{ + a[0] += b[0]; + a[1] += b[1]; + a[2] += b[2]; + a[3] += b[3]; + a[4] += b[4]; +} + +static __always_inline void fdifference(u64 *a, u64 *b) +{ + u64 tmp[5] = { 0 }; + u64 b0; + u64 b1; + u64 b2; + u64 b3; + u64 b4; + memcpy(tmp, b, 5 * sizeof(*b)); + b0 = tmp[0]; + b1 = tmp[1]; + b2 = tmp[2]; + b3 = tmp[3]; + b4 = tmp[4]; + tmp[0] = b0 + 0x3fffffffffff68LLU; + tmp[1] = b1 + 0x3ffffffffffff8LLU; + tmp[2] = b2 + 0x3ffffffffffff8LLU; + tmp[3] = b3 + 0x3ffffffffffff8LLU; + tmp[4] = b4 + 0x3ffffffffffff8LLU; + { + u64 xi = a[0]; + u64 yi = tmp[0]; + a[0] = yi - xi; + } + { + u64 xi = a[1]; + u64 yi = tmp[1]; + a[1] = yi - xi; + } + { + u64 xi = a[2]; + u64 yi = tmp[2]; + a[2] = yi - xi; + } + { + u64 xi = a[3]; + u64 yi = tmp[3]; + a[3] = yi - xi; + } + { + u64 xi = a[4]; + u64 yi = tmp[4]; + a[4] = yi - xi; + } +} + +static __always_inline void fscalar(u64 *output, u64 *b, u64 s) +{ + u128 tmp[5]; + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + { + u64 xi = b[0]; + tmp[0] = ((u128)(xi) * (s)); + } + { + u64 xi = b[1]; + tmp[1] = ((u128)(xi) * (s)); + } + { + u64 xi = b[2]; + tmp[2] = ((u128)(xi) * (s)); + } + { + u64 xi = b[3]; + tmp[3] = ((u128)(xi) * (s)); + } + { + u64 xi = b[4]; + tmp[4] = ((u128)(xi) * (s)); + } + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); +} + +static __always_inline void fmul(u64 *output, u64 *a, u64 *b) +{ + fmul_fmul(output, a, b); +} + +static __always_inline void crecip(u64 *output, u64 *input) +{ + crecip_crecip(output, input); +} + +static __always_inline void point_swap_conditional_step(u64 *a, u64 *b, + u64 swap1, u32 ctr) +{ + u32 i = ctr - 1; + u64 ai = a[i]; + u64 bi = b[i]; + u64 x = swap1 & (ai ^ bi); + u64 ai1 = ai ^ x; + u64 bi1 = bi ^ x; + a[i] = ai1; + b[i] = bi1; +} + +static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1) +{ + point_swap_conditional_step(a, b, swap1, 5); + point_swap_conditional_step(a, b, swap1, 4); + point_swap_conditional_step(a, b, swap1, 3); + point_swap_conditional_step(a, b, swap1, 2); + point_swap_conditional_step(a, b, swap1, 1); +} + +static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap) +{ + u64 swap1 = 0 - iswap; + point_swap_conditional5(a, b, swap1); + point_swap_conditional5(a + 5, b + 5, swap1); +} + +static __always_inline void point_copy(u64 *output, u64 *input) +{ + memcpy(output, input, 5 * sizeof(*input)); + memcpy(output + 5, input + 5, 5 * sizeof(*input)); +} + +static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p, + u64 *pq, u64 *qmqp) +{ + u64 *qx = qmqp; + u64 *x2 = pp; + u64 *z2 = pp + 5; + u64 *x3 = ppq; + u64 *z3 = ppq + 5; + u64 *x = p; + u64 *z = p + 5; + u64 *xprime = pq; + u64 *zprime = pq + 5; + u64 buf[40] = { 0 }; + u64 *origx = buf; + u64 *origxprime0 = buf + 5; + u64 *xxprime0; + u64 *zzprime0; + u64 *origxprime; + xxprime0 = buf + 25; + zzprime0 = buf + 30; + memcpy(origx, x, 5 * sizeof(*x)); + fsum(x, z); + fdifference(z, origx); + memcpy(origxprime0, xprime, 5 * sizeof(*xprime)); + fsum(xprime, zprime); + fdifference(zprime, origxprime0); + fmul(xxprime0, xprime, z); + fmul(zzprime0, x, zprime); + origxprime = buf + 5; + { + u64 *xx0; + u64 *zz0; + u64 *xxprime; + u64 *zzprime; + u64 *zzzprime; + xx0 = buf + 15; + zz0 = buf + 20; + xxprime = buf + 25; + zzprime = buf + 30; + zzzprime = buf + 35; + memcpy(origxprime, xxprime, 5 * sizeof(*xxprime)); + fsum(xxprime, zzprime); + fdifference(zzprime, origxprime); + fsquare_fsquare_times(x3, xxprime, 1); + fsquare_fsquare_times(zzzprime, zzprime, 1); + fmul(z3, zzzprime, qx); + fsquare_fsquare_times(xx0, x, 1); + fsquare_fsquare_times(zz0, z, 1); + { + u64 *zzz; + u64 *xx; + u64 *zz; + u64 scalar; + zzz = buf + 10; + xx = buf + 15; + zz = buf + 20; + fmul(x2, xx, zz); + fdifference(zz, xx); + scalar = 121665; + fscalar(zzz, zz, scalar); + fsum(zzz, xx); + fmul(z2, zzz, zz); + } + } +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt) +{ + u64 bit0 = (u64)(byt >> 7); + u64 bit; + point_swap_conditional(nq, nqpq, bit0); + addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q); + bit = (u64)(byt >> 7); + point_swap_conditional(nq2, nqpq2, bit); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, u8 byt) +{ + u8 byt1; + ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt); + byt1 = byt << 1; + ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt, u32 i) +{ + while (i--) { + ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2, + nqpq2, q, byt); + byt <<= 2; + } +} + +static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq, + u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, + u32 i) +{ + while (i--) { + u8 byte = n1[i]; + ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q, + byte, 4); + } +} + +static void ladder_cmult(u64 *result, u8 *n1, u64 *q) +{ + u64 point_buf[40] = { 0 }; + u64 *nq = point_buf; + u64 *nqpq = point_buf + 10; + u64 *nq2 = point_buf + 20; + u64 *nqpq2 = point_buf + 30; + point_copy(nqpq, q); + nq[0] = 1; + ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32); + point_copy(result, nq); +} + +static __always_inline void format_fexpand(u64 *output, const u8 *input) +{ + const u8 *x00 = input + 6; + const u8 *x01 = input + 12; + const u8 *x02 = input + 19; + const u8 *x0 = input + 24; + u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4; + i0 = get_unaligned_le64(input); + i1 = get_unaligned_le64(x00); + i2 = get_unaligned_le64(x01); + i3 = get_unaligned_le64(x02); + i4 = get_unaligned_le64(x0); + output0 = i0 & 0x7ffffffffffffLLU; + output1 = i1 >> 3 & 0x7ffffffffffffLLU; + output2 = i2 >> 6 & 0x7ffffffffffffLLU; + output3 = i3 >> 1 & 0x7ffffffffffffLLU; + output4 = i4 >> 12 & 0x7ffffffffffffLLU; + output[0] = output0; + output[1] = output1; + output[2] = output2; + output[3] = output3; + output[4] = output4; +} + +static __always_inline void format_fcontract_first_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_first_carry_full(u64 *input) +{ + format_fcontract_first_carry_pass(input); + modulo_carry_top(input); +} + +static __always_inline void format_fcontract_second_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_second_carry_full(u64 *input) +{ + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + format_fcontract_second_carry_pass(input); + modulo_carry_top(input); + i0 = input[0]; + i1 = input[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + input[0] = i0_; + input[1] = i1_; +} + +static __always_inline void format_fcontract_trim(u64 *input) +{ + u64 a0 = input[0]; + u64 a1 = input[1]; + u64 a2 = input[2]; + u64 a3 = input[3]; + u64 a4 = input[4]; + u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU); + u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU); + u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU); + u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU); + u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU); + u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4; + u64 a0_ = a0 - (0x7ffffffffffedLLU & mask); + u64 a1_ = a1 - (0x7ffffffffffffLLU & mask); + u64 a2_ = a2 - (0x7ffffffffffffLLU & mask); + u64 a3_ = a3 - (0x7ffffffffffffLLU & mask); + u64 a4_ = a4 - (0x7ffffffffffffLLU & mask); + input[0] = a0_; + input[1] = a1_; + input[2] = a2_; + input[3] = a3_; + input[4] = a4_; +} + +static __always_inline void format_fcontract_store(u8 *output, u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 o0 = t1 << 51 | t0; + u64 o1 = t2 << 38 | t1 >> 13; + u64 o2 = t3 << 25 | t2 >> 26; + u64 o3 = t4 << 12 | t3 >> 39; + u8 *b0 = output; + u8 *b1 = output + 8; + u8 *b2 = output + 16; + u8 *b3 = output + 24; + put_unaligned_le64(o0, b0); + put_unaligned_le64(o1, b1); + put_unaligned_le64(o2, b2); + put_unaligned_le64(o3, b3); +} + +static __always_inline void format_fcontract(u8 *output, u64 *input) +{ + format_fcontract_first_carry_full(input); + format_fcontract_second_carry_full(input); + format_fcontract_trim(input); + format_fcontract_store(output, input); +} + +static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point) +{ + u64 *x = point; + u64 *z = point + 5; + u64 buf[10] __aligned(32) = { 0 }; + u64 *zmone = buf; + u64 *sc = buf + 5; + crecip(zmone, z); + fmul(sc, x, zmone); + format_fcontract(scalar, sc); +} + +static void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + u64 buf0[10] __aligned(32) = { 0 }; + u64 *x0 = buf0; + u64 *z = buf0 + 5; + u64 *q; + format_fexpand(x0, basepoint); + z[0] = 1; + q = buf0; + { + u8 e[32] __aligned(32) = { 0 }; + u8 *scalar; + memcpy(e, secret, 32); + curve25519_clamp_secret(e); + scalar = e; + { + u64 buf[15] = { 0 }; + u64 *nq = buf; + u64 *x = nq; + x[0] = 1; + ladder_cmult(nq, scalar, q); + format_scalar_of_point(mypublic, nq); + memzero_explicit(buf, sizeof(buf)); + } + memzero_explicit(e, sizeof(e)); + } + memzero_explicit(buf0, sizeof(buf0)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c new file mode 100644 index 0000000000000..d62bd37b007f9 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +#include "curve25519-x86_64.c" + +static bool curve25519_use_bmi2 __ro_after_init; +static bool curve25519_use_adx __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { + &curve25519_use_bmi2, &curve25519_use_adx }; + +static void __init curve25519_fpu_init(void) +{ + curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) && + boot_cpu_has(X86_FEATURE_BMI2); + curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) && + boot_cpu_has(X86_FEATURE_BMI2) && + boot_cpu_has(X86_FEATURE_ADX); +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx(mypublic, secret, basepoint); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2(mypublic, secret, basepoint); + return true; + } + return false; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx_base(pub, secret); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2_base(pub, secret); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c new file mode 100644 index 0000000000000..e8af3e2e30f82 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c @@ -0,0 +1,2366 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright (c) 2017 Armando Faz . All Rights Reserved. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2018 Samuel Neves . All Rights Reserved. + */ + +enum { NUM_WORDS_ELTFP25519 = 4 }; +typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519]; +typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519]; + +#define mul_eltfp25519_1w_adx(c, a, b) do { \ + mul_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_1w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_1w_bmi2(c, a, b) do { \ + mul_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_1w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_adx(a) do { \ + sqr_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_1w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_bmi2(a) do { \ + sqr_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_1w_bmi2(a, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_adx(c, a, b) do { \ + mul2_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_2w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_bmi2(c, a, b) do { \ + mul2_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_2w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_adx(a) do { \ + sqr2_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_2w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_bmi2(a) do { \ + sqr2_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_2w_bmi2(a, m.buffer); \ +} while (0) + +#define sqrn_eltfp25519_1w_adx(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_adx(a); \ +} while (0) + +#define sqrn_eltfp25519_1w_bmi2(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_bmi2(a); \ +} while (0) + +#define copy_eltfp25519_1w(C, A) do { \ + (C)[0] = (A)[0]; \ + (C)[1] = (A)[1]; \ + (C)[2] = (A)[2]; \ + (C)[3] = (A)[3]; \ +} while (0) + +#define setzero_eltfp25519_1w(C) do { \ + (C)[0] = 0; \ + (C)[1] = 0; \ + (C)[2] = 0; \ + (C)[3] = 0; \ +} while (0) + +__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { + /* 1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL, + 0xffffffffffffffffUL, 0x5fffffffffffffffUL, + /* 2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL, + 0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL, + /* 3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL, + 0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL, + /* 4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL, + 0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL, + /* 5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL, + 0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL, + /* 6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL, + 0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL, + /* 7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL, + 0xc1c20d06231f7614UL, 0x2938218da274f972UL, + /* 8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL, + 0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL, + /* 9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL, + 0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL, + /* 10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL, + 0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL, + /* 11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL, + 0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL, + /* 12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL, + 0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL, + /* 13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL, + 0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL, + /* 14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL, + 0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL, + /* 15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL, + 0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL, + /* 16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL, + 0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL, + /* 17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL, + 0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL, + /* 18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL, + 0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL, + /* 19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL, + 0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL, + /* 20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL, + 0x9d4935467caaf22eUL, 0x5166408eee85ff49UL, + /* 21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL, + 0x5259729241159b1cUL, 0x6a621892d5b0ab33UL, + /* 22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL, + 0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL, + /* 23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL, + 0x23758739f630a257UL, 0x295a407a01a78580UL, + /* 24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL, + 0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL, + /* 25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL, + 0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL, + /* 26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL, + 0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL, + /* 27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL, + 0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL, + /* 28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL, + 0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL, + /* 29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL, + 0x74b4c4ceab102f64UL, 0x183abadd10139845UL, + /* 30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL, + 0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL, + /* 31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL, + 0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL, + /* 32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL, + 0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL, + /* 33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL, + 0xd88768e4904032d8UL, 0x131384427b3aaeecUL, + /* 34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL, + 0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL, + /* 35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL, + 0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL, + /* 36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL, + 0xa401760b882c797aUL, 0x1fc223e28dc88730UL, + /* 37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL, + 0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL, + /* 38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL, + 0xfdbf177988bbc586UL, 0x2959894fcad81df5UL, + /* 39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL, + 0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL, + /* 40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL, + 0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL, + /* 41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL, + 0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL, + /* 42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL, + 0x5c217736fa279374UL, 0x7dde05734afeb1faUL, + /* 43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL, + 0xe6053bf89595bf7aUL, 0x394faf38da245530UL, + /* 44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL, + 0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL, + /* 45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL, + 0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL, + /* 46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL, + 0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL, + /* 47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL, + 0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL, + /* 48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL, + 0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL, + /* 49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL, + 0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL, + /* 50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL, + 0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL, + /* 51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL, + 0xc189218075e91436UL, 0x6d9284169b3b8484UL, + /* 52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL, + 0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL, + /* 53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL, + 0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL, + /* 54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL, + 0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL, + /* 55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL, + 0x19346a65d3224a08UL, 0x0f5034e49b9af466UL, + /* 56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL, + 0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL, + /* 57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL, + 0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL, + /* 58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL, + 0xf826842130f5ad28UL, 0x3ea988f75301a441UL, + /* 59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL, + 0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL, + /* 60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL, + 0xd01469df811d644bUL, 0x77fea47d81a5d71fUL, + /* 61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL, + 0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL, + /* 62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL, + 0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL, + /* 63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL, + 0xbea450e1dbd885d5UL, 0x61b68649320f712cUL, + /* 64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL, + 0x25232973322dbef4UL, 0x445dc4758c17f770UL, + /* 65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL, + 0x1efebefdc053db34UL, 0x4adbe867c65daf99UL, + /* 66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL, + 0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL, + /* 67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL, + 0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL, + /* 68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL, + 0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL, + /* 69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL, + 0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL, + /* 70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL, + 0xbdaacb805831ca6fUL, 0x445b652dc916694fUL, + /* 71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL, + 0xa1823aafe04c314aUL, 0x790a2d94437cf586UL, + /* 72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL, + 0xbf70903b204f5169UL, 0x2f7a89891ba319feUL, + /* 73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL, + 0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL, + /* 74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL, + 0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL, + /* 75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL, + 0x674f1288f8e11217UL, 0x5682250f329f93d0UL, + /* 76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL, + 0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL, + /* 77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL, + 0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL, + /* 78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL, + 0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL, + /* 79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL, + 0x5deadacec9f04973UL, 0x29275b5d41d29b27UL, + /* 80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL, + 0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL, + /* 81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL, + 0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL, + /* 82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL, + 0x894d1d855ae52359UL, 0x68e122157b743d69UL, + /* 83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL, + 0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL, + /* 84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL, + 0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL, + /* 85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL, + 0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL, + /* 86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL, + 0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL, + /* 87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL, + 0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL, + /* 88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL, + 0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL, + /* 89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL, + 0x45adb16e76cefcf2UL, 0x01f768aead232999UL, + /* 90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL, + 0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL, + /* 91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL, + 0x5eefa966de2a701dUL, 0x23b20565de55e3efUL, + /* 92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL, + 0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL, + /* 93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL, + 0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL, + /* 94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL, + 0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL, + /* 95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL, + 0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL, + /* 96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL, + 0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL, + /* 97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL, + 0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL, + /* 98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL, + 0xb9886314844006b1UL, 0x40d2a72ab454cc60UL, + /* 99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL, + 0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL, + /* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL, + 0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL, + /* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL, + 0x40727064c416d74fUL, 0x6e15c6114b502ef0UL, + /* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL, + 0x4a497962066e6043UL, 0x705b3aab41355b44UL, + /* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL, + 0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL, + /* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL, + 0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL, + /* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL, + 0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL, + /* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL, + 0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL, + /* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL, + 0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL, + /* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL, + 0x2088ce1570033c68UL, 0x7fba1f495c837987UL, + /* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL, + 0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL, + /* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL, + 0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL, + /* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL, + 0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL, + /* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL, + 0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL, + /* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL, + 0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL, + /* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL, + 0x00f52e3f67280294UL, 0x566d4fc14730c509UL, + /* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL, + 0x216730fba68d6095UL, 0x22e8c3843f69cea7UL, + /* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL, + 0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL, + /* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL, + 0x508e862f121692fcUL, 0x3a81907fa093c291UL, + /* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL, + 0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL, + /* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL, + 0xbee595ce8a9df2e5UL, 0x25e496c722422236UL, + /* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL, + 0xe488de11d761e352UL, 0x0e878a01a085545cUL, + /* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL, + 0x9ea37a487ae80d67UL, 0x67a9958011e41794UL, + /* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL, + 0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL, + /* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL, + 0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL, + /* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL, + 0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL, + /* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL, + 0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL, + /* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL, + 0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL, + /* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL, + 0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL, + /* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL, + 0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL, + /* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL, + 0x97134556a9832d06UL, 0x269bb0360a84f8a0UL, + /* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL, + 0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL, + /* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL, + 0x904659bb686e3772UL, 0x7215c371746ba8c8UL, + /* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL, + 0x266fd5809208f294UL, 0x5c847085619a26b9UL, + /* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL, + 0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL, + /* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL, + 0x2b1641917b307614UL, 0x117c554fc4f45b7cUL, + /* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL, + 0xd7e803f4171b2827UL, 0x1015e87487d225eaUL, + /* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL, + 0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL, + /* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL, + 0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL, + /* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL, + 0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL, + /* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL, + 0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL, + /* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL, + 0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL, + /* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL, + 0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL, + /* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL, + 0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL, + /* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL, + 0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL, + /* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL, + 0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL, + /* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL, + 0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL, + /* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL, + 0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL, + /* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL, + 0xec219c48fbd21604UL, 0x1aaf1af517c36731UL, + /* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL, + 0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL, + /* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL, + 0x4dbbc207f531561aUL, 0x0253b7f082128a27UL, + /* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL, + 0x52d17436309d4253UL, 0x356f97e13efae576UL, + /* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL, + 0x0c776128bed92c98UL, 0x1d34ae93032885b8UL, + /* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL, + 0x66124c6f97bda770UL, 0x0f81a0290654124aUL, + /* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL, + 0xff08d03f93d8c20aUL, 0x52a148199faef26bUL, + /* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL, + 0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL, + /* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL, + 0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL, + /* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL, + 0x5da643cb4bf30035UL, 0x77db28d63940f721UL, + /* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL, + 0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL, + /* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL, + 0x140a69245ca575edUL, 0x0cf1c37134273a4cUL, + /* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL, + 0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL, + /* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL, + 0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL, + /* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL, + 0x497d723f802e88e1UL, 0x30684dea602f408dUL, + /* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL, + 0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL, + /* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL, + 0x7468c7423a543258UL, 0x4a7f11464eb5642fUL, + /* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL, + 0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL, + /* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL, + 0x026df551dbb85c20UL, 0x74fcd91047e21901UL, + /* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL, + 0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL, + /* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL, + 0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL, + /* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL, + 0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL, + /* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL, + 0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL, + /* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL, + 0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL, + /* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL, + 0x13033ac001f66697UL, 0x273b24fe3b367d75UL, + /* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL, + 0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL, + /* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL, + 0xacc63ca34b8ec145UL, 0x74621888fee66574UL, + /* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL, + 0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL, + /* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL, + 0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL, + /* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL, + 0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL, + /* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL, + 0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL, + /* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL, + 0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL, + /* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL, + 0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL, + /* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL, + 0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL, + /* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL, + 0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL, + /* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL, + 0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL, + /* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL, + 0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL, + /* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL, + 0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL, + /* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL, + 0x81004b71e33cc191UL, 0x44e6be345122803cUL, + /* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL, + 0x49c8c4281af60c29UL, 0x21edb518de701aeeUL, + /* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL, + 0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL, + /* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL, + 0x12bc8d6915783712UL, 0x498194c0fc620abbUL, + /* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL, + 0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL, + /* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL, + 0x1e60c24598c71fffUL, 0x59f2f014979983efUL, + /* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL, + 0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL, + /* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL, + 0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL, + /* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL, + 0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL, + /* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL, + 0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL, + /* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL, + 0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL, + /* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL, + 0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL, + /* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL, + 0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL, + /* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL, + 0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL, + /* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL, + 0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL, + /* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL, + 0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL, + /* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL, + 0x883ada83a6a1652cUL, 0x585f1974034d6c17UL, + /* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL, + 0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL, + /* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL, + 0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL, + /* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL, + 0x33979624f0e917beUL, 0x2c018dc527356b30UL, + /* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL, + 0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL, + /* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL, + 0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL, + /* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL, + 0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL, + /* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL, + 0x345ead5e972d091eUL, 0x18c8df11a83103baUL, + /* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL, + 0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL, + /* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL, + 0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL, + /* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL, + 0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL, + /* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL, + 0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL, + /* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL, + 0x79952a008221e738UL, 0x4322e1a7535cd2bbUL, + /* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL, + 0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL, + /* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL, + 0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL, + /* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL, + 0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL, + /* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL, + 0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL, + /* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL, + 0x1df4c0af01314a60UL, 0x09a62dab89289527UL, + /* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL, + 0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL, + /* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL, + 0x49b96853d7a7084aUL, 0x4980a319601420a8UL, + /* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL, + 0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL, + /* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL, + 0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL, + /* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL, + 0xddeb34a061615d99UL, 0x5129cecceb64b773UL, + /* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL, + 0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL, + /* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL, + 0x680bd77c73edad2eUL, 0x487c02354edd9041UL, + /* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL, + 0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL, + /* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL, + 0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL, + /* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL, + 0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL, + /* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL, + 0xe9834262d13921edUL, 0x27fedafaa54bb592UL, + /* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL, + 0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL, + /* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL, + 0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL, + /* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL, + 0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL, + /* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL, + 0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL, + /* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL, + 0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL, + /* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL, + 0x645b426f3d1d58acUL, 0x4804a82227a557bcUL, + /* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL, + 0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL, + /* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL, + 0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL, + /* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL, + 0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL, + /* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL, + 0xc26ccff352b37ec7UL, 0x056f68341d797b21UL, + /* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL, + 0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL, + /* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL, + 0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL, + /* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL, + 0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL, + /* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL, + 0x2c43ecea0107c1ddUL, 0x526028809372de35UL, + /* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL, + 0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL, + /* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL, + 0x899fc38fc4b5c515UL, 0x250386b124ffc207UL, + /* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL, + 0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL, + /* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL, + 0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL, + /* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL, + 0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL, + /* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL, + 0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL, + /* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL, + 0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL, + /* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL, + 0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL, + /* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL, + 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL +}; + +#ifdef CONFIG_AS_ADX +/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] + * a is two 256-bit integers: a0[0:3] and a1[4:7] + * b is two 256-bit integers: b0[0:3] and b1[4:7] + */ +static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "xorl %%r14d, %%r14d ;" + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "adox %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adox %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, 64(%0);" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "adox %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adox %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 72(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 80(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 88(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "movq %%r8, 64(%0) ;" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "addq %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adcq %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 72(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 80(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 88(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 40(%1), %%r8, %%r14 ;" /* B[1]*B[0] */ + "xorl %%r15d, %%r15d;" + "mulx 48(%1), %%r9, %%r10 ;" /* B[2]*B[0] */ + "adcx %%r14, %%r9 ;" + "mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */ + "adcx %%rax, %%r10 ;" + "movq 56(%1), %%rdx ;" /* B[3] */ + "mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */ + "adcx %%rax, %%rbx ;" + "movq 40(%1), %%rdx ;" /* B[1] */ + "adcx %%r15, %%r13 ;" + "mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq 32(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + "movq 40(%1), %%rdx ;" /* B[1] */ + "mulx 32(%1), %%r8, %%r9 ;" /* B[0]*B[1] */ + "mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */ + "mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */ + + "movq 48(%1), %%rdx ;" /* B[2] */ + "mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */ + "mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox 64(%1), %%r8 ;" + "mulx 104(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 72(%1), %%r9 ;" + "mulx 112(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 80(%1), %%r10 ;" + "mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 88(%1), %%r11 ;" + /****************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 104(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /****************************************/ + "adcq $0, %%rcx ;" + "addq 64(%1), %%r8 ;" + "adcq 72(%1), %%r9 ;" + "adcq 80(%1), %%r10 ;" + "adcq 88(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r9; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[0]*B[1] */ + "adox %%r9, %%r10 ;" + "movq %%r10, 8(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */ + "adox %%r11, %%r15 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */ + "adox %%r13, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 8(%0), %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 16(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 16(%0), %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 24(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 24(%0), %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 32(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq %%r14, 48(%0) ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + "movq %%rax, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static __always_inline void +add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "xorl %%ecx, %%ecx ;" + "movq (%2), %%r8 ;" + "adcx (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcx 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcx 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcx 24(%1), %%r11 ;" + "cmovc %%eax, %%ecx ;" + "xorl %%eax, %%eax ;" + "adcx %%rcx, %%r8 ;" + "adcx %%rax, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rax, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rax, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $38, %%ecx ;" + "cmovc %%ecx, %%eax ;" + "addq %%rax, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static __always_inline void +add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%2), %%r8 ;" + "addq (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcq 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcq 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcq 24(%1), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +static __always_inline void +sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%1), %%r8 ;" + "subq (%2), %%r8 ;" + "movq 8(%1), %%r9 ;" + "sbbq 8(%2), %%r9 ;" + "movq 16(%1), %%r10 ;" + "sbbq 16(%2), %%r10 ;" + "movq 24(%1), %%r11 ;" + "sbbq 24(%2), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "sbbq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "sbbq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "sbbq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} + +/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */ +static __always_inline void +mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) +{ + const u64 a24 = 121666; + asm volatile( + "movq %2, %%rdx ;" + "mulx (%1), %%r8, %%r10 ;" + "mulx 8(%1), %%r9, %%r11 ;" + "addq %%r10, %%r9 ;" + "mulx 16(%1), %%r10, %%rax ;" + "adcq %%r11, %%r10 ;" + "mulx 24(%1), %%r11, %%rcx ;" + "adcq %%rax, %%r11 ;" + /**************************/ + "adcq $0, %%rcx ;" + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/ + "imul %%rdx, %%rcx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(a24) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} + +#ifdef CONFIG_AS_ADX +static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[4]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_adx(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 2); + mul_eltfp25519_1w_adx(T[0], a, T[2]); + mul_eltfp25519_1w_adx(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 1); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 10); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 20); + mul_eltfp25519_1w_adx(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 10); + mul_eltfp25519_1w_adx(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_adx(T[0], 50); + mul_eltfp25519_1w_adx(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 100); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 50); + mul_eltfp25519_1w_adx(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[5]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_bmi2(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 2); + mul_eltfp25519_1w_bmi2(T[0], a, T[2]); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 1); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 10); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 20); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 10); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_bmi2(T[0], 50); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 100); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 50); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +/* Given c, a 256-bit number, fred_eltfp25519_1w updates c + * with a number such that 0 <= C < 2**255-19. + */ +static __always_inline void fred_eltfp25519_1w(u64 *const c) +{ + u64 tmp0 = 38, tmp1 = 19; + asm volatile( + "btrq $63, %3 ;" /* Put bit 255 in carry flag and clear */ + "cmovncl %k5, %k4 ;" /* c[255] ? 38 : 19 */ + + /* Add either 19 or 38 to c */ + "addq %4, %0 ;" + "adcq $0, %1 ;" + "adcq $0, %2 ;" + "adcq $0, %3 ;" + + /* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */ + "movl $0, %k4 ;" + "cmovnsl %k5, %k4 ;" /* c[255] ? 0 : 19 */ + "btrq $63, %3 ;" /* Clear bit 255 */ + + /* Subtract 19 if necessary */ + "subq %4, %0 ;" + "sbbq $0, %1 ;" + "sbbq $0, %2 ;" + "sbbq $0, %3 ;" + + : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0), + "+r"(tmp1) + : + : "memory", "cc"); +} + +static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py) +{ + u64 temp; + asm volatile( + "test %9, %9 ;" + "movq %0, %8 ;" + "cmovnzq %4, %0 ;" + "cmovnzq %8, %4 ;" + "movq %1, %8 ;" + "cmovnzq %5, %1 ;" + "cmovnzq %8, %5 ;" + "movq %2, %8 ;" + "cmovnzq %6, %2 ;" + "cmovnzq %8, %6 ;" + "movq %3, %8 ;" + "cmovnzq %7, %3 ;" + "cmovnzq %8, %7 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]), + "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]), + "=r"(temp) + : "r"(bit) + : "cc" + ); +} + +static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py) +{ + asm volatile( + "test %4, %4 ;" + "cmovnzq %5, %0 ;" + "cmovnzq %6, %1 ;" + "cmovnzq %7, %2 ;" + "cmovnzq %8, %3 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]) + : "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3]) + : "cc" + ); +} + +static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_adx(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_adx(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_adx(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_adx(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_adx(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_adx(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_adx(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_adx(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_adx(A, Qz); + mul_eltfp25519_1w_adx((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_adx(C, &P[4 * k], B); /* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_adx(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_adx(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_adx(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_adx(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_adx(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_adx(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_adx(A, Zr1); + mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_bmi2(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_bmi2(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_bmi2(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_bmi2(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_bmi2(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_bmi2(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_bmi2(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_bmi2(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_bmi2(A, Qz); + mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_bmi2(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_bmi2(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_bmi2(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_bmi2(A, Zr1); + mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519.c b/net/wireguard/crypto/zinc/curve25519/curve25519.c new file mode 100644 index 0000000000000..82c669e4123c4 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the Curve25519 ECDH algorithm, using either + * a 32-bit implementation or a 64-bit implementation with 128-bit integers, + * depending on what is supported by the target compiler. + * + * Information: https://cr.yp.to/ecdh.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include // For crypto_memneq. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "curve25519-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) +#include "curve25519-arm-glue.c" +#else +static bool *const curve25519_nobs[] __initconst = { }; +static void __init curve25519_fpu_init(void) +{ +} +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + return false; +} +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "curve25519-hacl64.c" +#else +#include "curve25519-fiat32.c" +#endif + +static const u8 null_point[CURVE25519_KEY_SIZE] = { 0 }; + +bool curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (!curve25519_arch(mypublic, secret, basepoint)) + curve25519_generic(mypublic, secret, basepoint); + return crypto_memneq(mypublic, null_point, CURVE25519_KEY_SIZE); +} +EXPORT_SYMBOL(curve25519); + +bool curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + static const u8 basepoint[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; + + if (unlikely(!crypto_memneq(secret, null_point, CURVE25519_KEY_SIZE))) + return false; + + if (curve25519_base_arch(pub, secret)) + return crypto_memneq(pub, null_point, CURVE25519_KEY_SIZE); + return curve25519(pub, secret, basepoint); +} +EXPORT_SYMBOL(curve25519_generate_public); + +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + get_random_bytes_wait(secret, CURVE25519_KEY_SIZE); + curve25519_clamp_secret(secret); +} +EXPORT_SYMBOL(curve25519_generate_secret); + +#include "../selftest/curve25519.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init curve25519_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + curve25519_fpu_init(); + if (!selftest_run("curve25519", curve25519_selftest, curve25519_nobs, + ARRAY_SIZE(curve25519_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c new file mode 100644 index 0000000000000..291fe4ba98b0c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); +asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool poly1305_use_neon __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { &poly1305_use_neon }; + +static void __init poly1305_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + poly1305_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + poly1305_use_neon = elf_hwcap & HWCAP_NEON; +#endif +} + +#if defined(CONFIG_ZINC_ARCH_ARM64) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1, h2; + }; + }; + u64 is_base2_26; + u64 r[2]; +}; +#elif defined(CONFIG_ZINC_ARCH_ARM) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1; + u32 h2; + } __packed; + }; + u32 r[4]; + u32 is_base2_26; +}; +#endif + +/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit + * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON + * and then having to go back to scalar -- because the user is silly and has + * called the update function from two separate contexts -- then we need to + * convert back to the original base before proceeding. The below function is + * written for 64-bit integers, and so we have to swap words at the end on + * big-endian 32-bit. It is possible to reason that the initial reduction below + * is sufficient given the implementation invariants. However, for an avoidance + * of doubt and because this is not performance critical, we do the full + * reduction anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->h2 = state->h[4] >> 24; + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) { + state->h0 = rol64(state->h0, 32); + state->h1 = rol64(state->h1, 32); + } +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->h2 >> 2) + (state->h2 & ~3ULL); + state->h2 &= 3; + state->h0 += cy; + state->h1 += (cy = ULT(state->h0, cy)); + state->h2 += ULT(state->h1, cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_arm(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_arm(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + poly1305_blocks_neon(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_arm(ctx, mac, nonce); + } else + poly1305_emit_neon(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl new file mode 100644 index 0000000000000..468f41b76fbd2 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl @@ -0,0 +1,1276 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# IALU(*)/gcc-4.4 NEON +# +# ARM11xx(ARMv6) 7.78/+100% - +# Cortex-A5 6.35/+130% 3.00 +# Cortex-A8 6.25/+115% 2.36 +# Cortex-A9 5.10/+95% 2.55 +# Cortex-A15 3.85/+85% 1.25(**) +# Snapdragon S4 5.70/+100% 1.48(**) +# +# (*) this is for -march=armv6, i.e. with bunch of ldrb loading data; +# (**) these are trade-off results, they can be improved by ~8% but at +# the cost of 15/12% regression on Cortex-A5/A7, it's even possible +# to improve Cortex-A9 result, but then A5/A7 loose more than 20%; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +($ctx,$inp,$len,$padbit)=map("r$_",(0..3)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text +#if defined(__thumb2__) +.syntax unified +.thumb +#else +.code 32 +#endif + +.globl poly1305_emit +.globl poly1305_blocks +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: +.Lpoly1305_init: + stmdb sp!,{r4-r11} + + eor r3,r3,r3 + cmp $inp,#0 + str r3,[$ctx,#0] @ zero hash value + str r3,[$ctx,#4] + str r3,[$ctx,#8] + str r3,[$ctx,#12] + str r3,[$ctx,#16] + str r3,[$ctx,#36] @ is_base2_26 + add $ctx,$ctx,#20 + +#ifdef __thumb2__ + it eq +#endif + moveq r0,#0 + beq .Lno_key + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + adr r11,.Lpoly1305_init + ldr r12,.LOPENSSL_armcap +#endif + ldrb r4,[$inp,#0] + mov r10,#0x0fffffff + ldrb r5,[$inp,#1] + and r3,r10,#-4 @ 0x0ffffffc + ldrb r6,[$inp,#2] + ldrb r7,[$inp,#3] + orr r4,r4,r5,lsl#8 + ldrb r5,[$inp,#4] + orr r4,r4,r6,lsl#16 + ldrb r6,[$inp,#5] + orr r4,r4,r7,lsl#24 + ldrb r7,[$inp,#6] + and r4,r4,r10 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,[r11,r12] @ OPENSSL_armcap_P +# ifdef __APPLE__ + ldr r12,[r12] +# endif +#endif + ldrb r8,[$inp,#7] + orr r5,r5,r6,lsl#8 + ldrb r6,[$inp,#8] + orr r5,r5,r7,lsl#16 + ldrb r7,[$inp,#9] + orr r5,r5,r8,lsl#24 + ldrb r8,[$inp,#10] + and r5,r5,r3 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + tst r12,#ARMV7_NEON @ check for NEON +# ifdef __APPLE__ + adr r9,poly1305_blocks_neon + adr r11,poly1305_blocks +# ifdef __thumb2__ + it ne +# endif + movne r11,r9 + adr r12,poly1305_emit + adr r10,poly1305_emit_neon +# ifdef __thumb2__ + it ne +# endif + movne r12,r10 +# else +# ifdef __thumb2__ + itete eq +# endif + addeq r12,r11,#(poly1305_emit-.Lpoly1305_init) + addne r12,r11,#(poly1305_emit_neon-.Lpoly1305_init) + addeq r11,r11,#(poly1305_blocks-.Lpoly1305_init) + addne r11,r11,#(poly1305_blocks_neon-.Lpoly1305_init) +# endif +# ifdef __thumb2__ + orr r12,r12,#1 @ thumb-ify address + orr r11,r11,#1 +# endif +#endif + ldrb r9,[$inp,#11] + orr r6,r6,r7,lsl#8 + ldrb r7,[$inp,#12] + orr r6,r6,r8,lsl#16 + ldrb r8,[$inp,#13] + orr r6,r6,r9,lsl#24 + ldrb r9,[$inp,#14] + and r6,r6,r3 + + ldrb r10,[$inp,#15] + orr r7,r7,r8,lsl#8 + str r4,[$ctx,#0] + orr r7,r7,r9,lsl#16 + str r5,[$ctx,#4] + orr r7,r7,r10,lsl#24 + str r6,[$ctx,#8] + and r7,r7,r3 + str r7,[$ctx,#12] +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + stmia r2,{r11,r12} @ fill functions table + mov r0,#1 +#else + mov r0,#0 +#endif +.Lno_key: + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_init,.-poly1305_init +___ +{ +my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12)); +my ($s1,$s2,$s3)=($r1,$r2,$r3); + +$code.=<<___; +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: +.Lpoly1305_blocks: + stmdb sp!,{r3-r11,lr} + + ands $len,$len,#-16 + beq .Lno_data + + cmp $padbit,#0 + add $len,$len,$inp @ end pointer + sub sp,sp,#32 + + ldmia $ctx,{$h0-$r3} @ load context + + str $ctx,[sp,#12] @ offload stuff + mov lr,$inp + str $len,[sp,#16] + str $r1,[sp,#20] + str $r2,[sp,#24] + str $r3,[sp,#28] + b .Loop + +.Loop: +#if __ARM_ARCH__<7 + ldrb r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ 1<<128 + ldrb r1,[lr,#-15] + ldrb r2,[lr,#-14] + ldrb r3,[lr,#-13] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-12] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-11] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-10] + adds $h0,$h0,r3 @ accumulate input + + ldrb r3,[lr,#-9] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-8] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-7] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-6] + adcs $h1,$h1,r3 + + ldrb r3,[lr,#-5] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-4] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-3] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-2] + adcs $h2,$h2,r3 + + ldrb r3,[lr,#-1] + orr r1,r0,r1,lsl#8 + str lr,[sp,#8] @ offload input pointer + orr r2,r1,r2,lsl#16 + add $s1,$r1,$r1,lsr#2 + orr r3,r2,r3,lsl#24 +#else + ldr r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ padbit + ldr r1,[lr,#-12] + ldr r2,[lr,#-8] + ldr r3,[lr,#-4] +# ifdef __ARMEB__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +# endif + adds $h0,$h0,r0 @ accumulate input + str lr,[sp,#8] @ offload input pointer + adcs $h1,$h1,r1 + add $s1,$r1,$r1,lsr#2 + adcs $h2,$h2,r2 +#endif + add $s2,$r2,$r2,lsr#2 + adcs $h3,$h3,r3 + add $s3,$r3,$r3,lsr#2 + + umull r2,r3,$h1,$r0 + adc $h4,$h4,#0 + umull r0,r1,$h0,$r0 + umlal r2,r3,$h4,$s1 + umlal r0,r1,$h3,$s1 + ldr $r1,[sp,#20] @ reload $r1 + umlal r2,r3,$h2,$s3 + umlal r0,r1,$h1,$s3 + umlal r2,r3,$h3,$s2 + umlal r0,r1,$h2,$s2 + umlal r2,r3,$h0,$r1 + str r0,[sp,#0] @ future $h0 + mul r0,$s2,$h4 + ldr $r2,[sp,#24] @ reload $r2 + adds r2,r2,r1 @ d1+=d0>>32 + eor r1,r1,r1 + adc lr,r3,#0 @ future $h2 + str r2,[sp,#4] @ future $h1 + + mul r2,$s3,$h4 + eor r3,r3,r3 + umlal r0,r1,$h3,$s3 + ldr $r3,[sp,#28] @ reload $r3 + umlal r2,r3,$h3,$r0 + umlal r0,r1,$h2,$r0 + umlal r2,r3,$h2,$r1 + umlal r0,r1,$h1,$r1 + umlal r2,r3,$h1,$r2 + umlal r0,r1,$h0,$r2 + umlal r2,r3,$h0,$r3 + ldr $h0,[sp,#0] + mul $h4,$r0,$h4 + ldr $h1,[sp,#4] + + adds $h2,lr,r0 @ d2+=d1>>32 + ldr lr,[sp,#8] @ reload input pointer + adc r1,r1,#0 + adds $h3,r2,r1 @ d3+=d2>>32 + ldr r0,[sp,#16] @ reload end pointer + adc r3,r3,#0 + add $h4,$h4,r3 @ h4+=d3>>32 + + and r1,$h4,#-4 + and $h4,$h4,#3 + add r1,r1,r1,lsr#2 @ *=5 + adds $h0,$h0,r1 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + cmp r0,lr @ done yet? + bhi .Loop + + ldr $ctx,[sp,#12] + add sp,sp,#32 + stmia $ctx,{$h0-$h4} @ store the result + +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r3-r11,pc} +#else + ldmia sp!,{r3-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_blocks,.-poly1305_blocks +___ +} +{ +my ($ctx,$mac,$nonce)=map("r$_",(0..2)); +my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11)); +my $g4=$h4; + +$code.=<<___; +.type poly1305_emit,%function +.align 5 +poly1305_emit: + stmdb sp!,{r4-r11} +.Lpoly1305_emit_enter: + + ldmia $ctx,{$h0-$h4} + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + +#ifdef __thumb2__ + it ne +#endif + movne $h0,$g0 + ldr $g0,[$nonce,#0] +#ifdef __thumb2__ + it ne +#endif + movne $h1,$g1 + ldr $g1,[$nonce,#4] +#ifdef __thumb2__ + it ne +#endif + movne $h2,$g2 + ldr $g2,[$nonce,#8] +#ifdef __thumb2__ + it ne +#endif + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +#if __ARM_ARCH__>=7 +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] +#else + strb $h0,[$mac,#0] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#4] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#8] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#12] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#1] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#5] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#9] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#13] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#2] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#6] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#10] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#14] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#3] + strb $h1,[$mac,#7] + strb $h2,[$mac,#11] + strb $h3,[$mac,#15] +#endif + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_emit,.-poly1305_emit +___ +{ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9)); +my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14)); +my ($T0,$T1,$MASK) = map("q$_",(15,4,0)); + +my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7)); + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.fpu neon + +.type poly1305_init_neon,%function +.align 5 +poly1305_init_neon: +.Lpoly1305_init_neon: + ldr r4,[$ctx,#20] @ load key base 2^32 + ldr r5,[$ctx,#24] + ldr r6,[$ctx,#28] + ldr r7,[$ctx,#32] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + and r3,r3,#0x03ffffff + and r4,r4,#0x03ffffff + and r5,r5,#0x03ffffff + + vdup.32 $R0,r2 @ r^1 in both lanes + add r2,r3,r3,lsl#2 @ *5 + vdup.32 $R1,r3 + add r3,r4,r4,lsl#2 + vdup.32 $S1,r2 + vdup.32 $R2,r4 + add r4,r5,r5,lsl#2 + vdup.32 $S2,r3 + vdup.32 $R3,r5 + add r5,r6,r6,lsl#2 + vdup.32 $S3,r4 + vdup.32 $R4,r6 + vdup.32 $S4,r5 + + mov $zeros,#2 @ counter + +.Lsquare_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + + vmull.u32 $D0,$R0,${R0}[1] + vmull.u32 $D1,$R1,${R0}[1] + vmull.u32 $D2,$R2,${R0}[1] + vmull.u32 $D3,$R3,${R0}[1] + vmull.u32 $D4,$R4,${R0}[1] + + vmlal.u32 $D0,$R4,${S1}[1] + vmlal.u32 $D1,$R0,${R1}[1] + vmlal.u32 $D2,$R1,${R1}[1] + vmlal.u32 $D3,$R2,${R1}[1] + vmlal.u32 $D4,$R3,${R1}[1] + + vmlal.u32 $D0,$R3,${S2}[1] + vmlal.u32 $D1,$R4,${S2}[1] + vmlal.u32 $D3,$R1,${R2}[1] + vmlal.u32 $D2,$R0,${R2}[1] + vmlal.u32 $D4,$R2,${R2}[1] + + vmlal.u32 $D0,$R2,${S3}[1] + vmlal.u32 $D3,$R0,${R3}[1] + vmlal.u32 $D1,$R3,${S3}[1] + vmlal.u32 $D2,$R4,${S3}[1] + vmlal.u32 $D4,$R1,${R3}[1] + + vmlal.u32 $D3,$R4,${S4}[1] + vmlal.u32 $D0,$R1,${S4}[1] + vmlal.u32 $D1,$R2,${S4}[1] + vmlal.u32 $D2,$R3,${S4}[1] + vmlal.u32 $D4,$R0,${R4}[1] + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + @ and P. Schwabe + @ + @ H0>>+H1>>+H2>>+H3>>+H4 + @ H3>>+H4>>*5+H0>>+H1 + @ + @ Trivia. + @ + @ Result of multiplication of n-bit number by m-bit number is + @ n+m bits wide. However! Even though 2^n is a n+1-bit number, + @ m-bit number multiplied by 2^n is still n+m bits wide. + @ + @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2, + @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit + @ one is n+1 bits wide. + @ + @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that + @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4 + @ can be 27. However! In cases when their width exceeds 26 bits + @ they are limited by 2^26+2^6. This in turn means that *sum* + @ of the products with these values can still be viewed as sum + @ of 52-bit numbers as long as the amount of addends is not a + @ power of 2. For example, + @ + @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4, + @ + @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or + @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than + @ 8 * (2^52) or 2^55. However, the value is then multiplied by + @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12), + @ which is less than 32 * (2^52) or 2^57. And when processing + @ data we are looking at triple as many addends... + @ + @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and + @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the + @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while + @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32 + @ instruction accepts 2x32-bit input and writes 2x64-bit result. + @ This means that result of reduction have to be compressed upon + @ loop wrap-around. This can be done in the process of reduction + @ to minimize amount of instructions [as well as amount of + @ 128-bit instructions, which benefits low-end processors], but + @ one has to watch for H2 (which is narrower than H0) and 5*H4 + @ not being wider than 58 bits, so that result of right shift + @ by 26 bits fits in 32 bits. This is also useful on x86, + @ because it allows to use paddd in place for paddq, which + @ benefits Atom, where paddq is ridiculously slow. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vbic.i32 $D4#lo,#0xfc000000 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vbic.i32 $D2#lo,#0xfc000000 + + vshr.u32 $T0#lo,$D0#lo,#26 + vbic.i32 $D0#lo,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + + subs $zeros,$zeros,#1 + beq .Lsquare_break_neon + + add $tbl0,$ctx,#(48+0*9*4) + add $tbl1,$ctx,#(48+1*9*4) + + vtrn.32 $R0,$D0#lo @ r^2:r^1 + vtrn.32 $R2,$D2#lo + vtrn.32 $R3,$D3#lo + vtrn.32 $R1,$D1#lo + vtrn.32 $R4,$D4#lo + + vshl.u32 $S2,$R2,#2 @ *5 + vshl.u32 $S3,$R3,#2 + vshl.u32 $S1,$R1,#2 + vshl.u32 $S4,$R4,#2 + vadd.i32 $S2,$S2,$R2 + vadd.i32 $S1,$S1,$R1 + vadd.i32 $S3,$S3,$R3 + vadd.i32 $S4,$S4,$R4 + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0,:32] + vst1.32 {${S4}[1]},[$tbl1,:32] + + b .Lsquare_neon + +.align 4 +.Lsquare_break_neon: + add $tbl0,$ctx,#(48+2*4*9) + add $tbl1,$ctx,#(48+3*4*9) + + vmov $R0,$D0#lo @ r^4:r^3 + vshl.u32 $S1,$D1#lo,#2 @ *5 + vmov $R1,$D1#lo + vshl.u32 $S2,$D2#lo,#2 + vmov $R2,$D2#lo + vshl.u32 $S3,$D3#lo,#2 + vmov $R3,$D3#lo + vshl.u32 $S4,$D4#lo,#2 + vmov $R4,$D4#lo + vadd.i32 $S1,$S1,$D1#lo + vadd.i32 $S2,$S2,$D2#lo + vadd.i32 $S3,$S3,$D3#lo + vadd.i32 $S4,$S4,$D4#lo + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0] + vst1.32 {${S4}[1]},[$tbl1] + + ret @ bx lr +.size poly1305_init_neon,.-poly1305_init_neon + +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +#endif +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + ands $len,$len,#-16 + beq .Lno_data_neon + + cmp $len,#64 + bhs .Lenter_neon + tst ip,ip @ is_base2_26? + beq .Lpoly1305_blocks + +.Lenter_neon: + stmdb sp!,{r4-r7} + vstmdb sp!,{d8-d15} @ ABI specification says so + + tst ip,ip @ is_base2_26? + bne .Lbase2_26_neon + + stmdb sp!,{r1-r3,lr} + bl .Lpoly1305_init_neon + + ldr r4,[$ctx,#0] @ load hash value base 2^32 + ldr r5,[$ctx,#4] + ldr r6,[$ctx,#8] + ldr r7,[$ctx,#12] + ldr ip,[$ctx,#16] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + veor $D0#lo,$D0#lo,$D0#lo + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + veor $D1#lo,$D1#lo,$D1#lo + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + veor $D2#lo,$D2#lo,$D2#lo + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + veor $D3#lo,$D3#lo,$D3#lo + and r3,r3,#0x03ffffff + orr r6,r6,ip,lsl#24 + veor $D4#lo,$D4#lo,$D4#lo + and r4,r4,#0x03ffffff + mov r1,#1 + and r5,r5,#0x03ffffff + str r1,[$ctx,#36] @ is_base2_26 + + vmov.32 $D0#lo[0],r2 + vmov.32 $D1#lo[0],r3 + vmov.32 $D2#lo[0],r4 + vmov.32 $D3#lo[0],r5 + vmov.32 $D4#lo[0],r6 + adr $zeros,.Lzeros + + ldmia sp!,{r1-r3,lr} + b .Lbase2_32_neon + +.align 4 +.Lbase2_26_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ load hash value + + veor $D0#lo,$D0#lo,$D0#lo + veor $D1#lo,$D1#lo,$D1#lo + veor $D2#lo,$D2#lo,$D2#lo + veor $D3#lo,$D3#lo,$D3#lo + veor $D4#lo,$D4#lo,$D4#lo + vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + adr $zeros,.Lzeros + vld1.32 {$D4#lo[0]},[$ctx] + sub $ctx,$ctx,#16 @ rewind + +.Lbase2_32_neon: + add $in2,$inp,#32 + mov $padbit,$padbit,lsl#24 + tst $len,#31 + beq .Leven + + vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]! + vmov.32 $H4#lo[0],$padbit + sub $len,$len,#16 + add $in2,$inp,#32 + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3#lo,$H3#lo,#18 + + vsri.u32 $H3#lo,$H2#lo,#14 + vshl.u32 $H2#lo,$H2#lo,#12 + vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi + + vbic.i32 $H3#lo,#0xfc000000 + vsri.u32 $H2#lo,$H1#lo,#20 + vshl.u32 $H1#lo,$H1#lo,#6 + + vbic.i32 $H2#lo,#0xfc000000 + vsri.u32 $H1#lo,$H0#lo,#26 + vadd.i32 $H3#hi,$H3#lo,$D3#lo + + vbic.i32 $H0#lo,#0xfc000000 + vbic.i32 $H1#lo,#0xfc000000 + vadd.i32 $H2#hi,$H2#lo,$D2#lo + + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + + mov $tbl1,$zeros + add $tbl0,$ctx,#48 + + cmp $len,$len + b .Long_tail + +.align 4 +.Leven: + subs $len,$len,#64 + it lo + movlo $in2,$zeros + + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + itt hi + addhi $tbl1,$ctx,#(48+1*9*4) + addhi $tbl0,$ctx,#(48+3*9*4) + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3,$H3,#18 + + vsri.u32 $H3,$H2,#14 + vshl.u32 $H2,$H2,#12 + + vbic.i32 $H3,#0xfc000000 + vsri.u32 $H2,$H1,#20 + vshl.u32 $H1,$H1,#6 + + vbic.i32 $H2,#0xfc000000 + vsri.u32 $H1,$H0,#26 + + vbic.i32 $H0,#0xfc000000 + vbic.i32 $H1,#0xfc000000 + + bls .Lskip_loop + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + b .Loop_neon + +.align 5 +.Loop_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + @ \___________________/ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + @ \___________________/ \____________________/ + @ + @ Note that we start with inp[2:3]*r^2. This is because it + @ doesn't depend on reduction in previous iteration. + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ inp[2:3]*r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1] + vmull.u32 $D2,$H2#hi,${R0}[1] + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,${R0}[1] + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,${R0}[1] + vmlal.u32 $D2,$H1#hi,${R1}[1] + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,${R0}[1] + + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,${R0}[1] + subs $len,$len,#64 + vmlal.u32 $D0,$H4#hi,${S1}[1] + it lo + movlo $in2,$zeros + vmlal.u32 $D3,$H2#hi,${R1}[1] + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D1,$H0#hi,${R1}[1] + vmlal.u32 $D4,$H3#hi,${R1}[1] + + vmlal.u32 $D0,$H3#hi,${S2}[1] + vmlal.u32 $D3,$H1#hi,${R2}[1] + vmlal.u32 $D4,$H2#hi,${R2}[1] + vmlal.u32 $D1,$H4#hi,${S2}[1] + vmlal.u32 $D2,$H0#hi,${R2}[1] + + vmlal.u32 $D3,$H0#hi,${R3}[1] + vmlal.u32 $D0,$H2#hi,${S3}[1] + vmlal.u32 $D4,$H1#hi,${R3}[1] + vmlal.u32 $D1,$H3#hi,${S3}[1] + vmlal.u32 $D2,$H4#hi,${S3}[1] + + vmlal.u32 $D3,$H4#hi,${S4}[1] + vmlal.u32 $D0,$H1#hi,${S4}[1] + vmlal.u32 $D4,$H0#hi,${R4}[1] + vmlal.u32 $D1,$H2#hi,${S4}[1] + vmlal.u32 $D2,$H3#hi,${S4}[1] + + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4 and accumulate + + vmlal.u32 $D3,$H3#lo,${R0}[0] + vmlal.u32 $D0,$H0#lo,${R0}[0] + vmlal.u32 $D4,$H4#lo,${R0}[0] + vmlal.u32 $D1,$H1#lo,${R0}[0] + vmlal.u32 $D2,$H2#lo,${R0}[0] + vld1.32 ${S4}[0],[$tbl0,:32] + + vmlal.u32 $D3,$H2#lo,${R1}[0] + vmlal.u32 $D0,$H4#lo,${S1}[0] + vmlal.u32 $D4,$H3#lo,${R1}[0] + vmlal.u32 $D1,$H0#lo,${R1}[0] + vmlal.u32 $D2,$H1#lo,${R1}[0] + + vmlal.u32 $D3,$H1#lo,${R2}[0] + vmlal.u32 $D0,$H3#lo,${S2}[0] + vmlal.u32 $D4,$H2#lo,${R2}[0] + vmlal.u32 $D1,$H4#lo,${S2}[0] + vmlal.u32 $D2,$H0#lo,${R2}[0] + + vmlal.u32 $D3,$H0#lo,${R3}[0] + vmlal.u32 $D0,$H2#lo,${S3}[0] + vmlal.u32 $D4,$H1#lo,${R3}[0] + vmlal.u32 $D1,$H3#lo,${S3}[0] + vmlal.u32 $D3,$H4#lo,${S4}[0] + + vmlal.u32 $D2,$H4#lo,${S3}[0] + vmlal.u32 $D0,$H1#lo,${S4}[0] + vmlal.u32 $D4,$H0#lo,${R4}[0] + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vmlal.u32 $D1,$H2#lo,${S4}[0] + vmlal.u32 $D2,$H3#lo,${S4}[0] + + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 + vrev32.8 $H3,$H3 +# endif + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction interleaved with base 2^32 -> base 2^26 of + @ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vshl.u32 $H3,$H3,#18 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vsri.u32 $H3,$H2,#14 + vbic.i32 $D4#lo,#0xfc000000 + vshl.u32 $H2,$H2,#12 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vbic.i32 $H3,#0xfc000000 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec] + vsri.u32 $H2,$H1,#20 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vshl.u32 $H1,$H1,#6 + vbic.i32 $D2#lo,#0xfc000000 + vbic.i32 $H2,#0xfc000000 + + vshrn.u64 $T0#lo,$D0,#26 @ re-narrow + vmovn.i64 $D0#lo,$D0 + vsri.u32 $H1,$H0,#26 + vbic.i32 $H0,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vbic.i32 $D0#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + vbic.i32 $H1,#0xfc000000 + + bhi .Loop_neon + +.Lskip_loop: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + add $tbl1,$ctx,#(48+0*9*4) + add $tbl0,$ctx,#(48+1*9*4) + adds $len,$len,#32 + it ne + movne $len,#0 + bne .Long_tail + + vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H3#hi,$H3#lo,$D3#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + vadd.i32 $H4#hi,$H4#lo,$D4#lo + +.Long_tail: + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant + vmull.u32 $D2,$H2#hi,$R0 + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,$R0 + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,$R0 + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,$R0 + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,$R0 + + vmlal.u32 $D0,$H4#hi,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#hi,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#hi,$R1 + vmlal.u32 $D4,$H3#hi,$R1 + vmlal.u32 $D2,$H1#hi,$R1 + + vmlal.u32 $D3,$H1#hi,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#hi,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#hi,$R2 + vmlal.u32 $D1,$H4#hi,$S2 + vmlal.u32 $D2,$H0#hi,$R2 + + vmlal.u32 $D3,$H0#hi,$R3 + it ne + addne $tbl1,$ctx,#(48+2*9*4) + vmlal.u32 $D0,$H2#hi,$S3 + it ne + addne $tbl0,$ctx,#(48+3*9*4) + vmlal.u32 $D4,$H1#hi,$R3 + vmlal.u32 $D1,$H3#hi,$S3 + vmlal.u32 $D2,$H4#hi,$S3 + + vmlal.u32 $D3,$H4#hi,$S4 + vorn $MASK,$MASK,$MASK @ all-ones, can be redundant + vmlal.u32 $D0,$H1#hi,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#hi,$R4 + vmlal.u32 $D1,$H2#hi,$S4 + vmlal.u32 $D2,$H3#hi,$S4 + + beq .Lshort_tail + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4:r^3 and accumulate + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + + vmlal.u32 $D2,$H2#lo,$R0 + vmlal.u32 $D0,$H0#lo,$R0 + vmlal.u32 $D3,$H3#lo,$R0 + vmlal.u32 $D1,$H1#lo,$R0 + vmlal.u32 $D4,$H4#lo,$R0 + + vmlal.u32 $D0,$H4#lo,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#lo,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#lo,$R1 + vmlal.u32 $D4,$H3#lo,$R1 + vmlal.u32 $D2,$H1#lo,$R1 + + vmlal.u32 $D3,$H1#lo,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#lo,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#lo,$R2 + vmlal.u32 $D1,$H4#lo,$S2 + vmlal.u32 $D2,$H0#lo,$R2 + + vmlal.u32 $D3,$H0#lo,$R3 + vmlal.u32 $D0,$H2#lo,$S3 + vmlal.u32 $D4,$H1#lo,$R3 + vmlal.u32 $D1,$H3#lo,$S3 + vmlal.u32 $D2,$H4#lo,$S3 + + vmlal.u32 $D3,$H4#lo,$S4 + vorn $MASK,$MASK,$MASK @ all-ones + vmlal.u32 $D0,$H1#lo,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#lo,$R4 + vmlal.u32 $D1,$H2#lo,$S4 + vmlal.u32 $D2,$H3#lo,$S4 + +.Lshort_tail: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ horizontal addition + + vadd.i64 $D3#lo,$D3#lo,$D3#hi + vadd.i64 $D0#lo,$D0#lo,$D0#hi + vadd.i64 $D4#lo,$D4#lo,$D4#hi + vadd.i64 $D1#lo,$D1#lo,$D1#hi + vadd.i64 $D2#lo,$D2#lo,$D2#hi + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction, but without narrowing + + vshr.u64 $T0,$D3,#26 + vand.i64 $D3,$D3,$MASK + vshr.u64 $T1,$D0,#26 + vand.i64 $D0,$D0,$MASK + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + + vshr.u64 $T0,$D4,#26 + vand.i64 $D4,$D4,$MASK + vshr.u64 $T1,$D1,#26 + vand.i64 $D1,$D1,$MASK + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + + vadd.i64 $D0,$D0,$T0 + vshl.u64 $T0,$T0,#2 + vshr.u64 $T1,$D2,#26 + vand.i64 $D2,$D2,$MASK + vadd.i64 $D0,$D0,$T0 @ h4 -> h0 + vadd.i64 $D3,$D3,$T1 @ h2 -> h3 + + vshr.u64 $T0,$D0,#26 + vand.i64 $D0,$D0,$MASK + vshr.u64 $T1,$D3,#26 + vand.i64 $D3,$D3,$MASK + vadd.i64 $D1,$D1,$T0 @ h0 -> h1 + vadd.i64 $D4,$D4,$T1 @ h3 -> h4 + + cmp $len,#0 + bne .Leven + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ store hash value + + vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + vst1.32 {$D4#lo[0]},[$ctx] + + vldmia sp!,{d8-d15} @ epilogue + ldmia sp!,{r4-r7} +.Lno_data_neon: + ret @ bx lr +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +#ifdef __KERNEL__ +.globl poly1305_emit_neon +#endif +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + + stmdb sp!,{r4-r11} + + tst ip,ip + beq .Lpoly1305_emit_enter + + ldmia $ctx,{$h0-$h4} + eor $g0,$g0,$g0 + + adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32 + mov $h1,$h1,lsr#6 + adcs $h1,$h1,$h2,lsl#20 + mov $h2,$h2,lsr#12 + adcs $h2,$h2,$h3,lsl#14 + mov $h3,$h3,lsr#18 + adcs $h3,$h3,$h4,lsl#8 + adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ... + + and $g0,$h4,#-4 @ ... so reduce + and $h4,$h3,#3 + add $g0,$g0,$g0,lsr#2 @ *= 5 + adds $h0,$h0,$g0 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + + it ne + movne $h0,$g0 + ldr $g0,[$nonce,#0] + it ne + movne $h1,$g1 + ldr $g1,[$nonce,#4] + it ne + movne $h2,$g2 + ldr $g2,[$nonce,#8] + it ne + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 @ accumulate nonce + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] @ store the result + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] + + ldmia sp!,{r4-r11} + ret @ bx lr +.size poly1305_emit_neon,.-poly1305_emit_neon + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +# ifndef __KERNEL__ +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.Lpoly1305_init +# endif +#endif +___ +} } +$code.=<<___; +.align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} +close STDOUT; # enforce flush diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl new file mode 100644 index 0000000000000..d513b45a149bf --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl @@ -0,0 +1,974 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for ARMv8. +# +# June 2015 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc-4.9 NEON +# +# Apple A7 1.86/+5% 0.72 +# Cortex-A53 2.69/+58% 1.47 +# Cortex-A57 2.70/+7% 1.14 +# Denver 1.64/+50% 1.18(*) +# X-Gene 2.13/+68% 2.27 +# Mongoose 1.77/+75% 1.12 +# Kryo 2.70/+55% 1.13 +# +# (*) estimate based on resources availability is less than 1.0, +# i.e. measured result is worse than expected, presumably binary +# translator is not almighty; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3)); +my ($mac,$nonce)=($inp,$len); + +my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text + +// forward "declarations" are required for Apple +.globl poly1305_blocks +.globl poly1305_emit +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: + cmp $inp,xzr + stp xzr,xzr,[$ctx] // zero hash value + stp xzr,xzr,[$ctx,#16] // [along with is_base2_26] + + csel x0,xzr,x0,eq + b.eq .Lno_key + +#ifndef __KERNEL__ +# ifdef __ILP32__ + ldrsw $t1,.LOPENSSL_armcap_P +# else + ldr $t1,.LOPENSSL_armcap_P +# endif + adr $t0,.LOPENSSL_armcap_P + ldr w17,[$t0,$t1] +#endif + + ldp $r0,$r1,[$inp] // load key + mov $s1,#0xfffffffc0fffffff + movk $s1,#0x0fff,lsl#48 +#ifdef __AARCH64EB__ + rev $r0,$r0 // flip bytes + rev $r1,$r1 +#endif + and $r0,$r0,$s1 // &=0ffffffc0fffffff + and $s1,$s1,#-4 + and $r1,$r1,$s1 // &=0ffffffc0ffffffc + stp $r0,$r1,[$ctx,#32] // save key value + +#ifndef __KERNEL__ + tst w17,#ARMV7_NEON + + adr $d0,poly1305_blocks + adr $r0,poly1305_blocks_neon + adr $d1,poly1305_emit + adr $r1,poly1305_emit_neon + + csel $d0,$d0,$r0,eq + csel $d1,$d1,$r1,eq + +# ifdef __ILP32__ + stp w12,w13,[$len] +# else + stp $d0,$d1,[$len] +# endif + + mov x0,#1 +#else + mov x0,#0 +#endif +.Lno_key: + ret +.size poly1305_init,.-poly1305_init + +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: + ands $len,$len,#-16 + b.eq .Lno_data + + ldp $h0,$h1,[$ctx] // load hash value + ldp $r0,$r1,[$ctx,#32] // load key value + ldr $h2,[$ctx,#16] + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + b .Loop + +.align 5 +.Loop: + ldp $t0,$t1,[$inp],#16 // load input + sub $len,$len,#16 +#ifdef __AARCH64EB__ + rev $t0,$t0 + rev $t1,$t1 +#endif + adds $h0,$h0,$t0 // accumulate input + adcs $h1,$h1,$t1 + + mul $d0,$h0,$r0 // h0*r0 + adc $h2,$h2,$padbit + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + cbnz $len,.Loop + + stp $h0,$h1,[$ctx] // store hash value + str $h2,[$ctx,#16] + +.Lno_data: + ret +.size poly1305_blocks,.-poly1305_blocks + +.type poly1305_emit,%function +.align 5 +poly1305_emit: + ldp $h0,$h1,[$ctx] // load hash base 2^64 + ldr $h2,[$ctx,#16] + ldp $t0,$t1,[$nonce] // load nonce + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit,.-poly1305_emit +___ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8)); +my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13)); +my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18)); +my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23)); +my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28)); +my ($T0,$T1,$MASK) = map("v$_",(29..31)); + +my ($in2,$zeros)=("x16","x17"); +my $is_base2_26 = $zeros; # borrow + +$code.=<<___; +.type __poly1305_mult,%function +.align 5 +__poly1305_mult: + mul $d0,$h0,$r0 // h0*r0 + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + ret +.size __poly1305_mult,.-__poly1305_mult + +.type __poly1305_splat,%function +.align 5 +__poly1305_splat: + and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x13,$h0,#26,#26 + extr x14,$h1,$h0,#52 + and x14,x14,#0x03ffffff + ubfx x15,$h1,#14,#26 + extr x16,$h2,$h1,#40 + + str w12,[$ctx,#16*0] // r0 + add w12,w13,w13,lsl#2 // r1*5 + str w13,[$ctx,#16*1] // r1 + add w13,w14,w14,lsl#2 // r2*5 + str w12,[$ctx,#16*2] // s1 + str w14,[$ctx,#16*3] // r2 + add w14,w15,w15,lsl#2 // r3*5 + str w13,[$ctx,#16*4] // s2 + str w15,[$ctx,#16*5] // r3 + add w15,w16,w16,lsl#2 // r4*5 + str w14,[$ctx,#16*6] // s3 + str w16,[$ctx,#16*7] // r4 + str w15,[$ctx,#16*8] // s4 + + ret +.size __poly1305_splat,.-__poly1305_splat + +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +.globl poly1305_emit_neon +#endif + +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr $is_base2_26,[$ctx,#24] + cmp $len,#128 + b.hs .Lblocks_neon + cbz $is_base2_26,poly1305_blocks + +.Lblocks_neon: + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + + ands $len,$len,#-16 + b.eq .Lno_data_neon + + cbz $is_base2_26,.Lbase2_64_neon + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + tst $len,#31 + b.eq .Leven_neon + + ldp $r0,$r1,[$ctx,#32] // load key value + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $d2,$h2,xzr // can be partially reduced... + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + + and $t0,$d2,#-4 // ... so reduce + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$h0,$t0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + ldr x30,[sp,#8] + + cbz $padbit,.Lstore_base2_64_neon + + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + cbnz $len,.Leven_neon + + stp w10,w11,[$ctx] // store hash value base 2^26 + stp w12,w13,[$ctx,#8] + str w14,[$ctx,#16] + b .Lno_data_neon + +.align 4 +.Lstore_base2_64_neon: + stp $h0,$h1,[$ctx] // store hash value base 2^64 + stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed + b .Lno_data_neon + +.align 4 +.Lbase2_64_neon: + ldp $r0,$r1,[$ctx,#32] // load key value + + ldp $h0,$h1,[$ctx] // load hash value base 2^64 + ldr $h2,[$ctx,#16] + + tst $len,#31 + b.eq .Linit_neon + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + +.Linit_neon: + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + + ////////////////////////////////// initialize r^n table + mov $h0,$r0 // r^1 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + mov $h1,$r1 + mov $h2,xzr + add $ctx,$ctx,#48+12 + bl __poly1305_splat + + bl __poly1305_mult // r^2 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^3 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^4 + sub $ctx,$ctx,#4 + bl __poly1305_splat + ldr x30,[sp,#8] + + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + mov x4,#1 + str x4,[$ctx,#-24] // set is_base2_26 + sub $ctx,$ctx,#48 // restore original $ctx + b .Ldo_neon + +.align 4 +.Leven_neon: + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + +.Ldo_neon: + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + ldp x9,x13,[$in2],#48 + + lsl $padbit,$padbit,#24 + add x15,$ctx,#48 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN23_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN23_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + fmov $IN23_2,x8 + fmov $IN23_3,x10 + fmov $IN23_4,x12 + + ldp x8,x12,[$inp],#16 // inp[0:1] + ldp x9,x13,[$inp],#48 + + ld1 {$R0,$R1,$S1,$R2},[x15],#64 + ld1 {$S2,$R3,$S3,$R4},[x15],#64 + ld1 {$S4},[x15] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN01_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN01_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + movi $MASK.2d,#-1 + fmov $IN01_2,x8 + fmov $IN01_3,x10 + fmov $IN01_4,x12 + ushr $MASK.2d,$MASK.2d,#38 + + b.ls .Lskip_loop + +.align 4 +.Loop_neon: + //////////////////////////////////////////////////////////////// + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + // \___________________/ + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + // \___________________/ \____________________/ + // + // Note that we start with inp[2:3]*r^2. This is because it + // doesn't depend on reduction in previous iteration. + //////////////////////////////////////////////////////////////// + // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 + // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 + // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 + // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 + // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 + + subs $len,$len,#64 + umull $ACC4,$IN23_0,${R4}[2] + csel $in2,$zeros,$in2,lo + umull $ACC3,$IN23_0,${R3}[2] + umull $ACC2,$IN23_0,${R2}[2] + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + umull $ACC1,$IN23_0,${R1}[2] + ldp x9,x13,[$in2],#48 + umull $ACC0,$IN23_0,${R0}[2] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + umlal $ACC4,$IN23_1,${R3}[2] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC3,$IN23_1,${R2}[2] + and x5,x9,#0x03ffffff + umlal $ACC2,$IN23_1,${R1}[2] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN23_1,${R0}[2] + ubfx x7,x9,#26,#26 + umlal $ACC0,$IN23_1,${S4}[2] + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + + umlal $ACC4,$IN23_2,${R2}[2] + extr x8,x12,x8,#52 + umlal $ACC3,$IN23_2,${R1}[2] + extr x9,x13,x9,#52 + umlal $ACC2,$IN23_2,${R0}[2] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC1,$IN23_2,${S4}[2] + fmov $IN23_0,x4 + umlal $ACC0,$IN23_2,${S3}[2] + and x8,x8,#0x03ffffff + + umlal $ACC4,$IN23_3,${R1}[2] + and x9,x9,#0x03ffffff + umlal $ACC3,$IN23_3,${R0}[2] + ubfx x10,x12,#14,#26 + umlal $ACC2,$IN23_3,${S4}[2] + ubfx x11,x13,#14,#26 + umlal $ACC1,$IN23_3,${S3}[2] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC0,$IN23_3,${S2}[2] + fmov $IN23_1,x6 + + add $IN01_2,$IN01_2,$H2 + add x12,$padbit,x12,lsr#40 + umlal $ACC4,$IN23_4,${R0}[2] + add x13,$padbit,x13,lsr#40 + umlal $ACC3,$IN23_4,${S4}[2] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC2,$IN23_4,${S3}[2] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN23_4,${S2}[2] + fmov $IN23_2,x8 + umlal $ACC0,$IN23_4,${S1}[2] + fmov $IN23_3,x10 + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4 and accumulate + + add $IN01_0,$IN01_0,$H0 + fmov $IN23_4,x12 + umlal $ACC3,$IN01_2,${R1}[0] + ldp x8,x12,[$inp],#16 // inp[0:1] + umlal $ACC0,$IN01_2,${S3}[0] + ldp x9,x13,[$inp],#48 + umlal $ACC4,$IN01_2,${R2}[0] + umlal $ACC1,$IN01_2,${S4}[0] + umlal $ACC2,$IN01_2,${R0}[0] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3}[0] + umlal $ACC4,$IN01_0,${R4}[0] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC2,$IN01_0,${R2}[0] + and x5,x9,#0x03ffffff + umlal $ACC0,$IN01_0,${R0}[0] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN01_0,${R1}[0] + ubfx x7,x9,#26,#26 + + add $IN01_3,$IN01_3,$H3 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + umlal $ACC3,$IN01_1,${R2}[0] + extr x8,x12,x8,#52 + umlal $ACC4,$IN01_1,${R3}[0] + extr x9,x13,x9,#52 + umlal $ACC0,$IN01_1,${S4}[0] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC2,$IN01_1,${R1}[0] + fmov $IN01_0,x4 + umlal $ACC1,$IN01_1,${R0}[0] + and x8,x8,#0x03ffffff + + add $IN01_4,$IN01_4,$H4 + and x9,x9,#0x03ffffff + umlal $ACC3,$IN01_3,${R0}[0] + ubfx x10,x12,#14,#26 + umlal $ACC0,$IN01_3,${S2}[0] + ubfx x11,x13,#14,#26 + umlal $ACC4,$IN01_3,${R1}[0] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC1,$IN01_3,${S3}[0] + fmov $IN01_1,x6 + umlal $ACC2,$IN01_3,${S4}[0] + add x12,$padbit,x12,lsr#40 + + umlal $ACC3,$IN01_4,${S4}[0] + add x13,$padbit,x13,lsr#40 + umlal $ACC0,$IN01_4,${S1}[0] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC4,$IN01_4,${R0}[0] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN01_4,${S2}[0] + fmov $IN01_2,x8 + umlal $ACC2,$IN01_4,${S3}[0] + fmov $IN01_3,x10 + fmov $IN01_4,x12 + + ///////////////////////////////////////////////////////////////// + // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + // and P. Schwabe + // + // [see discussion in poly1305-armv4 module] + + ushr $T0.2d,$ACC3,#26 + xtn $H3,$ACC3 + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + bic $H3,#0xfc,lsl#24 // &=0x03ffffff + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + xtn $H4,$ACC4 + ushr $T1.2d,$ACC1,#26 + xtn $H1,$ACC1 + bic $H4,#0xfc,lsl#24 + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + shrn $T1.2s,$ACC2,#26 + xtn $H2,$ACC2 + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + bic $H1,#0xfc,lsl#24 + add $H3,$H3,$T1.2s // h2 -> h3 + bic $H2,#0xfc,lsl#24 + + shrn $T0.2s,$ACC0,#26 + xtn $H0,$ACC0 + ushr $T1.2s,$H3,#26 + bic $H3,#0xfc,lsl#24 + bic $H0,#0xfc,lsl#24 + add $H1,$H1,$T0.2s // h0 -> h1 + add $H4,$H4,$T1.2s // h3 -> h4 + + b.hi .Loop_neon + +.Lskip_loop: + dup $IN23_2,${IN23_2}[0] + add $IN01_2,$IN01_2,$H2 + + //////////////////////////////////////////////////////////////// + // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + adds $len,$len,#32 + b.ne .Long_tail + + dup $IN23_2,${IN01_2}[0] + add $IN23_0,$IN01_0,$H0 + add $IN23_3,$IN01_3,$H3 + add $IN23_1,$IN01_1,$H1 + add $IN23_4,$IN01_4,$H4 + +.Long_tail: + dup $IN23_0,${IN23_0}[0] + umull2 $ACC0,$IN23_2,${S3} + umull2 $ACC3,$IN23_2,${R1} + umull2 $ACC4,$IN23_2,${R2} + umull2 $ACC2,$IN23_2,${R0} + umull2 $ACC1,$IN23_2,${S4} + + dup $IN23_1,${IN23_1}[0] + umlal2 $ACC0,$IN23_0,${R0} + umlal2 $ACC2,$IN23_0,${R2} + umlal2 $ACC3,$IN23_0,${R3} + umlal2 $ACC4,$IN23_0,${R4} + umlal2 $ACC1,$IN23_0,${R1} + + dup $IN23_3,${IN23_3}[0] + umlal2 $ACC0,$IN23_1,${S4} + umlal2 $ACC3,$IN23_1,${R2} + umlal2 $ACC2,$IN23_1,${R1} + umlal2 $ACC4,$IN23_1,${R3} + umlal2 $ACC1,$IN23_1,${R0} + + dup $IN23_4,${IN23_4}[0] + umlal2 $ACC3,$IN23_3,${R0} + umlal2 $ACC4,$IN23_3,${R1} + umlal2 $ACC0,$IN23_3,${S2} + umlal2 $ACC1,$IN23_3,${S3} + umlal2 $ACC2,$IN23_3,${S4} + + umlal2 $ACC3,$IN23_4,${S4} + umlal2 $ACC0,$IN23_4,${S1} + umlal2 $ACC4,$IN23_4,${R0} + umlal2 $ACC1,$IN23_4,${S2} + umlal2 $ACC2,$IN23_4,${S3} + + b.eq .Lshort_tail + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4:r^3 and accumulate + + add $IN01_0,$IN01_0,$H0 + umlal $ACC3,$IN01_2,${R1} + umlal $ACC0,$IN01_2,${S3} + umlal $ACC4,$IN01_2,${R2} + umlal $ACC1,$IN01_2,${S4} + umlal $ACC2,$IN01_2,${R0} + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3} + umlal $ACC0,$IN01_0,${R0} + umlal $ACC4,$IN01_0,${R4} + umlal $ACC1,$IN01_0,${R1} + umlal $ACC2,$IN01_0,${R2} + + add $IN01_3,$IN01_3,$H3 + umlal $ACC3,$IN01_1,${R2} + umlal $ACC0,$IN01_1,${S4} + umlal $ACC4,$IN01_1,${R3} + umlal $ACC1,$IN01_1,${R0} + umlal $ACC2,$IN01_1,${R1} + + add $IN01_4,$IN01_4,$H4 + umlal $ACC3,$IN01_3,${R0} + umlal $ACC0,$IN01_3,${S2} + umlal $ACC4,$IN01_3,${R1} + umlal $ACC1,$IN01_3,${S3} + umlal $ACC2,$IN01_3,${S4} + + umlal $ACC3,$IN01_4,${S4} + umlal $ACC0,$IN01_4,${S1} + umlal $ACC4,$IN01_4,${R0} + umlal $ACC1,$IN01_4,${S2} + umlal $ACC2,$IN01_4,${S3} + +.Lshort_tail: + //////////////////////////////////////////////////////////////// + // horizontal add + + addp $ACC3,$ACC3,$ACC3 + ldp d8,d9,[sp,#16] // meet ABI requirements + addp $ACC0,$ACC0,$ACC0 + ldp d10,d11,[sp,#32] + addp $ACC4,$ACC4,$ACC4 + ldp d12,d13,[sp,#48] + addp $ACC1,$ACC1,$ACC1 + ldp d14,d15,[sp,#64] + addp $ACC2,$ACC2,$ACC2 + + //////////////////////////////////////////////////////////////// + // lazy reduction, but without narrowing + + ushr $T0.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + and $ACC4,$ACC4,$MASK.2d + ushr $T1.2d,$ACC1,#26 + and $ACC1,$ACC1,$MASK.2d + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + ushr $T1.2d,$ACC2,#26 + and $ACC2,$ACC2,$MASK.2d + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + add $ACC3,$ACC3,$T1.2d // h2 -> h3 + + ushr $T0.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + ushr $T1.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + add $ACC1,$ACC1,$T0.2d // h0 -> h1 + add $ACC4,$ACC4,$T1.2d // h3 -> h4 + + //////////////////////////////////////////////////////////////// + // write the result, can be partially reduced + + st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16 + st1 {$ACC4}[0],[$ctx] + +.Lno_data_neon: + ldr x29,[sp],#80 + ret +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr $is_base2_26,[$ctx,#24] + cbz $is_base2_26,poly1305_emit + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $h2,$h2,xzr // can be partially reduced... + + ldp $t0,$t1,[$nonce] // load nonce + + and $d0,$h2,#-4 // ... so reduce + add $d0,$d0,$h2,lsr#2 + and $h2,$h2,#3 + adds $h0,$h0,$d0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit_neon,.-poly1305_emit_neon +#endif + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +#ifdef __ILP32__ +.long OPENSSL_armcap_P-. +#else +.quad OPENSSL_armcap_P-. +#endif +#endif +.align 2 +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or + s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or + (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or + (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or + (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or + (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or + (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1)); + + s/\.[124]([sd])\[/.$1\[/; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c new file mode 100644 index 0000000000000..527ccc3b59cc8 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +struct poly1305_internal { + u32 h[5]; + u32 r[5]; + u32 s[4]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + st->r[0] = (get_unaligned_le32(&key[0])) & 0x3ffffff; + st->r[1] = (get_unaligned_le32(&key[3]) >> 2) & 0x3ffff03; + st->r[2] = (get_unaligned_le32(&key[6]) >> 4) & 0x3ffc0ff; + st->r[3] = (get_unaligned_le32(&key[9]) >> 6) & 0x3f03fff; + st->r[4] = (get_unaligned_le32(&key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + st->s[0] = st->r[1] * 5; + st->s[1] = st->r[2] * 5; + st->s[2] = st->r[3] * 5; + st->s[3] = st->r[4] * 5; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; + st->h[3] = 0; + st->h[4] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u32 hibit = padbit << 24; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + r3 = st->r[3]; + r4 = st->r[4]; + + s1 = st->s[0]; + s2 = st->s[1]; + s3 = st->s[2]; + s4 = st->s[3]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + while (len >= POLY1305_BLOCK_SIZE) { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; + st->h[3] = h3; + st->h[4] = h4; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c new file mode 100644 index 0000000000000..131f1dda1b1db --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +typedef __uint128_t u128; + +struct poly1305_internal { + u64 r[3]; + u64 h[3]; + u64 s[2]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&key[0]); + t1 = get_unaligned_le64(&key[8]); + + st->r[0] = t0 & 0xffc0fffffffULL; + st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + st->r[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + st->s[0] = st->r[1] * 20; + st->s[1] = st->r[2] * 20; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u64 hibit = ((u64)padbit) << 40; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + s1 = st->s[0]; + s2 = st->s[1]; + + while (len >= POLY1305_BLOCK_SIZE) { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c new file mode 100644 index 0000000000000..a540e9c4eee8b --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_mips(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + poly1305_blocks_mips(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + poly1305_emit_mips(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S new file mode 100644 index 0000000000000..4291c156815b3 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#else +#define MSB 3 +#define LSB 0 +#endif + +#define POLY1305_BLOCK_SIZE 16 +.text +#define H0 $t0 +#define H1 $t1 +#define H2 $t2 +#define H3 $t3 +#define H4 $t4 + +#define R0 $t5 +#define R1 $t6 +#define R2 $t7 +#define R3 $t8 + +#define O0 $s0 +#define O1 $s4 +#define O2 $v1 +#define O3 $t9 +#define O4 $s5 + +#define S1 $s1 +#define S2 $s2 +#define S3 $s3 + +#define SC $at +#define CA $v0 + +/* Input arguments */ +#define poly $a0 +#define src $a1 +#define srclen $a2 +#define hibit $a3 + +/* Location in the opaque buffer + * R[0..3], CA, H[0..4] + */ +#define PTR_POLY1305_R(n) ( 0 + (n*4)) ## ($a0) +#define PTR_POLY1305_CA (16 ) ## ($a0) +#define PTR_POLY1305_H(n) (20 + (n*4)) ## ($a0) + +#define POLY1305_BLOCK_SIZE 16 +#define POLY1305_STACK_SIZE 32 + +.set noat +.align 4 +.globl poly1305_blocks_mips +.ent poly1305_blocks_mips +poly1305_blocks_mips: + .frame $sp, POLY1305_STACK_SIZE, $ra + /* srclen &= 0xFFFFFFF0 */ + ins srclen, $zero, 0, 4 + + addiu $sp, -(POLY1305_STACK_SIZE) + + /* check srclen >= 16 bytes */ + beqz srclen, .Lpoly1305_blocks_mips_end + + /* Calculate last round based on src address pointer. + * last round src ptr (srclen) = src + (srclen & 0xFFFFFFF0) + */ + addu srclen, src + + lw R0, PTR_POLY1305_R(0) + lw R1, PTR_POLY1305_R(1) + lw R2, PTR_POLY1305_R(2) + lw R3, PTR_POLY1305_R(3) + + /* store the used save registers. */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Sx = Rx + (Rx >> 2) */ + srl S1, R1, 2 + srl S2, R2, 2 + srl S3, R3, 2 + addu S1, R1 + addu S2, R2 + addu S3, R3 + + addiu SC, $zero, 1 + +.Lpoly1305_loop: + lwl O0, 0+MSB(src) + lwl O1, 4+MSB(src) + lwl O2, 8+MSB(src) + lwl O3,12+MSB(src) + lwr O0, 0+LSB(src) + lwr O1, 4+LSB(src) + lwr O2, 8+LSB(src) + lwr O3,12+LSB(src) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh O0 + wsbh O1 + wsbh O2 + wsbh O3 + rotr O0, 16 + rotr O1, 16 + rotr O2, 16 + rotr O3, 16 +#endif + + /* h0 = (u32)(d0 = (u64)h0 + inp[0] + c 'Carry_previous cycle'); */ + addu H0, CA + sltu CA, H0, CA + addu O0, H0 + sltu H0, O0, H0 + addu CA, H0 + + /* h1 = (u32)(d1 = (u64)h1 + (d0 >> 32) + inp[4]); */ + addu H1, CA + sltu CA, H1, CA + addu O1, H1 + sltu H1, O1, H1 + addu CA, H1 + + /* h2 = (u32)(d2 = (u64)h2 + (d1 >> 32) + inp[8]); */ + addu H2, CA + sltu CA, H2, CA + addu O2, H2 + sltu H2, O2, H2 + addu CA, H2 + + /* h3 = (u32)(d3 = (u64)h3 + (d2 >> 32) + inp[12]); */ + addu H3, CA + sltu CA, H3, CA + addu O3, H3 + sltu H3, O3, H3 + addu CA, H3 + + /* h4 += (u32)(d3 >> 32) + padbit; */ + addu H4, hibit + addu O4, H4, CA + + /* D0 */ + multu O0, R0 + maddu O1, S3 + maddu O2, S2 + maddu O3, S1 + mfhi CA + mflo H0 + + /* D1 */ + multu O0, R1 + maddu O1, R0 + maddu O2, S3 + maddu O3, S2 + maddu O4, S1 + maddu CA, SC + mfhi CA + mflo H1 + + /* D2 */ + multu O0, R2 + maddu O1, R1 + maddu O2, R0 + maddu O3, S3 + maddu O4, S2 + maddu CA, SC + mfhi CA + mflo H2 + + /* D4 */ + mul H4, O4, R0 + + /* D3 */ + multu O0, R3 + maddu O1, R2 + maddu O2, R1 + maddu O3, R0 + maddu O4, S3 + maddu CA, SC + mfhi CA + mflo H3 + + addiu src, POLY1305_BLOCK_SIZE + + /* h4 += (u32)(d3 >> 32); */ + addu O4, H4, CA + /* h4 &= 3 */ + andi H4, O4, 3 + /* c = (h4 >> 2) + (h4 & ~3U); */ + srl CA, O4, 2 + ins O4, $zero, 0, 2 + + addu CA, O4 + + /* able to do a 16 byte block. */ + bne src, srclen, .Lpoly1305_loop + + /* restore the used save registers. */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + + /* store Hx and Carry */ + sw CA, PTR_POLY1305_CA + sw H0, PTR_POLY1305_H(0) + sw H1, PTR_POLY1305_H(1) + sw H2, PTR_POLY1305_H(2) + sw H3, PTR_POLY1305_H(3) + sw H4, PTR_POLY1305_H(4) + +.Lpoly1305_blocks_mips_end: + addiu $sp, POLY1305_STACK_SIZE + + /* Jump Back */ + jr $ra +.end poly1305_blocks_mips +.set at + +/* Input arguments CTX=$a0, MAC=$a1, NONCE=$a2 */ +#define MAC $a1 +#define NONCE $a2 + +#define G0 $t5 +#define G1 $t6 +#define G2 $t7 +#define G3 $t8 +#define G4 $t9 + +.set noat +.align 4 +.globl poly1305_emit_mips +.ent poly1305_emit_mips +poly1305_emit_mips: + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Add left over carry */ + addu H0, CA + sltu CA, H0, CA + addu H1, CA + sltu CA, H1, CA + addu H2, CA + sltu CA, H2, CA + addu H3, CA + sltu CA, H3, CA + addu H4, CA + + /* compare to modulus by computing h + -p */ + addiu G0, H0, 5 + sltu CA, G0, H0 + addu G1, H1, CA + sltu CA, G1, H1 + addu G2, H2, CA + sltu CA, G2, H2 + addu G3, H3, CA + sltu CA, G3, H3 + addu G4, H4, CA + + srl SC, G4, 2 + + /* if there was carry into 131st bit, h3:h0 = g3:g0 */ + movn H0, G0, SC + movn H1, G1, SC + movn H2, G2, SC + movn H3, G3, SC + + lwl G0, 0+MSB(NONCE) + lwl G1, 4+MSB(NONCE) + lwl G2, 8+MSB(NONCE) + lwl G3,12+MSB(NONCE) + lwr G0, 0+LSB(NONCE) + lwr G1, 4+LSB(NONCE) + lwr G2, 8+LSB(NONCE) + lwr G3,12+LSB(NONCE) + + /* mac = (h + nonce) % (2^128) */ + addu H0, G0 + sltu CA, H0, G0 + + /* H1 */ + addu H1, CA + sltu CA, H1, CA + addu H1, G1 + sltu G1, H1, G1 + addu CA, G1 + + /* H2 */ + addu H2, CA + sltu CA, H2, CA + addu H2, G2 + sltu G2, H2, G2 + addu CA, G2 + + /* H3 */ + addu H3, CA + addu H3, G3 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh H0 + wsbh H1 + wsbh H2 + wsbh H3 + rotr H0, 16 + rotr H1, 16 + rotr H2, 16 + rotr H3, 16 +#endif + + /* store MAC */ + swl H0, 0+MSB(MAC) + swl H1, 4+MSB(MAC) + swl H2, 8+MSB(MAC) + swl H3,12+MSB(MAC) + swr H0, 0+LSB(MAC) + swr H1, 4+LSB(MAC) + swr H2, 8+LSB(MAC) + swr H3,12+LSB(MAC) + + jr $ra +.end poly1305_emit_mips + +#define PR0 $t0 +#define PR1 $t1 +#define PR2 $t2 +#define PR3 $t3 +#define PT0 $t4 + +/* Input arguments CTX=$a0, KEY=$a1 */ + +.align 4 +.globl poly1305_init_mips +.ent poly1305_init_mips +poly1305_init_mips: + lwl PR0, 0+MSB($a1) + lwl PR1, 4+MSB($a1) + lwl PR2, 8+MSB($a1) + lwl PR3,12+MSB($a1) + lwr PR0, 0+LSB($a1) + lwr PR1, 4+LSB($a1) + lwr PR2, 8+LSB($a1) + lwr PR3,12+LSB($a1) + + /* store Hx and Carry */ + sw $zero, PTR_POLY1305_CA + sw $zero, PTR_POLY1305_H(0) + sw $zero, PTR_POLY1305_H(1) + sw $zero, PTR_POLY1305_H(2) + sw $zero, PTR_POLY1305_H(3) + sw $zero, PTR_POLY1305_H(4) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh PR0 + wsbh PR1 + wsbh PR2 + wsbh PR3 + rotr PR0, 16 + rotr PR1, 16 + rotr PR2, 16 + rotr PR3, 16 +#endif + + lui PT0, 0x0FFF + ori PT0, 0xFFFC + + /* AND 0x0fffffff; */ + ext PR0, PR0, 0, (32-4) + + /* AND 0x0ffffffc; */ + and PR1, PT0 + and PR2, PT0 + and PR3, PT0 + + /* store Rx */ + sw PR0, PTR_POLY1305_R(0) + sw PR1, PTR_POLY1305_R(1) + sw PR2, PTR_POLY1305_R(2) + sw PR3, PTR_POLY1305_R(3) + + /* Jump Back */ + jr $ra +.end poly1305_init_mips diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl new file mode 100644 index 0000000000000..4adb9d4007ce6 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl @@ -0,0 +1,466 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# Poly1305 hash for MIPS64. +# +# May 2016 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc +# R1x000 5.64/+120% (big-endian) +# Octeon II 3.80/+280% (little-endian) + +###################################################################### +# There is a number of MIPS ABI in use, O32 and N32/64 are most +# widely used. Then there is a new contender: NUBI. It appears that if +# one picks the latter, it's possible to arrange code in ABI neutral +# manner. Therefore let's stick to NUBI register layout: +# +($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23)); +($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31)); +# +# The return value is placed in $a0. Following coding rules facilitate +# interoperability: +# +# - never ever touch $tp, "thread pointer", former $gp [o32 can be +# excluded from the rule, because it's specified volatile]; +# - copy return value to $t0, former $v0 [or to $a0 if you're adapting +# old code]; +# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary; +# +# For reference here is register layout for N32/64 MIPS ABIs: +# +# ($zero,$at,$v0,$v1)=map("\$$_",(0..3)); +# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25)); +# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); +# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); +# +# +# +###################################################################### + +$flavour = shift || "64"; # supported flavours are o32,n32,64,nubi32,nubi64 + +die "MIPS64 only" unless ($flavour =~ /64|n32/i); + +$v0 = ($flavour =~ /nubi/i) ? $a0 : $t0; +$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0x0003f000" : "0x00030000"; + +($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3); +($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$at,$t0,$t1); + +$code.=<<___; +#if (defined(_MIPS_ARCH_MIPS64R3) || defined(_MIPS_ARCH_MIPS64R5) || \\ + defined(_MIPS_ARCH_MIPS64R6)) \\ + && !defined(_MIPS_ARCH_MIPS64R2) +# define _MIPS_ARCH_MIPS64R2 +#endif + +#if defined(_MIPS_ARCH_MIPS64R6) +# define dmultu(rs,rt) +# define mflo(rd,rs,rt) dmulu rd,rs,rt +# define mfhi(rd,rs,rt) dmuhu rd,rs,rt +#else +# define dmultu(rs,rt) dmultu rs,rt +# define mflo(rd,rs,rt) mflo rd +# define mfhi(rd,rs,rt) mfhi rd +#endif + +#ifdef __KERNEL__ +# define poly1305_init poly1305_init_mips +# define poly1305_blocks poly1305_blocks_mips +# define poly1305_emit poly1305_emit_mips +#endif + +#if defined(__MIPSEB__) && !defined(MIPSEB) +# define MIPSEB +#endif + +#ifdef MIPSEB +# define MSB 0 +# define LSB 7 +#else +# define MSB 7 +# define LSB 0 +#endif + +.text +.set noat +.set noreorder + +.align 5 +.globl poly1305_init +.ent poly1305_init +poly1305_init: + .frame $sp,0,$ra + .set reorder + + sd $zero,0($ctx) + sd $zero,8($ctx) + sd $zero,16($ctx) + + beqz $inp,.Lno_key + +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + li $tmp0,1 + dsll $tmp0,32 + daddiu $tmp0,-63 + dsll $tmp0,28 + daddiu $tmp0,-1 # 0ffffffc0fffffff + + and $in0,$tmp0 + daddiu $tmp0,-3 # 0ffffffc0ffffffc + and $in1,$tmp0 + + sd $in0,24($ctx) + dsrl $tmp0,$in1,2 + sd $in1,32($ctx) + daddu $tmp0,$in1 # s1 = r1 + (r1 >> 2) + sd $tmp0,40($ctx) + +.Lno_key: + li $v0,0 # return 0 + jr $ra +.end poly1305_init +___ +{ +my ($h0,$h1,$h2,$r0,$r1,$s1,$d0,$d1,$d2) = + ($s0,$s1,$s2,$s3,$s4,$s5,$in0,$in1,$t2); + +$code.=<<___; +.align 5 +.globl poly1305_blocks +.ent poly1305_blocks +poly1305_blocks: + .set noreorder + dsrl $len,4 # number of complete blocks + bnez $len,poly1305_blocks_internal + nop + jr $ra + nop +.end poly1305_blocks + +.align 5 +.ent poly1305_blocks_internal +poly1305_blocks_internal: + .frame $sp,6*8,$ra + .mask $SAVED_REGS_MASK,-8 + .set noreorder + dsubu $sp,6*8 + sd $s5,40($sp) + sd $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi prologue + sd $s3,24($sp) + sd $s2,16($sp) + sd $s1,8($sp) + sd $s0,0($sp) +___ +$code.=<<___; + .set reorder + + ld $h0,0($ctx) # load hash value + ld $h1,8($ctx) + ld $h2,16($ctx) + + ld $r0,24($ctx) # load key + ld $r1,32($ctx) + ld $s1,40($ctx) + +.Loop: +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) # load input + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) # load input + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif + daddiu $len,-1 + daddiu $inp,16 +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + daddu $h0,$in0 # accumulate input + daddu $h1,$in1 + sltu $tmp0,$h0,$in0 + sltu $tmp1,$h1,$in1 + daddu $h1,$tmp0 + + dmultu ($r0,$h0) # h0*r0 + daddu $h2,$padbit + sltu $tmp0,$h1,$tmp0 + mflo ($d0,$r0,$h0) + mfhi ($d1,$r0,$h0) + + dmultu ($s1,$h1) # h1*5*r1 + daddu $tmp0,$tmp1 + daddu $h2,$tmp0 + mflo ($tmp0,$s1,$h1) + mfhi ($tmp1,$s1,$h1) + + dmultu ($r1,$h0) # h0*r1 + daddu $d0,$tmp0 + daddu $d1,$tmp1 + mflo ($tmp2,$r1,$h0) + mfhi ($d2,$r1,$h0) + sltu $tmp0,$d0,$tmp0 + daddu $d1,$tmp0 + + dmultu ($r0,$h1) # h1*r0 + daddu $d1,$tmp2 + sltu $tmp2,$d1,$tmp2 + mflo ($tmp0,$r0,$h1) + mfhi ($tmp1,$r0,$h1) + daddu $d2,$tmp2 + + dmultu ($s1,$h2) # h2*5*r1 + daddu $d1,$tmp0 + daddu $d2,$tmp1 + mflo ($tmp2,$s1,$h2) + + dmultu ($r0,$h2) # h2*r0 + sltu $tmp0,$d1,$tmp0 + daddu $d2,$tmp0 + mflo ($tmp3,$r0,$h2) + + daddu $d1,$tmp2 + daddu $d2,$tmp3 + sltu $tmp2,$d1,$tmp2 + daddu $d2,$tmp2 + + li $tmp0,-4 # final reduction + and $tmp0,$d2 + dsrl $tmp1,$d2,2 + andi $h2,$d2,3 + daddu $tmp0,$tmp1 + daddu $h0,$d0,$tmp0 + sltu $tmp0,$h0,$tmp0 + daddu $h1,$d1,$tmp0 + sltu $tmp0,$h1,$tmp0 + daddu $h2,$h2,$tmp0 + + bnez $len,.Loop + + sd $h0,0($ctx) # store hash value + sd $h1,8($ctx) + sd $h2,16($ctx) + + .set noreorder + ld $s5,40($sp) # epilogue + ld $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi epilogue + ld $s3,24($sp) + ld $s2,16($sp) + ld $s1,8($sp) + ld $s0,0($sp) +___ +$code.=<<___; + jr $ra + daddu $sp,6*8 +.end poly1305_blocks_internal +___ +} +{ +my ($ctx,$mac,$nonce) = ($a0,$a1,$a2); + +$code.=<<___; +.align 5 +.globl poly1305_emit +.ent poly1305_emit +poly1305_emit: + .frame $sp,0,$ra + .set reorder + + ld $tmp0,0($ctx) + ld $tmp1,8($ctx) + ld $tmp2,16($ctx) + + daddiu $in0,$tmp0,5 # compare to modulus + sltiu $tmp3,$in0,5 + daddu $in1,$tmp1,$tmp3 + sltu $tmp3,$in1,$tmp3 + daddu $tmp2,$tmp2,$tmp3 + + dsrl $tmp2,2 # see if it carried/borrowed + dsubu $tmp2,$zero,$tmp2 + nor $tmp3,$zero,$tmp2 + + and $in0,$tmp2 + and $tmp0,$tmp3 + and $in1,$tmp2 + and $tmp1,$tmp3 + or $in0,$tmp0 + or $in1,$tmp1 + + lwu $tmp0,0($nonce) # load nonce + lwu $tmp1,4($nonce) + lwu $tmp2,8($nonce) + lwu $tmp3,12($nonce) + dsll $tmp1,32 + dsll $tmp3,32 + or $tmp0,$tmp1 + or $tmp2,$tmp3 + + daddu $in0,$tmp0 # accumulate nonce + daddu $in1,$tmp2 + sltu $tmp0,$in0,$tmp0 + daddu $in1,$tmp0 + + dsrl $tmp0,$in0,8 # write mac value + dsrl $tmp1,$in0,16 + dsrl $tmp2,$in0,24 + sb $in0,0($mac) + dsrl $tmp3,$in0,32 + sb $tmp0,1($mac) + dsrl $tmp0,$in0,40 + sb $tmp1,2($mac) + dsrl $tmp1,$in0,48 + sb $tmp2,3($mac) + dsrl $tmp2,$in0,56 + sb $tmp3,4($mac) + dsrl $tmp3,$in1,8 + sb $tmp0,5($mac) + dsrl $tmp0,$in1,16 + sb $tmp1,6($mac) + dsrl $tmp1,$in1,24 + sb $tmp2,7($mac) + + sb $in1,8($mac) + dsrl $tmp2,$in1,32 + sb $tmp3,9($mac) + dsrl $tmp3,$in1,40 + sb $tmp0,10($mac) + dsrl $tmp0,$in1,48 + sb $tmp1,11($mac) + dsrl $tmp1,$in1,56 + sb $tmp2,12($mac) + sb $tmp3,13($mac) + sb $tmp0,14($mac) + sb $tmp1,15($mac) + + jr $ra +.end poly1305_emit +.rdata +.align 2 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +$output=pop and open STDOUT,">$output"; +print $code; +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c new file mode 100644 index 0000000000000..ce48a42f7654c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); + +static bool poly1305_use_avx __ro_after_init; +static bool poly1305_use_avx2 __ro_after_init; +static bool poly1305_use_avx512 __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { + &poly1305_use_avx, &poly1305_use_avx2, &poly1305_use_avx512 }; + +static void __init poly1305_fpu_init(void) +{ + poly1305_use_avx = + boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); + poly1305_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + poly1305_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; +#endif +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_x86_64(ctx, key); + return true; +} + +struct poly1305_arch_internal { + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; + u64 r[2]; + u64 pad; + struct { u32 r2, r1, r4, r3; } rn[9]; +}; + +/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + !state->is_base2_26 || !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_x86_64(ctx, mac, nonce); + } else + poly1305_emit_avx(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl new file mode 100644 index 0000000000000..c9a8e0037bdbd --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl @@ -0,0 +1,4266 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for x86_64. +# +# March 2015 +# +# Initial release. +# +# December 2016 +# +# Add AVX512F+VL+BW code path. +# +# November 2017 +# +# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be +# executed even on Knights Landing. Trigger for modification was +# observation that AVX512 code paths can negatively affect overall +# Skylake-X system performance. Since we are likely to suppress +# AVX512F capability flag [at least on Skylake-X], conversion serves +# as kind of "investment protection". Note that next *lake processor, +# Cannonlake, has AVX512IFMA code path to execute... +# +# Numbers are cycles per processed byte with poly1305_blocks alone, +# measured with rdtsc at fixed clock frequency. +# +# IALU/gcc-4.8(*) AVX(**) AVX2 AVX-512 +# P4 4.46/+120% - +# Core 2 2.41/+90% - +# Westmere 1.88/+120% - +# Sandy Bridge 1.39/+140% 1.10 +# Haswell 1.14/+175% 1.11 0.65 +# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35] +# Silvermont 2.83/+95% - +# Knights L 3.60/? 1.65 1.10 0.41(***) +# Goldmont 1.70/+180% - +# VIA Nano 1.82/+150% - +# Sledgehammer 1.38/+160% - +# Bulldozer 2.30/+130% 0.97 +# Ryzen 1.15/+200% 1.08 1.18 +# +# (*) improvement coefficients relative to clang are more modest and +# are ~50% on most processors, in both cases we are comparing to +# __int128 code; +# (**) SSE2 implementation was attempted, but among non-AVX processors +# it was faster than integer-only code only on older Intel P4 and +# Core processors, 50-30%, less newer processor is, but slower on +# contemporary ones, for example almost 2x slower on Atom, and as +# former are naturally disappearing, SSE2 is deemed unnecessary; +# (***) strangely enough performance seems to vary from core to core, +# listed result is best case; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +$code.=<<___ if $kernel; +#include +___ + +if ($avx) { +$code.=<<___ if $kernel; +.section .rodata +___ +$code.=<<___; +.align 64 +.Lconst: +.Lmask24: +.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 +.L129: +.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 +.Lmask26: +.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 +.Lpermd_avx2: +.long 2,2,2,3,2,0,2,1 +.Lpermd_avx512: +.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 + +.L2_44_inp_permd: +.long 0,1,1,2,2,3,7,7 +.L2_44_inp_shift: +.quad 0,12,24,64 +.L2_44_mask: +.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff +.L2_44_shift_rgt: +.quad 44,44,42,64 +.L2_44_shift_lft: +.quad 8,8,10,64 + +.align 64 +.Lx_mask44: +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.Lx_mask42: +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +___ +} +$code.=<<___ if (!$kernel); +.asciz "Poly1305 for x86_64, CRYPTOGAMS by " +.align 16 +___ + +my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); +my ($mac,$nonce)=($inp,$len); # *_emit arguments +my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); +my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); + +sub poly1305_iteration { +# input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 +# output: $h0-$h2 *= $r0-$r1 +$code.=<<___; + mulq $h0 # h0*r1 + mov %rax,$d2 + mov $r0,%rax + mov %rdx,$d3 + + mulq $h0 # h0*r0 + mov %rax,$h0 # future $h0 + mov $r0,%rax + mov %rdx,$d1 + + mulq $h1 # h1*r0 + add %rax,$d2 + mov $s1,%rax + adc %rdx,$d3 + + mulq $h1 # h1*s1 + mov $h2,$h1 # borrow $h1 + add %rax,$h0 + adc %rdx,$d1 + + imulq $s1,$h1 # h2*s1 + add $h1,$d2 + mov $d1,$h1 + adc \$0,$d3 + + imulq $r0,$h2 # h2*r0 + add $d2,$h1 + mov \$-4,%rax # mask value + adc $h2,$d3 + + and $d3,%rax # last reduction step + mov $d3,$h2 + shr \$2,$d3 + and \$3,$h2 + add $d3,%rax + add %rax,$h0 + adc \$0,$h1 + adc \$0,$h2 +___ +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^64 +# unsigned __int64 r[2]; # key value base 2^64 + +$code.=<<___; +.text +___ +$code.=<<___ if (!$kernel); +.extern OPENSSL_ia32cap_P + +.globl poly1305_init_x86_64 +.hidden poly1305_init_x86_64 +.globl poly1305_blocks_x86_64 +.hidden poly1305_blocks_x86_64 +.globl poly1305_emit_x86_64 +.hidden poly1305_emit_x86_64 +___ +&declare_function("poly1305_init_x86_64", 32, 3); +$code.=<<___; + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + + cmp \$0,$inp + je .Lno_key +___ +$code.=<<___ if (!$kernel); + lea poly1305_blocks_x86_64(%rip),%r10 + lea poly1305_emit_x86_64(%rip),%r11 +___ +$code.=<<___ if (!$kernel && $avx); + mov OPENSSL_ia32cap_P+4(%rip),%r9 + lea poly1305_blocks_avx(%rip),%rax + lea poly1305_emit_avx(%rip),%rcx + bt \$`60-32`,%r9 # AVX? + cmovc %rax,%r10 + cmovc %rcx,%r11 +___ +$code.=<<___ if (!$kernel && $avx>1); + lea poly1305_blocks_avx2(%rip),%rax + bt \$`5+32`,%r9 # AVX2? + cmovc %rax,%r10 +___ +$code.=<<___ if (!$kernel && $avx>3); + mov \$`(1<<31|1<<21|1<<16)`,%rax + shr \$32,%r9 + and %rax,%r9 + cmp %rax,%r9 + je .Linit_base2_44 +___ +$code.=<<___; + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + and 8($inp),%rcx + mov %rax,24($ctx) + mov %rcx,32($ctx) +___ +$code.=<<___ if (!$kernel && $flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if (!$kernel && $flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax +.Lno_key: + ret +___ +&end_function("poly1305_init_x86_64"); + +&declare_function("poly1305_blocks_x86_64", 32, 4); +$code.=<<___; +.cfi_startproc +.Lblocks: + shr \$4,$len + jz .Lno_data # too short + + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + push $ctx +.cfi_push $ctx +.Lblocks_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2 + + mov $s1,$r1 + shr \$2,$s1 + mov $r1,%rax + add $r1,$s1 # s1 = r1 + (r1 >> 2) + jmp .Loop + +.align 32 +.Loop: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 +___ + + &poly1305_iteration(); + +$code.=<<___; + mov $r1,%rax + dec %r15 # len-=16 + jnz .Loop + + mov 0(%rsp),$ctx +.cfi_restore $ctx + + mov $h0,0($ctx) # store hash value + mov $h1,8($ctx) + mov $h2,16($ctx) + + mov 8(%rsp),%r15 +.cfi_restore %r15 + mov 16(%rsp),%r14 +.cfi_restore %r14 + mov 24(%rsp),%r13 +.cfi_restore %r13 + mov 32(%rsp),%r12 +.cfi_restore %r12 + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lno_data: +.Lblocks_epilogue: + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_x86_64"); + +&declare_function("poly1305_emit_x86_64", 32, 3); +$code.=<<___; +.Lemit: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_x86_64"); +if ($avx) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX\n"; +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int32 h[5]; # current hash value base 2^26 +# unsigned __int32 is_base2_26; +# unsigned __int64 r[2]; # key value base 2^64 +# unsigned __int64 pad; +# struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9]; +# +# where r^n are base 2^26 digits of degrees of multiplier key. There are +# 5 digits, but last four are interleaved with multiples of 5, totalling +# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. + +my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) = + map("%xmm$_",(0..15)); + +$code.=<<___; +.type __poly1305_block,\@abi-omnipotent +.align 32 +__poly1305_block: + push $ctx +___ + &poly1305_iteration(); +$code.=<<___; + pop $ctx + ret +.size __poly1305_block,.-__poly1305_block + +.type __poly1305_init_avx,\@abi-omnipotent +.align 32 +__poly1305_init_avx: + push %rbp + mov %rsp,%rbp + mov $r0,$h0 + mov $r1,$h1 + xor $h2,$h2 + + lea 48+64($ctx),$ctx # size optimization + + mov $r1,%rax + call __poly1305_block # r^2 + + mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26 + mov \$0x3ffffff,%edx + mov $h0,$d1 + and $h0#d,%eax + mov $r0,$d2 + and $r0#d,%edx + mov %eax,`16*0+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*0+4-64`($ctx) + shr \$26,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*1+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*1+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*2+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*2+4-64`($ctx) + shr \$26,$d2 + + mov $h1,%rax + mov $r1,%rdx + shl \$12,%rax + shl \$12,%rdx + or $d1,%rax + or $d2,%rdx + and \$0x3ffffff,%eax + and \$0x3ffffff,%edx + mov %eax,`16*3+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*3+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*4+0-64`($ctx) + mov $h1,$d1 + mov %edx,`16*4+4-64`($ctx) + mov $r1,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + shr \$14,$d1 + shr \$14,$d2 + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*5+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*5+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*6+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*6+4-64`($ctx) + shr \$26,$d2 + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+0-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d2#d,`16*7+4-64`($ctx) + lea ($d2,$d2,4),$d2 # *5 + mov $d1#d,`16*8+0-64`($ctx) + mov $d2#d,`16*8+4-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^3 + + mov \$0x3ffffff,%eax # save r^3 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+12-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+12-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+12-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+12-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+12-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+12-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+12-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^4 + + mov \$0x3ffffff,%eax # save r^4 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+8-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+8-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+8-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+8-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+8-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+8-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+8-64`($ctx) + + lea -48-64($ctx),$ctx # size [de-]optimization + pop %rbp + ret +.size __poly1305_init_avx,.-__poly1305_init_avx +___ + +&declare_function("poly1305_blocks_avx", 32, 4); +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx: + and \$-16,$len + jz .Lno_data_avx + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx + + test \$31,$len + jz .Leven_avx + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx_body: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + + call __poly1305_block + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + sub \$16,%r15 + jz .Lstore_base2_26_avx + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + jmp .Lproceed_avx + +.align 32 +.Lstore_base2_64_avx: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx + +.align 16 +.Lstore_base2_26_avx: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx: +.Lblocks_avx_epilogue: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$31,$len + jz .Linit_avx + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + +.Linit_avx: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx: + mov %r15,$len + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx_epilogue: + jmp .Ldo_avx +.cfi_endproc + +.align 32 +.Leven_avx: +.cfi_startproc + vmovd 4*0($ctx),$H0 # load hash value + vmovd 4*1($ctx),$H1 + vmovd 4*2($ctx),$H2 + vmovd 4*3($ctx),$H3 + vmovd 4*4($ctx),$H4 + +.Ldo_avx: +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + and \$-32,%rsp + sub \$-8,%rsp + lea -0x58(%rsp),%r11 + sub \$0x178,%rsp + +___ +$code.=<<___ if ($win64); + lea -0xf8(%rsp),%r11 + sub \$0x218,%rsp + vmovdqa %xmm6,0x50(%r11) + vmovdqa %xmm7,0x60(%r11) + vmovdqa %xmm8,0x70(%r11) + vmovdqa %xmm9,0x80(%r11) + vmovdqa %xmm10,0x90(%r11) + vmovdqa %xmm11,0xa0(%r11) + vmovdqa %xmm12,0xb0(%r11) + vmovdqa %xmm13,0xc0(%r11) + vmovdqa %xmm14,0xd0(%r11) + vmovdqa %xmm15,0xe0(%r11) +.Ldo_avx_body: +___ +$code.=<<___; + sub \$64,$len + lea -32($inp),%rax + cmovc %rax,$inp + + vmovdqu `16*3`($ctx),$D4 # preload r0^2 + lea `16*3+64`($ctx),$ctx # size optimization + lea .Lconst(%rip),%rcx + + ################################################################ + # load input + vmovdqu 16*2($inp),$T0 + vmovdqu 16*3($inp),$T1 + vmovdqa 64(%rcx),$MASK # .Lmask26 + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + vpsrlq \$40,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + jbe .Lskip_loop_avx + + # expand and copy pre-calculated table to stack + vmovdqu `16*1-64`($ctx),$D1 + vmovdqu `16*2-64`($ctx),$D2 + vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434 + vpshufd \$0x44,$D4,$D0 # xx12 -> 1212 + vmovdqa $D3,-0x90(%r11) + vmovdqa $D0,0x00(%rsp) + vpshufd \$0xEE,$D1,$D4 + vmovdqu `16*3-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x80(%r11) + vmovdqa $D1,0x10(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqu `16*4-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x70(%r11) + vmovdqa $D2,0x20(%rsp) + vpshufd \$0xEE,$D0,$D4 + vmovdqu `16*5-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D4,-0x60(%r11) + vmovdqa $D0,0x30(%rsp) + vpshufd \$0xEE,$D1,$D3 + vmovdqu `16*6-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D3,-0x50(%r11) + vmovdqa $D1,0x40(%rsp) + vpshufd \$0xEE,$D2,$D4 + vmovdqu `16*7-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D4,-0x40(%r11) + vmovdqa $D2,0x50(%rsp) + vpshufd \$0xEE,$D0,$D3 + vmovdqu `16*8-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D3,-0x30(%r11) + vmovdqa $D0,0x60(%rsp) + vpshufd \$0xEE,$D1,$D4 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x20(%r11) + vmovdqa $D1,0x70(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x10(%r11) + vmovdqa $D2,0x80(%rsp) + + jmp .Loop_avx + +.align 32 +.Loop_avx: + ################################################################ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + # \___________________/ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + # \___________________/ \____________________/ + # + # Note that we start with inp[2:3]*r^2. This is because it + # doesn't depend on reduction in previous iteration. + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # though note that $Tx and $Hx are "reversed" in this section, + # and $D4 is preloaded with r0^2... + + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vmovdqa $H2,0x20(%r11) # offload hash + vpmuludq $T2,$D4,$D2 # d3 = h2*r0 + vmovdqa 0x10(%rsp),$H2 # r1^2 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vmovdqa $H0,0x00(%r11) # + vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1 + vmovdqa $H1,0x10(%r11) # + vpmuludq $T3,$H2,$H1 # h3*r1 + vpaddq $H0,$D0,$D0 # d0 += h4*s1 + vpaddq $H1,$D4,$D4 # d4 += h3*r1 + vmovdqa $H3,0x30(%r11) # + vpmuludq $T2,$H2,$H0 # h2*r1 + vpmuludq $T1,$H2,$H1 # h1*r1 + vpaddq $H0,$D3,$D3 # d3 += h2*r1 + vmovdqa 0x30(%rsp),$H3 # r2^2 + vpaddq $H1,$D2,$D2 # d2 += h1*r1 + vmovdqa $H4,0x40(%r11) # + vpmuludq $T0,$H2,$H2 # h0*r1 + vpmuludq $T2,$H3,$H0 # h2*r2 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + + vmovdqa 0x40(%rsp),$H4 # s2^2 + vpaddq $H0,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H3,$H1 # h1*r2 + vpmuludq $T0,$H3,$H3 # h0*r2 + vpaddq $H1,$D3,$D3 # d3 += h1*r2 + vmovdqa 0x50(%rsp),$H2 # r3^2 + vpaddq $H3,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H4,$H0 # h4*s2 + vpmuludq $T3,$H4,$H4 # h3*s2 + vpaddq $H0,$D1,$D1 # d1 += h4*s2 + vmovdqa 0x60(%rsp),$H3 # s3^2 + vpaddq $H4,$D0,$D0 # d0 += h3*s2 + + vmovdqa 0x80(%rsp),$H4 # s4^2 + vpmuludq $T1,$H2,$H1 # h1*r3 + vpmuludq $T0,$H2,$H2 # h0*r3 + vpaddq $H1,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $T4,$H3,$H0 # h4*s3 + vpmuludq $T3,$H3,$H1 # h3*s3 + vpaddq $H0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*0($inp),$H0 # load input + vpaddq $H1,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H3,$H3 # h2*s3 + vpmuludq $T2,$H4,$T2 # h2*s4 + vpaddq $H3,$D0,$D0 # d0 += h2*s3 + + vmovdqu 16*1($inp),$H1 # + vpaddq $T2,$D1,$D1 # d1 += h2*s4 + vpmuludq $T3,$H4,$T3 # h3*s4 + vpmuludq $T4,$H4,$T4 # h4*s4 + vpsrldq \$6,$H0,$H2 # splat input + vpaddq $T3,$D2,$D2 # d2 += h3*s4 + vpaddq $T4,$D3,$D3 # d3 += h4*s4 + vpsrldq \$6,$H1,$H3 # + vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4 + vpmuludq $T1,$H4,$T0 # h1*s4 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpaddq $T4,$D4,$D4 # d4 += h0*r4 + vmovdqa -0x90(%r11),$T4 # r0^4 + vpaddq $T0,$D0,$D0 # d0 += h1*s4 + + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + #vpsrlq \$40,$H4,$H4 # 4 + vpsrldq \$`40/8`,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpand 0(%rcx),$H4,$H4 # .Lmask24 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpaddq 0x00(%r11),$H0,$H0 # add hash value + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + lea 16*2($inp),%rax + lea 16*4($inp),$inp + sub \$64,$len + cmovc %rax,$inp + + ################################################################ + # Now we accumulate (inp[0:1]+hash)*r^4 + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vmovdqa -0x80(%r11),$T2 # r1^4 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T0,$D2,$D2 + vpaddq $T1,$D3,$D3 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpmuludq -0x70(%r11),$H4,$T0 # h4*s1 + vpaddq $T4,$D4,$D4 + + vpaddq $T0,$D0,$D0 # d0 += h4*s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vmovdqa -0x60(%r11),$T3 # r2^4 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpmuludq $H1,$T2,$T1 # h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T1,$D2,$D2 # d2 += h1*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + + vmovdqa -0x50(%r11),$T4 # s2^4 + vpmuludq $H2,$T3,$T0 # h2*r2 + vpmuludq $H1,$T3,$T1 # h1*r2 + vpaddq $T0,$D4,$D4 # d4 += h2*r2 + vpaddq $T1,$D3,$D3 # d3 += h1*r2 + vmovdqa -0x40(%r11),$T2 # r3^4 + vpmuludq $H0,$T3,$T3 # h0*r2 + vpmuludq $H4,$T4,$T0 # h4*s2 + vpaddq $T3,$D2,$D2 # d2 += h0*r2 + vpaddq $T0,$D1,$D1 # d1 += h4*s2 + vmovdqa -0x30(%r11),$T3 # s3^4 + vpmuludq $H3,$T4,$T4 # h3*s2 + vpmuludq $H1,$T2,$T1 # h1*r3 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + + vmovdqa -0x10(%r11),$T4 # s4^4 + vpaddq $T1,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T2,$T2 # h0*r3 + vpmuludq $H4,$T3,$T0 # h4*s3 + vpaddq $T2,$D3,$D3 # d3 += h0*r3 + vpaddq $T0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*2($inp),$T0 # load input + vpmuludq $H3,$T3,$T2 # h3*s3 + vpmuludq $H2,$T3,$T3 # h2*s3 + vpaddq $T2,$D1,$D1 # d1 += h3*s3 + vmovdqu 16*3($inp),$T1 # + vpaddq $T3,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H2,$T4,$H2 # h2*s4 + vpmuludq $H3,$T4,$H3 # h3*s4 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $H2,$D1,$D1 # d1 += h2*s4 + vpmuludq $H4,$T4,$H4 # h4*s4 + vpsrldq \$6,$T1,$T3 # + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4 + vpmuludq -0x20(%r11),$H0,$H4 # h0*r4 + vpmuludq $H1,$T4,$H0 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + #vpsrlq \$40,$T4,$T4 # 4 + vpsrldq \$`40/8`,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpand 0(%rcx),$T4,$T4 # .Lmask24 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + ################################################################ + # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + # and P. Schwabe + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D0 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D0,$H0,$H0 + vpsllq \$2,$D0,$D0 + vpaddq $D0,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + ja .Loop_avx + +.Lskip_loop_avx: + ################################################################ + # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2 + add \$32,$len + jnz .Long_tail_avx + + vpaddq $H2,$T2,$T2 + vpaddq $H0,$T0,$T0 + vpaddq $H1,$T1,$T1 + vpaddq $H3,$T3,$T3 + vpaddq $H4,$T4,$T4 + +.Long_tail_avx: + vmovdqa $H2,0x20(%r11) + vmovdqa $H0,0x00(%r11) + vmovdqa $H1,0x10(%r11) + vmovdqa $H3,0x30(%r11) + vmovdqa $H4,0x40(%r11) + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $T2,$D4,$D2 # d2 = h2*r0 + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vpmuludq $T3,$H2,$H0 # h3*r1 + vpaddq $H0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n + vpmuludq $T2,$H2,$H1 # h2*r1 + vpaddq $H1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n + vpmuludq $T1,$H2,$H0 # h1*r1 + vpaddq $H0,$D2,$D2 # d2 += h1*r1 + vpmuludq $T0,$H2,$H2 # h0*r1 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + vpmuludq $T4,$H3,$H3 # h4*s1 + vpaddq $H3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n + vpmuludq $T2,$H4,$H1 # h2*r2 + vpaddq $H1,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H4,$H0 # h1*r2 + vpaddq $H0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n + vpmuludq $T0,$H4,$H4 # h0*r2 + vpaddq $H4,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H2,$H1 # h4*s2 + vpaddq $H1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n + vpmuludq $T3,$H2,$H2 # h3*s2 + vpaddq $H2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $T1,$H3,$H0 # h1*r3 + vpaddq $H0,$D4,$D4 # d4 += h1*r3 + vpmuludq $T0,$H3,$H3 # h0*r3 + vpaddq $H3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n + vpmuludq $T4,$H4,$H1 # h4*s3 + vpaddq $H1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n + vpmuludq $T3,$H4,$H0 # h3*s3 + vpaddq $H0,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H4,$H4 # h2*s3 + vpaddq $H4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $T0,$H2,$H2 # h0*r4 + vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4 + vpmuludq $T4,$H3,$H1 # h4*s4 + vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4 + vpmuludq $T3,$H3,$H0 # h3*s4 + vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4 + vpmuludq $T2,$H3,$H1 # h2*s4 + vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4 + vpmuludq $T1,$H3,$H3 # h1*s4 + vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4 + + jz .Lshort_tail_avx + + vmovdqu 16*0($inp),$H0 # load input + vmovdqu 16*1($inp),$H1 + + vpsrldq \$6,$H0,$H2 # splat input + vpsrldq \$6,$H1,$H3 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + vpsrlq \$40,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4 + vpaddq 0x00(%r11),$H0,$H0 + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + ################################################################ + # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpaddq $T0,$D0,$D0 # d0 += h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T1,$D1,$D1 # d1 += h1*r0 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpaddq $T0,$D2,$D2 # d2 += h2*r0 + vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T1,$D3,$D3 # d3 += h3*r0 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpaddq $T4,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2 + vpmuludq $H1,$T2,$T0 # h1*r1 + vpaddq $T0,$D2,$D2 # d2 += h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + vpmuludq $H4,$T3,$T3 # h4*s1 + vpaddq $T3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2 + vpmuludq $H2,$T4,$T1 # h2*r2 + vpaddq $T1,$D4,$D4 # d4 += h2*r2 + vpmuludq $H1,$T4,$T0 # h1*r2 + vpaddq $T0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3 + vpmuludq $H0,$T4,$T4 # h0*r2 + vpaddq $T4,$D2,$D2 # d2 += h0*r2 + vpmuludq $H4,$T2,$T1 # h4*s2 + vpaddq $T1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3 + vpmuludq $H3,$T2,$T2 # h3*s2 + vpaddq $T2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $H1,$T3,$T0 # h1*r3 + vpaddq $T0,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T3,$T3 # h0*r3 + vpaddq $T3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4 + vpmuludq $H4,$T4,$T1 # h4*s3 + vpaddq $T1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4 + vpmuludq $H3,$T4,$T0 # h3*s3 + vpaddq $T0,$D1,$D1 # d1 += h3*s3 + vpmuludq $H2,$T4,$T4 # h2*s3 + vpaddq $T4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H0,$T2,$T2 # h0*r4 + vpaddq $T2,$D4,$D4 # d4 += h0*r4 + vpmuludq $H4,$T3,$T1 # h4*s4 + vpaddq $T1,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$T3,$T0 # h3*s4 + vpaddq $T0,$D2,$D2 # d2 += h3*s4 + vpmuludq $H2,$T3,$T1 # h2*s4 + vpaddq $T1,$D1,$D1 # d1 += h2*s4 + vpmuludq $H1,$T3,$T3 # h1*s4 + vpaddq $T3,$D0,$D0 # d0 += h1*s4 + +.Lshort_tail_avx: + ################################################################ + # horizontal addition + + vpsrldq \$8,$D4,$T4 + vpsrldq \$8,$D3,$T3 + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$D0,$T0 + vpsrldq \$8,$D2,$T2 + vpaddq $T3,$D3,$D3 + vpaddq $T4,$D4,$D4 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$D2,$D2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D4,$H4 + vpand $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$H1 + vpand $MASK,$D1,$D1 + vpaddq $H1,$D2,$D2 # h1 -> h2 + + vpaddq $H4,$D0,$D0 + vpsllq \$2,$H4,$H4 + vpaddq $H4,$D0,$D0 # h4 -> h0 + + vpsrlq \$26,$D2,$H2 + vpand $MASK,$D2,$D2 + vpaddq $H2,$D3,$D3 # h2 -> h3 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vmovd $D0,`4*0-48-64`($ctx) # save partially reduced + vmovd $D1,`4*1-48-64`($ctx) + vmovd $D2,`4*2-48-64`($ctx) + vmovd $D3,`4*3-48-64`($ctx) + vmovd $D4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa 0x50(%r11),%xmm6 + vmovdqa 0x60(%r11),%xmm7 + vmovdqa 0x70(%r11),%xmm8 + vmovdqa 0x80(%r11),%xmm9 + vmovdqa 0x90(%r11),%xmm10 + vmovdqa 0xa0(%r11),%xmm11 + vmovdqa 0xb0(%r11),%xmm12 + vmovdqa 0xc0(%r11),%xmm13 + vmovdqa 0xd0(%r11),%xmm14 + vmovdqa 0xe0(%r11),%xmm15 + lea 0xf8(%r11),%rsp +.Ldo_avx_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_avx"); + +&declare_function("poly1305_emit_avx", 32, 3); +$code.=<<___; + cmpl \$0,20($ctx) # is_base2_26? + je .Lemit + + mov 0($ctx),%eax # load hash value base 2^26 + mov 4($ctx),%ecx + mov 8($ctx),%r8d + mov 12($ctx),%r11d + mov 16($ctx),%r10d + + shl \$26,%rcx # base 2^26 -> base 2^64 + mov %r8,%r9 + shl \$52,%r8 + add %rcx,%rax + shr \$12,%r9 + add %rax,%r8 # h0 + adc \$0,%r9 + + shl \$14,%r11 + mov %r10,%rax + shr \$24,%r10 + add %r11,%r9 + shl \$40,%rax + add %rax,%r9 # h1 + adc \$0,%r10 # h2 + + mov %r10,%rax # could be partially reduced, so reduce + mov %r10,%rcx + and \$3,%r10 + shr \$2,%rax + and \$-4,%rcx + add %rcx,%rax + add %rax,%r8 + adc \$0,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_avx"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if ($avx>1) { + +if ($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = + map("%ymm$_",(0..15)); +my $S4=$MASK; + +sub poly1305_blocks_avxN { + my ($avx512) = @_; + my $suffix = $avx512 ? "_avx512" : ""; +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx2$suffix + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx2$suffix: + and \$-16,$len + jz .Lno_data_avx2$suffix + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx2$suffix + + test \$63,$len + jz .Leven_avx2$suffix + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + +.Lbase2_26_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_26_pre_avx2$suffix + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + test %r15,%r15 + jz .Lstore_base2_26_avx2$suffix + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + jmp .Lproceed_avx2$suffix + +.align 32 +.Lstore_base2_64_avx2$suffix: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx2$suffix + +.align 16 +.Lstore_base2_26_avx2$suffix: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx2$suffix: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx2$suffix: +.Lblocks_avx2_epilogue$suffix: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx2$suffix: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$63,$len + jz .Linit_avx2$suffix + +.Lbase2_64_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_64_pre_avx2$suffix + +.Linit_avx2$suffix: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx2$suffix: + mov %r15,$len # restore $len +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d + mov \$`(1<<31|1<<30|1<<16)`,%r11d +___ +$code.=<<___; + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx2_epilogue$suffix: + jmp .Ldo_avx2$suffix +.cfi_endproc + +.align 32 +.Leven_avx2$suffix: +.cfi_startproc +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d +___ +$code.=<<___; + vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 + vmovd 4*1($ctx),%x#$H1 + vmovd 4*2($ctx),%x#$H2 + vmovd 4*3($ctx),%x#$H3 + vmovd 4*4($ctx),%x#$H4 + +.Ldo_avx2$suffix: +___ +$code.=<<___ if (!$kernel && $avx>2); + cmp \$512,$len + jb .Lskip_avx512 + and %r11d,%r9d + test \$`1<<16`,%r9d # check for AVX512F + jnz .Lblocks_avx512 +.Lskip_avx512$suffix: +___ +$code.=<<___ if ($avx > 2 && $avx512 && $kernel); + cmp \$512,$len + jae .Lblocks_avx512 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx2_body$suffix: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),$T0 # .Lpermd_avx2 + + # expand and copy pre-calculated table to stack + vmovdqu `16*0-64`($ctx),%x#$T2 + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$T3 + vmovdqu `16*2-64`($ctx),%x#$T4 + vmovdqu `16*3-64`($ctx),%x#$D0 + vmovdqu `16*4-64`($ctx),%x#$D1 + vmovdqu `16*5-64`($ctx),%x#$D2 + lea 0x90(%rsp),%rax # size optimization + vmovdqu `16*6-64`($ctx),%x#$D3 + vpermd $T2,$T0,$T2 # 00003412 -> 14243444 + vmovdqu `16*7-64`($ctx),%x#$D4 + vpermd $T3,$T0,$T3 + vmovdqu `16*8-64`($ctx),%x#$MASK + vpermd $T4,$T0,$T4 + vmovdqa $T2,0x00(%rsp) + vpermd $D0,$T0,$D0 + vmovdqa $T3,0x20-0x90(%rax) + vpermd $D1,$T0,$D1 + vmovdqa $T4,0x40-0x90(%rax) + vpermd $D2,$T0,$D2 + vmovdqa $D0,0x60-0x90(%rax) + vpermd $D3,$T0,$D3 + vmovdqa $D1,0x80-0x90(%rax) + vpermd $D4,$T0,$D4 + vmovdqa $D2,0xa0-0x90(%rax) + vpermd $MASK,$T0,$MASK + vmovdqa $D3,0xc0-0x90(%rax) + vmovdqa $D4,0xe0-0x90(%rax) + vmovdqa $MASK,0x100-0x90(%rax) + vmovdqa 64(%rcx),$MASK # .Lmask26 + + ################################################################ + # load input + vmovdqu 16*0($inp),%x#$T0 + vmovdqu 16*1($inp),%x#$T1 + vinserti128 \$1,16*2($inp),$T0,$T0 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$64,$len + jz .Ltail_avx2$suffix + jmp .Loop_avx2$suffix + +.align 32 +.Loop_avx2$suffix: + ################################################################ + # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 + # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 + # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2 + # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1 + # \________/\__________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqa `32*0`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqa `32*1`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqa `32*3`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqa `32*6-0x90`(%rax),$T3 # s3^4 + vmovdqa `32*8-0x90`(%rax),$S4 # s4^4 + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4 + # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + vmovdqa `32*4-0x90`(%rax),$T1 # s2 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vmovdqu 16*0($inp),%x#$T0 # load input + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + vinserti128 \$1,16*2($inp),$T0,$T0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vmovdqu 16*1($inp),%x#$T1 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqa `32*5-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpsrldq \$6,$T1,$T3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + vpunpckhqdq $T1,$T0,$T4 # 4 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # lazy reduction (interleaved with tail of input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$4,$T3,$T2 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpand $MASK,$T2,$T2 # 2 + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$30,$T3,$T3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + sub \$64,$len + jnz .Loop_avx2$suffix + + .byte 0x66,0x90 +.Ltail_avx2$suffix: + ################################################################ + # while above multiplications were by r^4 in all lanes, in last + # iteration we multiply least significant lane by r^4 and most + # significant one by r, so copy of above except that references + # to the precomputed table are displaced by 4... + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqu `32*0+4`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqu `32*1+4`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqu `32*3+4`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4 + vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1 + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vmovdqu `32*4+4-0x90`(%rax),$T1 # s2 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqu `32*5+4-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # horizontal addition + + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$H2,$T2 + vpsrldq \$8,$H3,$T3 + vpsrldq \$8,$H4,$T4 + vpsrldq \$8,$H0,$T0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + + vpermq \$0x2,$H3,$T3 + vpermq \$0x2,$H4,$T4 + vpermq \$0x2,$H0,$T0 + vpermq \$0x2,$D1,$T1 + vpermq \$0x2,$H2,$T2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa -0xb0(%r10),%xmm6 + vmovdqa -0xa0(%r10),%xmm7 + vmovdqa -0x90(%r10),%xmm8 + vmovdqa -0x80(%r10),%xmm9 + vmovdqa -0x70(%r10),%xmm10 + vmovdqa -0x60(%r10),%xmm11 + vmovdqa -0x50(%r10),%xmm12 + vmovdqa -0x40(%r10),%xmm13 + vmovdqa -0x30(%r10),%xmm14 + vmovdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx2_epilogue$suffix: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +if($avx > 2 && $avx512) { +my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); +my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); +my $PADBIT="%zmm30"; + +map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain +map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4)); +map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); +map(s/%y/%z/,($MASK)); + +$code.=<<___; +.cfi_startproc +.Lblocks_avx512: + mov \$15,%eax + kmovw %eax,%k2 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx512_body: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2 + + # expand pre-calculated table + vmovdqu `16*0-64`($ctx),%x#$D0 # will become expanded ${R0} + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$D1 # will become ... ${R1} + mov \$0x20,%rax + vmovdqu `16*2-64`($ctx),%x#$T0 # ... ${S1} + vmovdqu `16*3-64`($ctx),%x#$D2 # ... ${R2} + vmovdqu `16*4-64`($ctx),%x#$T1 # ... ${S2} + vmovdqu `16*5-64`($ctx),%x#$D3 # ... ${R3} + vmovdqu `16*6-64`($ctx),%x#$T3 # ... ${S3} + vmovdqu `16*7-64`($ctx),%x#$D4 # ... ${R4} + vmovdqu `16*8-64`($ctx),%x#$T4 # ... ${S4} + vpermd $D0,$T2,$R0 # 00003412 -> 14243444 + vpbroadcastq 64(%rcx),$MASK # .Lmask26 + vpermd $D1,$T2,$R1 + vpermd $T0,$T2,$S1 + vpermd $D2,$T2,$R2 + vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0 + vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304 + vpermd $T1,$T2,$S2 + vmovdqu64 $R1,0x00(%rsp,%rax){%k2} + vpsrlq \$32,$R1,$T1 + vpermd $D3,$T2,$R3 + vmovdqa64 $S1,0x40(%rsp){%k2} + vpermd $T3,$T2,$S3 + vpermd $D4,$T2,$R4 + vmovdqu64 $R2,0x40(%rsp,%rax){%k2} + vpermd $T4,$T2,$S4 + vmovdqa64 $S2,0x80(%rsp){%k2} + vmovdqu64 $R3,0x80(%rsp,%rax){%k2} + vmovdqa64 $S3,0xc0(%rsp){%k2} + vmovdqu64 $R4,0xc0(%rsp,%rax){%k2} + vmovdqa64 $S4,0x100(%rsp){%k2} + + ################################################################ + # calculate 5th through 8th powers of the key + # + # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1 + # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2 + # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3 + # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4 + # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0 + + vpmuludq $T0,$R0,$D0 # d0 = r0'*r0 + vpmuludq $T0,$R1,$D1 # d1 = r0'*r1 + vpmuludq $T0,$R2,$D2 # d2 = r0'*r2 + vpmuludq $T0,$R3,$D3 # d3 = r0'*r3 + vpmuludq $T0,$R4,$D4 # d4 = r0'*r4 + vpsrlq \$32,$R2,$T2 + + vpmuludq $T1,$S4,$M0 + vpmuludq $T1,$R0,$M1 + vpmuludq $T1,$R1,$M2 + vpmuludq $T1,$R2,$M3 + vpmuludq $T1,$R3,$M4 + vpsrlq \$32,$R3,$T3 + vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4 + vpaddq $M1,$D1,$D1 # d1 += r1'*r0 + vpaddq $M2,$D2,$D2 # d2 += r1'*r1 + vpaddq $M3,$D3,$D3 # d3 += r1'*r2 + vpaddq $M4,$D4,$D4 # d4 += r1'*r3 + + vpmuludq $T2,$S3,$M0 + vpmuludq $T2,$S4,$M1 + vpmuludq $T2,$R1,$M3 + vpmuludq $T2,$R2,$M4 + vpmuludq $T2,$R0,$M2 + vpsrlq \$32,$R4,$T4 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4 + vpaddq $M3,$D3,$D3 # d3 += r2'*r1 + vpaddq $M4,$D4,$D4 # d4 += r2'*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*r0 + + vpmuludq $T3,$S2,$M0 + vpmuludq $T3,$R0,$M3 + vpmuludq $T3,$R1,$M4 + vpmuludq $T3,$S3,$M1 + vpmuludq $T3,$S4,$M2 + vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2 + vpaddq $M3,$D3,$D3 # d3 += r3'*r0 + vpaddq $M4,$D4,$D4 # d4 += r3'*r1 + vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3 + vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4 + + vpmuludq $T4,$S4,$M3 + vpmuludq $T4,$R0,$M4 + vpmuludq $T4,$S1,$M0 + vpmuludq $T4,$S2,$M1 + vpmuludq $T4,$S3,$M2 + vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4 + vpaddq $M4,$D4,$D4 # d4 += r2'*r0 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3 + + ################################################################ + # load input + vmovdqu64 16*0($inp),%z#$T3 + vmovdqu64 16*4($inp),%z#$T4 + lea 16*8($inp),$inp + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D4,$M4 + vpandq $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$M1 + vpandq $MASK,$D1,$D1 + vpaddq $M1,$D2,$D2 # d1 -> d2 + + vpaddq $M4,$D0,$D0 + vpsllq \$2,$M4,$M4 + vpaddq $M4,$D0,$D0 # d4 -> d0 + + vpsrlq \$26,$D2,$M2 + vpandq $MASK,$D2,$D2 + vpaddq $M2,$D3,$D3 # d2 -> d3 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + ################################################################ + # at this point we have 14243444 in $R0-$S4 and 05060708 in + # $D0-$D4, ... + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + # ... since input 64-bit lanes are ordered as 73625140, we could + # "vperm" it to 76543210 (here and in each loop iteration), *or* + # we could just flow along, hence the goal for $R0-$S4 is + # 1858286838784888 ... + + vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512: + mov \$0x7777,%eax + kmovw %eax,%k1 + + vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4--- + vpermd $R1,$M0,$R1 + vpermd $R2,$M0,$R2 + vpermd $R3,$M0,$R3 + vpermd $R4,$M0,$R4 + + vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888 + vpermd $D1,$M0,${R1}{%k1} + vpermd $D2,$M0,${R2}{%k1} + vpermd $D3,$M0,${R3}{%k1} + vpermd $D4,$M0,${R4}{%k1} + + vpslld \$2,$R1,$S1 # *5 + vpslld \$2,$R2,$S2 + vpslld \$2,$R3,$S3 + vpslld \$2,$R4,$S4 + vpaddd $R1,$S1,$S1 + vpaddd $R2,$S2,$S2 + vpaddd $R3,$S3,$S3 + vpaddd $R4,$S4,$S4 + + vpbroadcastq 32(%rcx),$PADBIT # .L129 + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + vporq $T3,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$14,$T4,$T3 + vpsrlq \$40,$T4,$T4 # 4 + vpandq $MASK,$T2,$T2 # 2 + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$192,$len + jbe .Ltail_avx512 + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: + ################################################################ + # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8 + # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7 + # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6 + # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5 + # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4 + # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3 + # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2 + # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1 + # \________/\___________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4 + # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0 + # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1 + # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2 + # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpaddq $H0,$T0,$H0 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu64 16*0($inp),$T3 # load input + vmovdqu64 16*4($inp),$T4 + lea 16*8($inp),$inp + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpaddq $M3,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$S4,$M2 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + + vpsrlq \$26,$D3,$H3 + vpandq $MASK,$D3,$D3 + vpaddq $H3,$D4,$H4 # h3 -> h4 + + vporq $T3,$T2,$T2 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpandq $MASK,$T2,$T2 # 2 + + vpsrlq \$26,$H4,$D4 + vpandq $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpandq $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpandq $MASK,$H2,$H2 + vpaddq $D2,$D3,$H3 # h2 -> h3 + + vpsrlq \$14,$T4,$T3 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpandq $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + sub \$128,$len + ja .Loop_avx512 + +.Ltail_avx512: + ################################################################ + # while above multiplications were by r^8 in all lanes, in last + # iteration we multiply least significant lane by r^8 and most + # significant one by r, that's why table gets shifted... + + vpsrlq \$32,$R0,$R0 # 0105020603070408 + vpsrlq \$32,$R1,$R1 + vpsrlq \$32,$R2,$R2 + vpsrlq \$32,$S3,$S3 + vpsrlq \$32,$S4,$S4 + vpsrlq \$32,$R3,$R3 + vpsrlq \$32,$R4,$R4 + vpsrlq \$32,$S1,$S1 + vpsrlq \$32,$S2,$S2 + + ################################################################ + # load either next or last 64 byte of input + lea ($inp,$len),$inp + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu 16*0($inp),%x#$T0 + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vmovdqu 16*1($inp),%x#$T1 + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0 + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1 + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpmuludq $H3,$S4,$M2 + vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # horizontal addition + + mov \$1,%eax + vpermq \$0xb1,$H3,$D3 + vpermq \$0xb1,$D4,$H4 + vpermq \$0xb1,$H0,$D0 + vpermq \$0xb1,$H1,$D1 + vpermq \$0xb1,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + kmovw %eax,%k3 + vpermq \$0x2,$H3,$D3 + vpermq \$0x2,$H4,$D4 + vpermq \$0x2,$H0,$D0 + vpermq \$0x2,$H1,$D1 + vpermq \$0x2,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + vextracti64x4 \$0x1,$H3,%y#$D3 + vextracti64x4 \$0x1,$H4,%y#$D4 + vextracti64x4 \$0x1,$H0,%y#$D0 + vextracti64x4 \$0x1,$H1,%y#$D1 + vextracti64x4 \$0x1,$H2,%y#$D2 + vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case + vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2 + vpaddq $D0,$H0,${H0}{%k3}{z} + vpaddq $D1,$H1,${H1}{%k3}{z} + vpaddq $D2,$H2,${H2}{%k3}{z} +___ +map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT)); +map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK)); +$code.=<<___; + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2 + vpand $MASK,$T1,$T1 # 1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + vpaddq $D3,$H4,$H4 # h3 -> h4 + + lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 + add \$64,$len + jnz .Ltail_avx2$suffix + + vpsubq $T2,$H2,$H2 # undo input accumulation + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) + vzeroall +___ +$code.=<<___ if ($win64); + movdqa -0xb0(%r10),%xmm6 + movdqa -0xa0(%r10),%xmm7 + movdqa -0x90(%r10),%xmm8 + movdqa -0x80(%r10),%xmm9 + movdqa -0x70(%r10),%xmm10 + movdqa -0x60(%r10),%xmm11 + movdqa -0x50(%r10),%xmm12 + movdqa -0x40(%r10),%xmm13 + movdqa -0x30(%r10),%xmm14 + movdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx512_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + ret +.cfi_endproc +___ + +} + +} + +&declare_function("poly1305_blocks_avx2", 32, 4); +poly1305_blocks_avxN(0); +&end_function("poly1305_blocks_avx2"); + +if($kernel) { + $code .= "#endif\n"; +} + +####################################################################### +if ($avx>2) { +# On entry we have input length divisible by 64. But since inner loop +# processes 128 bytes per iteration, cases when length is not divisible +# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this +# reason stack layout is kept identical to poly1305_blocks_avx2. If not +# for this tail, we wouldn't have to even allocate stack frame... + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +&declare_function("poly1305_blocks_avx512", 32, 4); +poly1305_blocks_avxN(1); +&end_function("poly1305_blocks_avx512"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if (!$kernel && $avx>3) { +######################################################################## +# VPMADD52 version using 2^44 radix. +# +# One can argue that base 2^52 would be more natural. Well, even though +# some operations would be more natural, one has to recognize couple of +# things. Base 2^52 doesn't provide advantage over base 2^44 if you look +# at amount of multiply-n-accumulate operations. Secondly, it makes it +# impossible to pre-compute multiples of 5 [referred to as s[]/sN in +# reference implementations], which means that more such operations +# would have to be performed in inner loop, which in turn makes critical +# path longer. In other words, even though base 2^44 reduction might +# look less elegant, overall critical path is actually shorter... + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^44 +# unsigned __int64 s[2]; # key value*20 base 2^44 +# unsigned __int64 r[3]; # key value base 2^44 +# struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4]; +# # r^n positions reflect +# # placement in register, not +# # memory, R[3] is R[1]*20 + +$code.=<<___; +.type poly1305_init_base2_44,\@function,3 +.align 32 +poly1305_init_base2_44: + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + +.Linit_base2_44: + lea poly1305_blocks_vpmadd52(%rip),%r10 + lea poly1305_emit_base2_44(%rip),%r11 + + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + mov \$0x00000fffffffffff,%r8 + and 8($inp),%rcx + mov \$0x00000fffffffffff,%r9 + and %rax,%r8 + shrd \$44,%rcx,%rax + mov %r8,40($ctx) # r0 + and %r9,%rax + shr \$24,%rcx + mov %rax,48($ctx) # r1 + lea (%rax,%rax,4),%rax # *5 + mov %rcx,56($ctx) # r2 + shl \$2,%rax # magic <<2 + lea (%rcx,%rcx,4),%rcx # *5 + shl \$2,%rcx # magic <<2 + mov %rax,24($ctx) # s1 + mov %rcx,32($ctx) # s2 + movq \$-1,64($ctx) # write impossible value +___ +$code.=<<___ if ($flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if ($flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax + ret +.size poly1305_init_base2_44,.-poly1305_init_base2_44 +___ +{ +my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17)); +my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21)); +my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52,\@function,4 +.align 32 +poly1305_blocks_vpmadd52: + shr \$4,$len + jz .Lno_data_vpmadd52 # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + # if powers of the key are not calculated yet, process up to 3 + # blocks with this single-block subroutine, otherwise ensure that + # length is divisible by 2 blocks and pass the rest down to next + # subroutine... + + mov \$3,%rax + mov \$1,%r10 + cmp \$4,$len # is input long + cmovae %r10,%rax + test %r8,%r8 # is power value impossible? + cmovns %r10,%rax + + and $len,%rax # is input of favourable length? + jz .Lblocks_vpmadd52_4x + + sub %rax,$len + mov \$7,%r10d + mov \$1,%r11d + kmovw %r10d,%k7 + lea .L2_44_inp_permd(%rip),%r10 + kmovw %r11d,%k1 + + vmovq $padbit,%x#$PAD + vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd + vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift + vpermq \$0xcf,$PAD,$PAD + vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask + + vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value + vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys + vmovdqu64 32($ctx),${r1r0s2}{%k7}{z} + vmovdqu64 24($ctx),${r0s2s1}{%k7}{z} + + vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt + vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft + + jmp .Loop_vpmadd52 + +.align 32 +.Loop_vpmadd52: + vmovdqu32 0($inp),%x#$T0 # load input as ----3210 + lea 16($inp),$inp + + vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110 + vpsrlvq $inp_shift,$T0,$T0 + vpandq $reduc_mask,$T0,$T0 + vporq $PAD,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo # accumulate input + + vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value + vpermq \$0b01010101,$Dlo,${H1}{%k7}{z} + vpermq \$0b10101010,$Dlo,${H2}{%k7}{z} + + vpxord $Dlo,$Dlo,$Dlo + vpxord $Dhi,$Dhi,$Dhi + + vpmadd52luq $r2r1r0,$H0,$Dlo + vpmadd52huq $r2r1r0,$H0,$Dhi + + vpmadd52luq $r1r0s2,$H1,$Dlo + vpmadd52huq $r1r0s2,$H1,$Dhi + + vpmadd52luq $r0s2s1,$H2,$Dlo + vpmadd52huq $r0s2s1,$H2,$Dhi + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword + vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword + vpandq $reduc_mask,$Dlo,$Dlo + + vpaddq $T0,$Dhi,$Dhi + + vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword + + vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-) + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word + vpandq $reduc_mask,$Dlo,$Dlo + + vpermq \$0b10010011,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + vpermq \$0b10010011,$Dlo,${T0}{%k1}{z} + + vpaddq $T0,$Dlo,$Dlo + vpsllq \$2,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + dec %rax # len-=16 + jnz .Loop_vpmadd52 + + vmovdqu64 $Dlo,0($ctx){%k7} # store hash value + + test $len,$len + jnz .Lblocks_vpmadd52_4x + +.Lno_data_vpmadd52: + ret +.size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52 +___ +} +{ +######################################################################## +# As implied by its name 4x subroutine processes 4 blocks in parallel +# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power +# and is handled in 256-bit %ymm registers. + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_4x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_4x: + shr \$4,$len + jz .Lno_data_vpmadd52_4x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + +.Lblocks_vpmadd52_4x: + vpbroadcastq $padbit,$PAD + + vmovdqa64 .Lx_mask44(%rip),$mask44 + mov \$5,%eax + vmovdqa64 .Lx_mask42(%rip),$mask42 + kmovw %eax,%k1 # used in 2x path + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Lblocks_vpmadd52_2x_do + +.Lblocks_vpmadd52_4x_do: + vpbroadcastq 64($ctx),$R0 # load 4th power of the key + vpbroadcastq 96($ctx),$R1 + vpbroadcastq 128($ctx),$R2 + vpbroadcastq 160($ctx),$S1 + +.Lblocks_vpmadd52_4x_key_loaded: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + test \$7,$len # is len 8*n? + jz .Lblocks_vpmadd52_8x + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 3-1-2-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$4,$len + jz .Ltail_vpmadd52_4x + jmp .Loop_vpmadd52_4x + ud2 + +.align 32 +.Linit_vpmadd52: + vmovq 24($ctx),%x#$S1 # load key + vmovq 56($ctx),%x#$H2 + vmovq 32($ctx),%x#$S2 + vmovq 40($ctx),%x#$R0 + vmovq 48($ctx),%x#$R1 + + vmovdqa $R0,$H0 + vmovdqa $R1,$H1 + vmovdqa $H2,$R2 + + mov \$2,%eax + +.Lmul_init_vpmadd52: + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + dec %eax + jz .Ldone_init_vpmadd52 + + vpunpcklqdq $R1,$H1,$R1 # 1,2 + vpbroadcastq %x#$H1,%x#$H1 # 2,2 + vpunpcklqdq $R2,$H2,$R2 + vpbroadcastq %x#$H2,%x#$H2 + vpunpcklqdq $R0,$H0,$R0 + vpbroadcastq %x#$H0,%x#$H0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R1,$S1,$S1 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S1,$S1 + vpsllq \$2,$S2,$S2 + + jmp .Lmul_init_vpmadd52 + ud2 + +.align 32 +.Ldone_init_vpmadd52: + vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4 + vinserti128 \$1,%x#$R2,$H2,$R2 + vinserti128 \$1,%x#$R0,$H0,$R0 + + vpermq \$0b11011000,$R1,$R1 # 1,3,2,4 + vpermq \$0b11011000,$R2,$R2 + vpermq \$0b11011000,$R0,$R0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpaddq $R1,$S1,$S1 + vpsllq \$2,$S1,$S1 + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Ldone_init_vpmadd52_2x + + vmovdqu64 $R0,64($ctx) # save key powers + vpbroadcastq %x#$R0,$R0 # broadcast 4th power + vmovdqu64 $R1,96($ctx) + vpbroadcastq %x#$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpbroadcastq %x#$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpbroadcastq %x#$S1,$S1 + + jmp .Lblocks_vpmadd52_4x_key_loaded + ud2 + +.align 32 +.Ldone_init_vpmadd52_2x: + vmovdqu64 $R0,64($ctx) # save key powers + vpsrldq \$8,$R0,$R0 # 0-1-0-2 + vmovdqu64 $R1,96($ctx) + vpsrldq \$8,$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpsrldq \$8,$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpsrldq \$8,$S1,$S1 + jmp .Lblocks_vpmadd52_2x_key_loaded + ud2 + +.align 32 +.Lblocks_vpmadd52_2x_do: + vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers + vmovdqu64 160+8($ctx),${S1}{%k1}{z} + vmovdqu64 64+8($ctx),${R0}{%k1}{z} + vmovdqu64 96+8($ctx),${R1}{%k1}{z} + +.Lblocks_vpmadd52_2x_key_loaded: + vmovdqu64 16*0($inp),$T2 # load data + vpxorq $T3,$T3,$T3 + lea 16*2($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as x-1-x-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + jmp .Ltail_vpmadd52_2x + ud2 + +.align 32 +.Loop_vpmadd52_4x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$4,$len # len-=64 + jnz .Loop_vpmadd52_4x + +.Ltail_vpmadd52_4x: + vmovdqu64 128($ctx),$R2 # load all key powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + +.Ltail_vpmadd52_2x: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + # at this point $len is + # either 4*n+2 or 0... + sub \$2,$len # len-=32 + ja .Lblocks_vpmadd52_4x_do + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_4x: + ret +.size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x +___ +} +{ +######################################################################## +# As implied by its name 8x subroutine processes 8 blocks in parallel... +# This is intermediate version, as it's used only in cases when input +# length is either 8*n, 8*n+1 or 8*n+2... + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); +my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_8x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_8x: + shr \$4,$len + jz .Lno_data_vpmadd52_8x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + vmovdqa64 .Lx_mask44(%rip),$mask44 + vmovdqa64 .Lx_mask42(%rip),$mask42 + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + +.Lblocks_vpmadd52_8x: + ################################################################ + # fist we calculate more key powers + + vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + vpbroadcastq %x#$R2,$RR2 # broadcast 4th power + vpbroadcastq %x#$R0,$RR0 + vpbroadcastq %x#$R1,$RR1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $RR2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $RR2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $RR2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $RR2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $RR2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $RR2,$R0,$D2hi + + vpmadd52luq $RR0,$R0,$D0lo + vpmadd52huq $RR0,$R0,$D0hi + vpmadd52luq $RR0,$R1,$D1lo + vpmadd52huq $RR0,$R1,$D1hi + vpmadd52luq $RR0,$R2,$D2lo + vpmadd52huq $RR0,$R2,$D2hi + + vpmadd52luq $RR1,$S2,$D0lo + vpmadd52huq $RR1,$S2,$D0hi + vpmadd52luq $RR1,$R0,$D1lo + vpmadd52huq $RR1,$R0,$D1hi + vpmadd52luq $RR1,$R1,$D2lo + vpmadd52huq $RR1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$RR0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$RR1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$RR2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + + vpsrlq \$44,$RR0,$tmp # additional step + vpandq $mask44,$RR0,$RR0 + + vpaddq $tmp,$RR1,$RR1 + + ################################################################ + # At this point Rx holds 1324 powers, RRx - 5768, and the goal + # is 15263748, which reflects how data is loaded... + + vpunpcklqdq $R2,$RR2,$T2 # 3748 + vpunpckhqdq $R2,$RR2,$R2 # 1526 + vpunpcklqdq $R0,$RR0,$T0 + vpunpckhqdq $R0,$RR0,$R0 + vpunpcklqdq $R1,$RR1,$T1 + vpunpckhqdq $R1,$RR1,$R1 +___ +######## switch to %zmm +map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); +map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2); + +$code.=<<___; + vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748 + vshufi64x2 \$0x44,$R0,$T0,$RR0 + vshufi64x2 \$0x44,$R1,$T1,$RR1 + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + + vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4 + vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4 + vpaddq $RR2,$SS2,$SS2 + vpaddq $RR1,$SS1,$SS1 + vpsllq \$2,$SS2,$SS2 + vpsllq \$2,$SS1,$SS1 + + vpbroadcastq $padbit,$PAD + vpbroadcastq %x#$mask44,$mask44 + vpbroadcastq %x#$mask42,$mask42 + + vpbroadcastq %x#$SS1,$S1 # broadcast 8th power + vpbroadcastq %x#$SS2,$S2 + vpbroadcastq %x#$RR0,$R0 + vpbroadcastq %x#$RR1,$R1 + vpbroadcastq %x#$RR2,$R2 + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 73625140 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$8,$len + jz .Ltail_vpmadd52_8x + jmp .Loop_vpmadd52_8x + +.align 32 +.Loop_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$8,$len # len-=128 + jnz .Loop_vpmadd52_8x + +.Ltail_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$SS1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$SS1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$SS2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$SS2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$RR0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$RR0,$D2hi + + vpmadd52luq $H0,$RR0,$D0lo + vpmadd52huq $H0,$RR0,$D0hi + vpmadd52luq $H0,$RR1,$D1lo + vpmadd52huq $H0,$RR1,$D1hi + vpmadd52luq $H0,$RR2,$D2lo + vpmadd52huq $H0,$RR2,$D2hi + + vpmadd52luq $H1,$SS2,$D0lo + vpmadd52huq $H1,$SS2,$D0hi + vpmadd52luq $H1,$RR0,$D1lo + vpmadd52huq $H1,$RR0,$D1hi + vpmadd52luq $H1,$RR1,$D2lo + vpmadd52huq $H1,$RR1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vextracti64x4 \$1,$D0lo,%y#$T0 + vextracti64x4 \$1,$D0hi,%y#$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vextracti64x4 \$1,$D1lo,%y#$T1 + vextracti64x4 \$1,$D1hi,%y#$H1 + vextracti64x4 \$1,$D2lo,%y#$T2 + vextracti64x4 \$1,$D2hi,%y#$H2 +___ +######## switch back to %ymm +map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); + +$code.=<<___; + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + ################################################################ + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_8x: + ret +.size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x +___ +} +$code.=<<___; +.type poly1305_emit_base2_44,\@function,3 +.align 32 +poly1305_emit_base2_44: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r9,%rax + shr \$20,%r9 + shl \$44,%rax + mov %r10,%rcx + shr \$40,%r10 + shl \$24,%rcx + + add %rax,%r8 + adc %rcx,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +.size poly1305_emit_base2_44,.-poly1305_emit_base2_44 +___ +} } } +} + +if (!$kernel) +{ # chacha20-poly1305 helpers +my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order +$code.=<<___; +.globl xor128_encrypt_n_pad +.type xor128_encrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_encrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_enc + nop +.Loop_enc_xmm: + movdqu ($inp,$otp),%xmm0 + pxor ($otp),%xmm0 + movdqu %xmm0,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_enc_xmm + + and \$15,%r10 # len % 16 + jz .Ldone_enc + +.Ltail_enc: + mov \$16,$len + sub %r10,$len + xor %eax,%eax +.Loop_enc_byte: + mov ($inp,$otp),%al + xor ($otp),%al + mov %al,($out,$otp) + mov %al,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_enc_byte + + xor %eax,%eax +.Loop_enc_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_enc_pad + +.Ldone_enc: + mov $otp,%rax + ret +.size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad + +.globl xor128_decrypt_n_pad +.type xor128_decrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_decrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_dec + nop +.Loop_dec_xmm: + movdqu ($inp,$otp),%xmm0 + movdqa ($otp),%xmm1 + pxor %xmm0,%xmm1 + movdqu %xmm1,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_dec_xmm + + pxor %xmm1,%xmm1 + and \$15,%r10 # len % 16 + jz .Ldone_dec + +.Ltail_dec: + mov \$16,$len + sub %r10,$len + xor %eax,%eax + xor %r11,%r11 +.Loop_dec_byte: + mov ($inp,$otp),%r11b + mov ($otp),%al + xor %r11b,%al + mov %al,($out,$otp) + mov %r11b,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_dec_byte + + xor %eax,%eax +.Loop_dec_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_dec_pad + +.Ldone_dec: + mov $otp,%rax + ret +.size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad +___ +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + + jmp .Lcommon_seh_tail +.size se_handler,.-se_handler + +.type avx_handler,\@abi-omnipotent +.align 16 +avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 208($context),%rax # pull context->R11 + + lea 0x50(%rax),%rsi + lea 0xf8(%rax),%rax + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size avx_handler,.-avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_end_poly1305_init_x86_64 + .rva .LSEH_info_poly1305_init_x86_64 + + .rva .LSEH_begin_poly1305_blocks_x86_64 + .rva .LSEH_end_poly1305_blocks_x86_64 + .rva .LSEH_info_poly1305_blocks_x86_64 + + .rva .LSEH_begin_poly1305_emit_x86_64 + .rva .LSEH_end_poly1305_emit_x86_64 + .rva .LSEH_info_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_poly1305_blocks_avx + .rva .Lbase2_64_avx + .rva .LSEH_info_poly1305_blocks_avx_1 + + .rva .Lbase2_64_avx + .rva .Leven_avx + .rva .LSEH_info_poly1305_blocks_avx_2 + + .rva .Leven_avx + .rva .LSEH_end_poly1305_blocks_avx + .rva .LSEH_info_poly1305_blocks_avx_3 + + .rva .LSEH_begin_poly1305_emit_avx + .rva .LSEH_end_poly1305_emit_avx + .rva .LSEH_info_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_poly1305_blocks_avx2 + .rva .Lbase2_64_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_1 + + .rva .Lbase2_64_avx2 + .rva .Leven_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_2 + + .rva .Leven_avx2 + .rva .LSEH_end_poly1305_blocks_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_3 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_poly1305_blocks_avx512 + .rva .LSEH_end_poly1305_blocks_avx512 + .rva .LSEH_info_poly1305_blocks_avx512 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_poly1305_init_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 + +.LSEH_info_poly1305_blocks_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_body,.Lblocks_epilogue + +.LSEH_info_poly1305_emit_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); +.LSEH_info_poly1305_blocks_avx_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_emit_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); +.LSEH_info_poly1305_blocks_avx2_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[] +___ +$code.=<<___ if ($avx>2); +.LSEH_info_poly1305_blocks_avx512: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[] +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split('\n',$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + s/%r([a-z]+)#d/%e$1/g; + s/%r([0-9]+)#d/%r$1d/g; + s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305.c b/net/wireguard/crypto/zinc/poly1305/poly1305.c new file mode 100644 index 0000000000000..7d373b90e011c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the Poly1305 message authenticator. + * + * Information: https://cr.yp.to/mac.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "poly1305-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "poly1305-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64) +#include "poly1305-mips-glue.c" +#else +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + return false; +} +static inline bool poly1305_blocks_arch(void *ctx, const u8 *input, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + return false; +} +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + return false; +} +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "poly1305-donna64.c" +#else +#include "poly1305-donna32.c" +#endif + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]) +{ + ctx->nonce[0] = get_unaligned_le32(&key[16]); + ctx->nonce[1] = get_unaligned_le32(&key[20]); + ctx->nonce[2] = get_unaligned_le32(&key[24]); + ctx->nonce[3] = get_unaligned_le32(&key[28]); + + if (!poly1305_init_arch(ctx->opaque, key)) + poly1305_init_generic(ctx->opaque, key); + + ctx->num = 0; +} +EXPORT_SYMBOL(poly1305_init); + +static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len, + const u32 padbit, + simd_context_t *simd_context) +{ + if (!poly1305_blocks_arch(ctx, input, len, padbit, simd_context)) + poly1305_blocks_generic(ctx, input, len, padbit); +} + +static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!poly1305_emit_arch(ctx, mac, nonce, simd_context)) + poly1305_emit_generic(ctx, mac, nonce); +} + +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context) +{ + const size_t num = ctx->num; + size_t rem; + + if (num) { + rem = POLY1305_BLOCK_SIZE - num; + if (len < rem) { + memcpy(ctx->data + num, input, len); + ctx->num = num + len; + return; + } + memcpy(ctx->data + num, input, rem); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, + simd_context); + input += rem; + len -= rem; + } + + rem = len % POLY1305_BLOCK_SIZE; + len -= rem; + + if (len >= POLY1305_BLOCK_SIZE) { + poly1305_blocks(ctx->opaque, input, len, 1, simd_context); + input += len; + } + + if (rem) + memcpy(ctx->data, input, rem); + + ctx->num = rem; +} +EXPORT_SYMBOL(poly1305_update); + +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context) +{ + size_t num = ctx->num; + + if (num) { + ctx->data[num++] = 1; + while (num < POLY1305_BLOCK_SIZE) + ctx->data[num++] = 0; + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, + simd_context); + } + + poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context); + + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL(poly1305_final); + +#include "../selftest/poly1305.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + poly1305_fpu_init(); + if (!selftest_run("poly1305", poly1305_selftest, poly1305_nobs, + ARRAY_SIZE(poly1305_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Poly1305 one-time authenticator"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/selftest/blake2s.c b/net/wireguard/crypto/zinc/selftest/blake2s.c new file mode 100644 index 0000000000000..1b5c210dc7a84 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/blake2s.c @@ -0,0 +1,2090 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x69, 0x21, 0x7a, 0x30, 0x79, 0x90, 0x80, 0x94, + 0xe1, 0x11, 0x21, 0xd0, 0x42, 0x35, 0x4a, 0x7c, + 0x1f, 0x55, 0xb6, 0x48, 0x2c, 0xa1, 0xa5, 0x1e, + 0x1b, 0x25, 0x0d, 0xfd, 0x1e, 0xd0, 0xee, 0xf9 }, + { 0xe3, 0x4d, 0x74, 0xdb, 0xaf, 0x4f, 0xf4, 0xc6, + 0xab, 0xd8, 0x71, 0xcc, 0x22, 0x04, 0x51, 0xd2, + 0xea, 0x26, 0x48, 0x84, 0x6c, 0x77, 0x57, 0xfb, + 0xaa, 0xc8, 0x2f, 0xe5, 0x1a, 0xd6, 0x4b, 0xea }, + { 0xdd, 0xad, 0x9a, 0xb1, 0x5d, 0xac, 0x45, 0x49, + 0xba, 0x42, 0xf4, 0x9d, 0x26, 0x24, 0x96, 0xbe, + 0xf6, 0xc0, 0xba, 0xe1, 0xdd, 0x34, 0x2a, 0x88, + 0x08, 0xf8, 0xea, 0x26, 0x7c, 0x6e, 0x21, 0x0c }, + { 0xe8, 0xf9, 0x1c, 0x6e, 0xf2, 0x32, 0xa0, 0x41, + 0x45, 0x2a, 0xb0, 0xe1, 0x49, 0x07, 0x0c, 0xdd, + 0x7d, 0xd1, 0x76, 0x9e, 0x75, 0xb3, 0xa5, 0x92, + 0x1b, 0xe3, 0x78, 0x76, 0xc4, 0x5c, 0x99, 0x00 }, + { 0x0c, 0xc7, 0x0e, 0x00, 0x34, 0x8b, 0x86, 0xba, + 0x29, 0x44, 0xd0, 0xc3, 0x20, 0x38, 0xb2, 0x5c, + 0x55, 0x58, 0x4f, 0x90, 0xdf, 0x23, 0x04, 0xf5, + 0x5f, 0xa3, 0x32, 0xaf, 0x5f, 0xb0, 0x1e, 0x20 }, + { 0xec, 0x19, 0x64, 0x19, 0x10, 0x87, 0xa4, 0xfe, + 0x9d, 0xf1, 0xc7, 0x95, 0x34, 0x2a, 0x02, 0xff, + 0xc1, 0x91, 0xa5, 0xb2, 0x51, 0x76, 0x48, 0x56, + 0xae, 0x5b, 0x8b, 0x57, 0x69, 0xf0, 0xc6, 0xcd }, + { 0xe1, 0xfa, 0x51, 0x61, 0x8d, 0x7d, 0xf4, 0xeb, + 0x70, 0xcf, 0x0d, 0x5a, 0x9e, 0x90, 0x6f, 0x80, + 0x6e, 0x9d, 0x19, 0xf7, 0xf4, 0xf0, 0x1e, 0x3b, + 0x62, 0x12, 0x88, 0xe4, 0x12, 0x04, 0x05, 0xd6 }, + { 0x59, 0x80, 0x01, 0xfa, 0xfb, 0xe8, 0xf9, 0x4e, + 0xc6, 0x6d, 0xc8, 0x27, 0xd0, 0x12, 0xcf, 0xcb, + 0xba, 0x22, 0x28, 0x56, 0x9f, 0x44, 0x8e, 0x89, + 0xea, 0x22, 0x08, 0xc8, 0xbf, 0x76, 0x92, 0x93 }, + { 0xc7, 0xe8, 0x87, 0xb5, 0x46, 0x62, 0x36, 0x35, + 0xe9, 0x3e, 0x04, 0x95, 0x59, 0x8f, 0x17, 0x26, + 0x82, 0x19, 0x96, 0xc2, 0x37, 0x77, 0x05, 0xb9, + 0x3a, 0x1f, 0x63, 0x6f, 0x87, 0x2b, 0xfa, 0x2d }, + { 0xc3, 0x15, 0xa4, 0x37, 0xdd, 0x28, 0x06, 0x2a, + 0x77, 0x0d, 0x48, 0x19, 0x67, 0x13, 0x6b, 0x1b, + 0x5e, 0xb8, 0x8b, 0x21, 0xee, 0x53, 0xd0, 0x32, + 0x9c, 0x58, 0x97, 0x12, 0x6e, 0x9d, 0xb0, 0x2c }, + { 0xbb, 0x47, 0x3d, 0xed, 0xdc, 0x05, 0x5f, 0xea, + 0x62, 0x28, 0xf2, 0x07, 0xda, 0x57, 0x53, 0x47, + 0xbb, 0x00, 0x40, 0x4c, 0xd3, 0x49, 0xd3, 0x8c, + 0x18, 0x02, 0x63, 0x07, 0xa2, 0x24, 0xcb, 0xff }, + { 0x68, 0x7e, 0x18, 0x73, 0xa8, 0x27, 0x75, 0x91, + 0xbb, 0x33, 0xd9, 0xad, 0xf9, 0xa1, 0x39, 0x12, + 0xef, 0xef, 0xe5, 0x57, 0xca, 0xfc, 0x39, 0xa7, + 0x95, 0x26, 0x23, 0xe4, 0x72, 0x55, 0xf1, 0x6d }, + { 0x1a, 0xc7, 0xba, 0x75, 0x4d, 0x6e, 0x2f, 0x94, + 0xe0, 0xe8, 0x6c, 0x46, 0xbf, 0xb2, 0x62, 0xab, + 0xbb, 0x74, 0xf4, 0x50, 0xef, 0x45, 0x6d, 0x6b, + 0x4d, 0x97, 0xaa, 0x80, 0xce, 0x6d, 0xa7, 0x67 }, + { 0x01, 0x2c, 0x97, 0x80, 0x96, 0x14, 0x81, 0x6b, + 0x5d, 0x94, 0x94, 0x47, 0x7d, 0x4b, 0x68, 0x7d, + 0x15, 0xb9, 0x6e, 0xb6, 0x9c, 0x0e, 0x80, 0x74, + 0xa8, 0x51, 0x6f, 0x31, 0x22, 0x4b, 0x5c, 0x98 }, + { 0x91, 0xff, 0xd2, 0x6c, 0xfa, 0x4d, 0xa5, 0x13, + 0x4c, 0x7e, 0xa2, 0x62, 0xf7, 0x88, 0x9c, 0x32, + 0x9f, 0x61, 0xf6, 0xa6, 0x57, 0x22, 0x5c, 0xc2, + 0x12, 0xf4, 0x00, 0x56, 0xd9, 0x86, 0xb3, 0xf4 }, + { 0xd9, 0x7c, 0x82, 0x8d, 0x81, 0x82, 0xa7, 0x21, + 0x80, 0xa0, 0x6a, 0x78, 0x26, 0x83, 0x30, 0x67, + 0x3f, 0x7c, 0x4e, 0x06, 0x35, 0x94, 0x7c, 0x04, + 0xc0, 0x23, 0x23, 0xfd, 0x45, 0xc0, 0xa5, 0x2d }, + { 0xef, 0xc0, 0x4c, 0xdc, 0x39, 0x1c, 0x7e, 0x91, + 0x19, 0xbd, 0x38, 0x66, 0x8a, 0x53, 0x4e, 0x65, + 0xfe, 0x31, 0x03, 0x6d, 0x6a, 0x62, 0x11, 0x2e, + 0x44, 0xeb, 0xeb, 0x11, 0xf9, 0xc5, 0x70, 0x80 }, + { 0x99, 0x2c, 0xf5, 0xc0, 0x53, 0x44, 0x2a, 0x5f, + 0xbc, 0x4f, 0xaf, 0x58, 0x3e, 0x04, 0xe5, 0x0b, + 0xb7, 0x0d, 0x2f, 0x39, 0xfb, 0xb6, 0xa5, 0x03, + 0xf8, 0x9e, 0x56, 0xa6, 0x3e, 0x18, 0x57, 0x8a }, + { 0x38, 0x64, 0x0e, 0x9f, 0x21, 0x98, 0x3e, 0x67, + 0xb5, 0x39, 0xca, 0xcc, 0xae, 0x5e, 0xcf, 0x61, + 0x5a, 0xe2, 0x76, 0x4f, 0x75, 0xa0, 0x9c, 0x9c, + 0x59, 0xb7, 0x64, 0x83, 0xc1, 0xfb, 0xc7, 0x35 }, + { 0x21, 0x3d, 0xd3, 0x4c, 0x7e, 0xfe, 0x4f, 0xb2, + 0x7a, 0x6b, 0x35, 0xf6, 0xb4, 0x00, 0x0d, 0x1f, + 0xe0, 0x32, 0x81, 0xaf, 0x3c, 0x72, 0x3e, 0x5c, + 0x9f, 0x94, 0x74, 0x7a, 0x5f, 0x31, 0xcd, 0x3b }, + { 0xec, 0x24, 0x6e, 0xee, 0xb9, 0xce, 0xd3, 0xf7, + 0xad, 0x33, 0xed, 0x28, 0x66, 0x0d, 0xd9, 0xbb, + 0x07, 0x32, 0x51, 0x3d, 0xb4, 0xe2, 0xfa, 0x27, + 0x8b, 0x60, 0xcd, 0xe3, 0x68, 0x2a, 0x4c, 0xcd }, + { 0xac, 0x9b, 0x61, 0xd4, 0x46, 0x64, 0x8c, 0x30, + 0x05, 0xd7, 0x89, 0x2b, 0xf3, 0xa8, 0x71, 0x9f, + 0x4c, 0x81, 0x81, 0xcf, 0xdc, 0xbc, 0x2b, 0x79, + 0xfe, 0xf1, 0x0a, 0x27, 0x9b, 0x91, 0x10, 0x95 }, + { 0x7b, 0xf8, 0xb2, 0x29, 0x59, 0xe3, 0x4e, 0x3a, + 0x43, 0xf7, 0x07, 0x92, 0x23, 0xe8, 0x3a, 0x97, + 0x54, 0x61, 0x7d, 0x39, 0x1e, 0x21, 0x3d, 0xfd, + 0x80, 0x8e, 0x41, 0xb9, 0xbe, 0xad, 0x4c, 0xe7 }, + { 0x68, 0xd4, 0xb5, 0xd4, 0xfa, 0x0e, 0x30, 0x2b, + 0x64, 0xcc, 0xc5, 0xaf, 0x79, 0x29, 0x13, 0xac, + 0x4c, 0x88, 0xec, 0x95, 0xc0, 0x7d, 0xdf, 0x40, + 0x69, 0x42, 0x56, 0xeb, 0x88, 0xce, 0x9f, 0x3d }, + { 0xb2, 0xc2, 0x42, 0x0f, 0x05, 0xf9, 0xab, 0xe3, + 0x63, 0x15, 0x91, 0x93, 0x36, 0xb3, 0x7e, 0x4e, + 0x0f, 0xa3, 0x3f, 0xf7, 0xe7, 0x6a, 0x49, 0x27, + 0x67, 0x00, 0x6f, 0xdb, 0x5d, 0x93, 0x54, 0x62 }, + { 0x13, 0x4f, 0x61, 0xbb, 0xd0, 0xbb, 0xb6, 0x9a, + 0xed, 0x53, 0x43, 0x90, 0x45, 0x51, 0xa3, 0xe6, + 0xc1, 0xaa, 0x7d, 0xcd, 0xd7, 0x7e, 0x90, 0x3e, + 0x70, 0x23, 0xeb, 0x7c, 0x60, 0x32, 0x0a, 0xa7 }, + { 0x46, 0x93, 0xf9, 0xbf, 0xf7, 0xd4, 0xf3, 0x98, + 0x6a, 0x7d, 0x17, 0x6e, 0x6e, 0x06, 0xf7, 0x2a, + 0xd1, 0x49, 0x0d, 0x80, 0x5c, 0x99, 0xe2, 0x53, + 0x47, 0xb8, 0xde, 0x77, 0xb4, 0xdb, 0x6d, 0x9b }, + { 0x85, 0x3e, 0x26, 0xf7, 0x41, 0x95, 0x3b, 0x0f, + 0xd5, 0xbd, 0xb4, 0x24, 0xe8, 0xab, 0x9e, 0x8b, + 0x37, 0x50, 0xea, 0xa8, 0xef, 0x61, 0xe4, 0x79, + 0x02, 0xc9, 0x1e, 0x55, 0x4e, 0x9c, 0x73, 0xb9 }, + { 0xf7, 0xde, 0x53, 0x63, 0x61, 0xab, 0xaa, 0x0e, + 0x15, 0x81, 0x56, 0xcf, 0x0e, 0xa4, 0xf6, 0x3a, + 0x99, 0xb5, 0xe4, 0x05, 0x4f, 0x8f, 0xa4, 0xc9, + 0xd4, 0x5f, 0x62, 0x85, 0xca, 0xd5, 0x56, 0x94 }, + { 0x4c, 0x23, 0x06, 0x08, 0x86, 0x0a, 0x99, 0xae, + 0x8d, 0x7b, 0xd5, 0xc2, 0xcc, 0x17, 0xfa, 0x52, + 0x09, 0x6b, 0x9a, 0x61, 0xbe, 0xdb, 0x17, 0xcb, + 0x76, 0x17, 0x86, 0x4a, 0xd2, 0x9c, 0xa7, 0xa6 }, + { 0xae, 0xb9, 0x20, 0xea, 0x87, 0x95, 0x2d, 0xad, + 0xb1, 0xfb, 0x75, 0x92, 0x91, 0xe3, 0x38, 0x81, + 0x39, 0xa8, 0x72, 0x86, 0x50, 0x01, 0x88, 0x6e, + 0xd8, 0x47, 0x52, 0xe9, 0x3c, 0x25, 0x0c, 0x2a }, + { 0xab, 0xa4, 0xad, 0x9b, 0x48, 0x0b, 0x9d, 0xf3, + 0xd0, 0x8c, 0xa5, 0xe8, 0x7b, 0x0c, 0x24, 0x40, + 0xd4, 0xe4, 0xea, 0x21, 0x22, 0x4c, 0x2e, 0xb4, + 0x2c, 0xba, 0xe4, 0x69, 0xd0, 0x89, 0xb9, 0x31 }, + { 0x05, 0x82, 0x56, 0x07, 0xd7, 0xfd, 0xf2, 0xd8, + 0x2e, 0xf4, 0xc3, 0xc8, 0xc2, 0xae, 0xa9, 0x61, + 0xad, 0x98, 0xd6, 0x0e, 0xdf, 0xf7, 0xd0, 0x18, + 0x98, 0x3e, 0x21, 0x20, 0x4c, 0x0d, 0x93, 0xd1 }, + { 0xa7, 0x42, 0xf8, 0xb6, 0xaf, 0x82, 0xd8, 0xa6, + 0xca, 0x23, 0x57, 0xc5, 0xf1, 0xcf, 0x91, 0xde, + 0xfb, 0xd0, 0x66, 0x26, 0x7d, 0x75, 0xc0, 0x48, + 0xb3, 0x52, 0x36, 0x65, 0x85, 0x02, 0x59, 0x62 }, + { 0x2b, 0xca, 0xc8, 0x95, 0x99, 0x00, 0x0b, 0x42, + 0xc9, 0x5a, 0xe2, 0x38, 0x35, 0xa7, 0x13, 0x70, + 0x4e, 0xd7, 0x97, 0x89, 0xc8, 0x4f, 0xef, 0x14, + 0x9a, 0x87, 0x4f, 0xf7, 0x33, 0xf0, 0x17, 0xa2 }, + { 0xac, 0x1e, 0xd0, 0x7d, 0x04, 0x8f, 0x10, 0x5a, + 0x9e, 0x5b, 0x7a, 0xb8, 0x5b, 0x09, 0xa4, 0x92, + 0xd5, 0xba, 0xff, 0x14, 0xb8, 0xbf, 0xb0, 0xe9, + 0xfd, 0x78, 0x94, 0x86, 0xee, 0xa2, 0xb9, 0x74 }, + { 0xe4, 0x8d, 0x0e, 0xcf, 0xaf, 0x49, 0x7d, 0x5b, + 0x27, 0xc2, 0x5d, 0x99, 0xe1, 0x56, 0xcb, 0x05, + 0x79, 0xd4, 0x40, 0xd6, 0xe3, 0x1f, 0xb6, 0x24, + 0x73, 0x69, 0x6d, 0xbf, 0x95, 0xe0, 0x10, 0xe4 }, + { 0x12, 0xa9, 0x1f, 0xad, 0xf8, 0xb2, 0x16, 0x44, + 0xfd, 0x0f, 0x93, 0x4f, 0x3c, 0x4a, 0x8f, 0x62, + 0xba, 0x86, 0x2f, 0xfd, 0x20, 0xe8, 0xe9, 0x61, + 0x15, 0x4c, 0x15, 0xc1, 0x38, 0x84, 0xed, 0x3d }, + { 0x7c, 0xbe, 0xe9, 0x6e, 0x13, 0x98, 0x97, 0xdc, + 0x98, 0xfb, 0xef, 0x3b, 0xe8, 0x1a, 0xd4, 0xd9, + 0x64, 0xd2, 0x35, 0xcb, 0x12, 0x14, 0x1f, 0xb6, + 0x67, 0x27, 0xe6, 0xe5, 0xdf, 0x73, 0xa8, 0x78 }, + { 0xeb, 0xf6, 0x6a, 0xbb, 0x59, 0x7a, 0xe5, 0x72, + 0xa7, 0x29, 0x7c, 0xb0, 0x87, 0x1e, 0x35, 0x5a, + 0xcc, 0xaf, 0xad, 0x83, 0x77, 0xb8, 0xe7, 0x8b, + 0xf1, 0x64, 0xce, 0x2a, 0x18, 0xde, 0x4b, 0xaf }, + { 0x71, 0xb9, 0x33, 0xb0, 0x7e, 0x4f, 0xf7, 0x81, + 0x8c, 0xe0, 0x59, 0xd0, 0x08, 0x82, 0x9e, 0x45, + 0x3c, 0x6f, 0xf0, 0x2e, 0xc0, 0xa7, 0xdb, 0x39, + 0x3f, 0xc2, 0xd8, 0x70, 0xf3, 0x7a, 0x72, 0x86 }, + { 0x7c, 0xf7, 0xc5, 0x13, 0x31, 0x22, 0x0b, 0x8d, + 0x3e, 0xba, 0xed, 0x9c, 0x29, 0x39, 0x8a, 0x16, + 0xd9, 0x81, 0x56, 0xe2, 0x61, 0x3c, 0xb0, 0x88, + 0xf2, 0xb0, 0xe0, 0x8a, 0x1b, 0xe4, 0xcf, 0x4f }, + { 0x3e, 0x41, 0xa1, 0x08, 0xe0, 0xf6, 0x4a, 0xd2, + 0x76, 0xb9, 0x79, 0xe1, 0xce, 0x06, 0x82, 0x79, + 0xe1, 0x6f, 0x7b, 0xc7, 0xe4, 0xaa, 0x1d, 0x21, + 0x1e, 0x17, 0xb8, 0x11, 0x61, 0xdf, 0x16, 0x02 }, + { 0x88, 0x65, 0x02, 0xa8, 0x2a, 0xb4, 0x7b, 0xa8, + 0xd8, 0x67, 0x10, 0xaa, 0x9d, 0xe3, 0xd4, 0x6e, + 0xa6, 0x5c, 0x47, 0xaf, 0x6e, 0xe8, 0xde, 0x45, + 0x0c, 0xce, 0xb8, 0xb1, 0x1b, 0x04, 0x5f, 0x50 }, + { 0xc0, 0x21, 0xbc, 0x5f, 0x09, 0x54, 0xfe, 0xe9, + 0x4f, 0x46, 0xea, 0x09, 0x48, 0x7e, 0x10, 0xa8, + 0x48, 0x40, 0xd0, 0x2f, 0x64, 0x81, 0x0b, 0xc0, + 0x8d, 0x9e, 0x55, 0x1f, 0x7d, 0x41, 0x68, 0x14 }, + { 0x20, 0x30, 0x51, 0x6e, 0x8a, 0x5f, 0xe1, 0x9a, + 0xe7, 0x9c, 0x33, 0x6f, 0xce, 0x26, 0x38, 0x2a, + 0x74, 0x9d, 0x3f, 0xd0, 0xec, 0x91, 0xe5, 0x37, + 0xd4, 0xbd, 0x23, 0x58, 0xc1, 0x2d, 0xfb, 0x22 }, + { 0x55, 0x66, 0x98, 0xda, 0xc8, 0x31, 0x7f, 0xd3, + 0x6d, 0xfb, 0xdf, 0x25, 0xa7, 0x9c, 0xb1, 0x12, + 0xd5, 0x42, 0x58, 0x60, 0x60, 0x5c, 0xba, 0xf5, + 0x07, 0xf2, 0x3b, 0xf7, 0xe9, 0xf4, 0x2a, 0xfe }, + { 0x2f, 0x86, 0x7b, 0xa6, 0x77, 0x73, 0xfd, 0xc3, + 0xe9, 0x2f, 0xce, 0xd9, 0x9a, 0x64, 0x09, 0xad, + 0x39, 0xd0, 0xb8, 0x80, 0xfd, 0xe8, 0xf1, 0x09, + 0xa8, 0x17, 0x30, 0xc4, 0x45, 0x1d, 0x01, 0x78 }, + { 0x17, 0x2e, 0xc2, 0x18, 0xf1, 0x19, 0xdf, 0xae, + 0x98, 0x89, 0x6d, 0xff, 0x29, 0xdd, 0x98, 0x76, + 0xc9, 0x4a, 0xf8, 0x74, 0x17, 0xf9, 0xae, 0x4c, + 0x70, 0x14, 0xbb, 0x4e, 0x4b, 0x96, 0xaf, 0xc7 }, + { 0x3f, 0x85, 0x81, 0x4a, 0x18, 0x19, 0x5f, 0x87, + 0x9a, 0xa9, 0x62, 0xf9, 0x5d, 0x26, 0xbd, 0x82, + 0xa2, 0x78, 0xf2, 0xb8, 0x23, 0x20, 0x21, 0x8f, + 0x6b, 0x3b, 0xd6, 0xf7, 0xf6, 0x67, 0xa6, 0xd9 }, + { 0x1b, 0x61, 0x8f, 0xba, 0xa5, 0x66, 0xb3, 0xd4, + 0x98, 0xc1, 0x2e, 0x98, 0x2c, 0x9e, 0xc5, 0x2e, + 0x4d, 0xa8, 0x5a, 0x8c, 0x54, 0xf3, 0x8f, 0x34, + 0xc0, 0x90, 0x39, 0x4f, 0x23, 0xc1, 0x84, 0xc1 }, + { 0x0c, 0x75, 0x8f, 0xb5, 0x69, 0x2f, 0xfd, 0x41, + 0xa3, 0x57, 0x5d, 0x0a, 0xf0, 0x0c, 0xc7, 0xfb, + 0xf2, 0xcb, 0xe5, 0x90, 0x5a, 0x58, 0x32, 0x3a, + 0x88, 0xae, 0x42, 0x44, 0xf6, 0xe4, 0xc9, 0x93 }, + { 0xa9, 0x31, 0x36, 0x0c, 0xad, 0x62, 0x8c, 0x7f, + 0x12, 0xa6, 0xc1, 0xc4, 0xb7, 0x53, 0xb0, 0xf4, + 0x06, 0x2a, 0xef, 0x3c, 0xe6, 0x5a, 0x1a, 0xe3, + 0xf1, 0x93, 0x69, 0xda, 0xdf, 0x3a, 0xe2, 0x3d }, + { 0xcb, 0xac, 0x7d, 0x77, 0x3b, 0x1e, 0x3b, 0x3c, + 0x66, 0x91, 0xd7, 0xab, 0xb7, 0xe9, 0xdf, 0x04, + 0x5c, 0x8b, 0xa1, 0x92, 0x68, 0xde, 0xd1, 0x53, + 0x20, 0x7f, 0x5e, 0x80, 0x43, 0x52, 0xec, 0x5d }, + { 0x23, 0xa1, 0x96, 0xd3, 0x80, 0x2e, 0xd3, 0xc1, + 0xb3, 0x84, 0x01, 0x9a, 0x82, 0x32, 0x58, 0x40, + 0xd3, 0x2f, 0x71, 0x95, 0x0c, 0x45, 0x80, 0xb0, + 0x34, 0x45, 0xe0, 0x89, 0x8e, 0x14, 0x05, 0x3c }, + { 0xf4, 0x49, 0x54, 0x70, 0xf2, 0x26, 0xc8, 0xc2, + 0x14, 0xbe, 0x08, 0xfd, 0xfa, 0xd4, 0xbc, 0x4a, + 0x2a, 0x9d, 0xbe, 0xa9, 0x13, 0x6a, 0x21, 0x0d, + 0xf0, 0xd4, 0xb6, 0x49, 0x29, 0xe6, 0xfc, 0x14 }, + { 0xe2, 0x90, 0xdd, 0x27, 0x0b, 0x46, 0x7f, 0x34, + 0xab, 0x1c, 0x00, 0x2d, 0x34, 0x0f, 0xa0, 0x16, + 0x25, 0x7f, 0xf1, 0x9e, 0x58, 0x33, 0xfd, 0xbb, + 0xf2, 0xcb, 0x40, 0x1c, 0x3b, 0x28, 0x17, 0xde }, + { 0x9f, 0xc7, 0xb5, 0xde, 0xd3, 0xc1, 0x50, 0x42, + 0xb2, 0xa6, 0x58, 0x2d, 0xc3, 0x9b, 0xe0, 0x16, + 0xd2, 0x4a, 0x68, 0x2d, 0x5e, 0x61, 0xad, 0x1e, + 0xff, 0x9c, 0x63, 0x30, 0x98, 0x48, 0xf7, 0x06 }, + { 0x8c, 0xca, 0x67, 0xa3, 0x6d, 0x17, 0xd5, 0xe6, + 0x34, 0x1c, 0xb5, 0x92, 0xfd, 0x7b, 0xef, 0x99, + 0x26, 0xc9, 0xe3, 0xaa, 0x10, 0x27, 0xea, 0x11, + 0xa7, 0xd8, 0xbd, 0x26, 0x0b, 0x57, 0x6e, 0x04 }, + { 0x40, 0x93, 0x92, 0xf5, 0x60, 0xf8, 0x68, 0x31, + 0xda, 0x43, 0x73, 0xee, 0x5e, 0x00, 0x74, 0x26, + 0x05, 0x95, 0xd7, 0xbc, 0x24, 0x18, 0x3b, 0x60, + 0xed, 0x70, 0x0d, 0x45, 0x83, 0xd3, 0xf6, 0xf0 }, + { 0x28, 0x02, 0x16, 0x5d, 0xe0, 0x90, 0x91, 0x55, + 0x46, 0xf3, 0x39, 0x8c, 0xd8, 0x49, 0x16, 0x4a, + 0x19, 0xf9, 0x2a, 0xdb, 0xc3, 0x61, 0xad, 0xc9, + 0x9b, 0x0f, 0x20, 0xc8, 0xea, 0x07, 0x10, 0x54 }, + { 0xad, 0x83, 0x91, 0x68, 0xd9, 0xf8, 0xa4, 0xbe, + 0x95, 0xba, 0x9e, 0xf9, 0xa6, 0x92, 0xf0, 0x72, + 0x56, 0xae, 0x43, 0xfe, 0x6f, 0x98, 0x64, 0xe2, + 0x90, 0x69, 0x1b, 0x02, 0x56, 0xce, 0x50, 0xa9 }, + { 0x75, 0xfd, 0xaa, 0x50, 0x38, 0xc2, 0x84, 0xb8, + 0x6d, 0x6e, 0x8a, 0xff, 0xe8, 0xb2, 0x80, 0x7e, + 0x46, 0x7b, 0x86, 0x60, 0x0e, 0x79, 0xaf, 0x36, + 0x89, 0xfb, 0xc0, 0x63, 0x28, 0xcb, 0xf8, 0x94 }, + { 0xe5, 0x7c, 0xb7, 0x94, 0x87, 0xdd, 0x57, 0x90, + 0x24, 0x32, 0xb2, 0x50, 0x73, 0x38, 0x13, 0xbd, + 0x96, 0xa8, 0x4e, 0xfc, 0xe5, 0x9f, 0x65, 0x0f, + 0xac, 0x26, 0xe6, 0x69, 0x6a, 0xef, 0xaf, 0xc3 }, + { 0x56, 0xf3, 0x4e, 0x8b, 0x96, 0x55, 0x7e, 0x90, + 0xc1, 0xf2, 0x4b, 0x52, 0xd0, 0xc8, 0x9d, 0x51, + 0x08, 0x6a, 0xcf, 0x1b, 0x00, 0xf6, 0x34, 0xcf, + 0x1d, 0xde, 0x92, 0x33, 0xb8, 0xea, 0xaa, 0x3e }, + { 0x1b, 0x53, 0xee, 0x94, 0xaa, 0xf3, 0x4e, 0x4b, + 0x15, 0x9d, 0x48, 0xde, 0x35, 0x2c, 0x7f, 0x06, + 0x61, 0xd0, 0xa4, 0x0e, 0xdf, 0xf9, 0x5a, 0x0b, + 0x16, 0x39, 0xb4, 0x09, 0x0e, 0x97, 0x44, 0x72 }, + { 0x05, 0x70, 0x5e, 0x2a, 0x81, 0x75, 0x7c, 0x14, + 0xbd, 0x38, 0x3e, 0xa9, 0x8d, 0xda, 0x54, 0x4e, + 0xb1, 0x0e, 0x6b, 0xc0, 0x7b, 0xae, 0x43, 0x5e, + 0x25, 0x18, 0xdb, 0xe1, 0x33, 0x52, 0x53, 0x75 }, + { 0xd8, 0xb2, 0x86, 0x6e, 0x8a, 0x30, 0x9d, 0xb5, + 0x3e, 0x52, 0x9e, 0xc3, 0x29, 0x11, 0xd8, 0x2f, + 0x5c, 0xa1, 0x6c, 0xff, 0x76, 0x21, 0x68, 0x91, + 0xa9, 0x67, 0x6a, 0xa3, 0x1a, 0xaa, 0x6c, 0x42 }, + { 0xf5, 0x04, 0x1c, 0x24, 0x12, 0x70, 0xeb, 0x04, + 0xc7, 0x1e, 0xc2, 0xc9, 0x5d, 0x4c, 0x38, 0xd8, + 0x03, 0xb1, 0x23, 0x7b, 0x0f, 0x29, 0xfd, 0x4d, + 0xb3, 0xeb, 0x39, 0x76, 0x69, 0xe8, 0x86, 0x99 }, + { 0x9a, 0x4c, 0xe0, 0x77, 0xc3, 0x49, 0x32, 0x2f, + 0x59, 0x5e, 0x0e, 0xe7, 0x9e, 0xd0, 0xda, 0x5f, + 0xab, 0x66, 0x75, 0x2c, 0xbf, 0xef, 0x8f, 0x87, + 0xd0, 0xe9, 0xd0, 0x72, 0x3c, 0x75, 0x30, 0xdd }, + { 0x65, 0x7b, 0x09, 0xf3, 0xd0, 0xf5, 0x2b, 0x5b, + 0x8f, 0x2f, 0x97, 0x16, 0x3a, 0x0e, 0xdf, 0x0c, + 0x04, 0xf0, 0x75, 0x40, 0x8a, 0x07, 0xbb, 0xeb, + 0x3a, 0x41, 0x01, 0xa8, 0x91, 0x99, 0x0d, 0x62 }, + { 0x1e, 0x3f, 0x7b, 0xd5, 0xa5, 0x8f, 0xa5, 0x33, + 0x34, 0x4a, 0xa8, 0xed, 0x3a, 0xc1, 0x22, 0xbb, + 0x9e, 0x70, 0xd4, 0xef, 0x50, 0xd0, 0x04, 0x53, + 0x08, 0x21, 0x94, 0x8f, 0x5f, 0xe6, 0x31, 0x5a }, + { 0x80, 0xdc, 0xcf, 0x3f, 0xd8, 0x3d, 0xfd, 0x0d, + 0x35, 0xaa, 0x28, 0x58, 0x59, 0x22, 0xab, 0x89, + 0xd5, 0x31, 0x39, 0x97, 0x67, 0x3e, 0xaf, 0x90, + 0x5c, 0xea, 0x9c, 0x0b, 0x22, 0x5c, 0x7b, 0x5f }, + { 0x8a, 0x0d, 0x0f, 0xbf, 0x63, 0x77, 0xd8, 0x3b, + 0xb0, 0x8b, 0x51, 0x4b, 0x4b, 0x1c, 0x43, 0xac, + 0xc9, 0x5d, 0x75, 0x17, 0x14, 0xf8, 0x92, 0x56, + 0x45, 0xcb, 0x6b, 0xc8, 0x56, 0xca, 0x15, 0x0a }, + { 0x9f, 0xa5, 0xb4, 0x87, 0x73, 0x8a, 0xd2, 0x84, + 0x4c, 0xc6, 0x34, 0x8a, 0x90, 0x19, 0x18, 0xf6, + 0x59, 0xa3, 0xb8, 0x9e, 0x9c, 0x0d, 0xfe, 0xea, + 0xd3, 0x0d, 0xd9, 0x4b, 0xcf, 0x42, 0xef, 0x8e }, + { 0x80, 0x83, 0x2c, 0x4a, 0x16, 0x77, 0xf5, 0xea, + 0x25, 0x60, 0xf6, 0x68, 0xe9, 0x35, 0x4d, 0xd3, + 0x69, 0x97, 0xf0, 0x37, 0x28, 0xcf, 0xa5, 0x5e, + 0x1b, 0x38, 0x33, 0x7c, 0x0c, 0x9e, 0xf8, 0x18 }, + { 0xab, 0x37, 0xdd, 0xb6, 0x83, 0x13, 0x7e, 0x74, + 0x08, 0x0d, 0x02, 0x6b, 0x59, 0x0b, 0x96, 0xae, + 0x9b, 0xb4, 0x47, 0x72, 0x2f, 0x30, 0x5a, 0x5a, + 0xc5, 0x70, 0xec, 0x1d, 0xf9, 0xb1, 0x74, 0x3c }, + { 0x3e, 0xe7, 0x35, 0xa6, 0x94, 0xc2, 0x55, 0x9b, + 0x69, 0x3a, 0xa6, 0x86, 0x29, 0x36, 0x1e, 0x15, + 0xd1, 0x22, 0x65, 0xad, 0x6a, 0x3d, 0xed, 0xf4, + 0x88, 0xb0, 0xb0, 0x0f, 0xac, 0x97, 0x54, 0xba }, + { 0xd6, 0xfc, 0xd2, 0x32, 0x19, 0xb6, 0x47, 0xe4, + 0xcb, 0xd5, 0xeb, 0x2d, 0x0a, 0xd0, 0x1e, 0xc8, + 0x83, 0x8a, 0x4b, 0x29, 0x01, 0xfc, 0x32, 0x5c, + 0xc3, 0x70, 0x19, 0x81, 0xca, 0x6c, 0x88, 0x8b }, + { 0x05, 0x20, 0xec, 0x2f, 0x5b, 0xf7, 0xa7, 0x55, + 0xda, 0xcb, 0x50, 0xc6, 0xbf, 0x23, 0x3e, 0x35, + 0x15, 0x43, 0x47, 0x63, 0xdb, 0x01, 0x39, 0xcc, + 0xd9, 0xfa, 0xef, 0xbb, 0x82, 0x07, 0x61, 0x2d }, + { 0xaf, 0xf3, 0xb7, 0x5f, 0x3f, 0x58, 0x12, 0x64, + 0xd7, 0x66, 0x16, 0x62, 0xb9, 0x2f, 0x5a, 0xd3, + 0x7c, 0x1d, 0x32, 0xbd, 0x45, 0xff, 0x81, 0xa4, + 0xed, 0x8a, 0xdc, 0x9e, 0xf3, 0x0d, 0xd9, 0x89 }, + { 0xd0, 0xdd, 0x65, 0x0b, 0xef, 0xd3, 0xba, 0x63, + 0xdc, 0x25, 0x10, 0x2c, 0x62, 0x7c, 0x92, 0x1b, + 0x9c, 0xbe, 0xb0, 0xb1, 0x30, 0x68, 0x69, 0x35, + 0xb5, 0xc9, 0x27, 0xcb, 0x7c, 0xcd, 0x5e, 0x3b }, + { 0xe1, 0x14, 0x98, 0x16, 0xb1, 0x0a, 0x85, 0x14, + 0xfb, 0x3e, 0x2c, 0xab, 0x2c, 0x08, 0xbe, 0xe9, + 0xf7, 0x3c, 0xe7, 0x62, 0x21, 0x70, 0x12, 0x46, + 0xa5, 0x89, 0xbb, 0xb6, 0x73, 0x02, 0xd8, 0xa9 }, + { 0x7d, 0xa3, 0xf4, 0x41, 0xde, 0x90, 0x54, 0x31, + 0x7e, 0x72, 0xb5, 0xdb, 0xf9, 0x79, 0xda, 0x01, + 0xe6, 0xbc, 0xee, 0xbb, 0x84, 0x78, 0xea, 0xe6, + 0xa2, 0x28, 0x49, 0xd9, 0x02, 0x92, 0x63, 0x5c }, + { 0x12, 0x30, 0xb1, 0xfc, 0x8a, 0x7d, 0x92, 0x15, + 0xed, 0xc2, 0xd4, 0xa2, 0xde, 0xcb, 0xdd, 0x0a, + 0x6e, 0x21, 0x6c, 0x92, 0x42, 0x78, 0xc9, 0x1f, + 0xc5, 0xd1, 0x0e, 0x7d, 0x60, 0x19, 0x2d, 0x94 }, + { 0x57, 0x50, 0xd7, 0x16, 0xb4, 0x80, 0x8f, 0x75, + 0x1f, 0xeb, 0xc3, 0x88, 0x06, 0xba, 0x17, 0x0b, + 0xf6, 0xd5, 0x19, 0x9a, 0x78, 0x16, 0xbe, 0x51, + 0x4e, 0x3f, 0x93, 0x2f, 0xbe, 0x0c, 0xb8, 0x71 }, + { 0x6f, 0xc5, 0x9b, 0x2f, 0x10, 0xfe, 0xba, 0x95, + 0x4a, 0xa6, 0x82, 0x0b, 0x3c, 0xa9, 0x87, 0xee, + 0x81, 0xd5, 0xcc, 0x1d, 0xa3, 0xc6, 0x3c, 0xe8, + 0x27, 0x30, 0x1c, 0x56, 0x9d, 0xfb, 0x39, 0xce }, + { 0xc7, 0xc3, 0xfe, 0x1e, 0xeb, 0xdc, 0x7b, 0x5a, + 0x93, 0x93, 0x26, 0xe8, 0xdd, 0xb8, 0x3e, 0x8b, + 0xf2, 0xb7, 0x80, 0xb6, 0x56, 0x78, 0xcb, 0x62, + 0xf2, 0x08, 0xb0, 0x40, 0xab, 0xdd, 0x35, 0xe2 }, + { 0x0c, 0x75, 0xc1, 0xa1, 0x5c, 0xf3, 0x4a, 0x31, + 0x4e, 0xe4, 0x78, 0xf4, 0xa5, 0xce, 0x0b, 0x8a, + 0x6b, 0x36, 0x52, 0x8e, 0xf7, 0xa8, 0x20, 0x69, + 0x6c, 0x3e, 0x42, 0x46, 0xc5, 0xa1, 0x58, 0x64 }, + { 0x21, 0x6d, 0xc1, 0x2a, 0x10, 0x85, 0x69, 0xa3, + 0xc7, 0xcd, 0xde, 0x4a, 0xed, 0x43, 0xa6, 0xc3, + 0x30, 0x13, 0x9d, 0xda, 0x3c, 0xcc, 0x4a, 0x10, + 0x89, 0x05, 0xdb, 0x38, 0x61, 0x89, 0x90, 0x50 }, + { 0xa5, 0x7b, 0xe6, 0xae, 0x67, 0x56, 0xf2, 0x8b, + 0x02, 0xf5, 0x9d, 0xad, 0xf7, 0xe0, 0xd7, 0xd8, + 0x80, 0x7f, 0x10, 0xfa, 0x15, 0xce, 0xd1, 0xad, + 0x35, 0x85, 0x52, 0x1a, 0x1d, 0x99, 0x5a, 0x89 }, + { 0x81, 0x6a, 0xef, 0x87, 0x59, 0x53, 0x71, 0x6c, + 0xd7, 0xa5, 0x81, 0xf7, 0x32, 0xf5, 0x3d, 0xd4, + 0x35, 0xda, 0xb6, 0x6d, 0x09, 0xc3, 0x61, 0xd2, + 0xd6, 0x59, 0x2d, 0xe1, 0x77, 0x55, 0xd8, 0xa8 }, + { 0x9a, 0x76, 0x89, 0x32, 0x26, 0x69, 0x3b, 0x6e, + 0xa9, 0x7e, 0x6a, 0x73, 0x8f, 0x9d, 0x10, 0xfb, + 0x3d, 0x0b, 0x43, 0xae, 0x0e, 0x8b, 0x7d, 0x81, + 0x23, 0xea, 0x76, 0xce, 0x97, 0x98, 0x9c, 0x7e }, + { 0x8d, 0xae, 0xdb, 0x9a, 0x27, 0x15, 0x29, 0xdb, + 0xb7, 0xdc, 0x3b, 0x60, 0x7f, 0xe5, 0xeb, 0x2d, + 0x32, 0x11, 0x77, 0x07, 0x58, 0xdd, 0x3b, 0x0a, + 0x35, 0x93, 0xd2, 0xd7, 0x95, 0x4e, 0x2d, 0x5b }, + { 0x16, 0xdb, 0xc0, 0xaa, 0x5d, 0xd2, 0xc7, 0x74, + 0xf5, 0x05, 0x10, 0x0f, 0x73, 0x37, 0x86, 0xd8, + 0xa1, 0x75, 0xfc, 0xbb, 0xb5, 0x9c, 0x43, 0xe1, + 0xfb, 0xff, 0x3e, 0x1e, 0xaf, 0x31, 0xcb, 0x4a }, + { 0x86, 0x06, 0xcb, 0x89, 0x9c, 0x6a, 0xea, 0xf5, + 0x1b, 0x9d, 0xb0, 0xfe, 0x49, 0x24, 0xa9, 0xfd, + 0x5d, 0xab, 0xc1, 0x9f, 0x88, 0x26, 0xf2, 0xbc, + 0x1c, 0x1d, 0x7d, 0xa1, 0x4d, 0x2c, 0x2c, 0x99 }, + { 0x84, 0x79, 0x73, 0x1a, 0xed, 0xa5, 0x7b, 0xd3, + 0x7e, 0xad, 0xb5, 0x1a, 0x50, 0x7e, 0x30, 0x7f, + 0x3b, 0xd9, 0x5e, 0x69, 0xdb, 0xca, 0x94, 0xf3, + 0xbc, 0x21, 0x72, 0x60, 0x66, 0xad, 0x6d, 0xfd }, + { 0x58, 0x47, 0x3a, 0x9e, 0xa8, 0x2e, 0xfa, 0x3f, + 0x3b, 0x3d, 0x8f, 0xc8, 0x3e, 0xd8, 0x86, 0x31, + 0x27, 0xb3, 0x3a, 0xe8, 0xde, 0xae, 0x63, 0x07, + 0x20, 0x1e, 0xdb, 0x6d, 0xde, 0x61, 0xde, 0x29 }, + { 0x9a, 0x92, 0x55, 0xd5, 0x3a, 0xf1, 0x16, 0xde, + 0x8b, 0xa2, 0x7c, 0xe3, 0x5b, 0x4c, 0x7e, 0x15, + 0x64, 0x06, 0x57, 0xa0, 0xfc, 0xb8, 0x88, 0xc7, + 0x0d, 0x95, 0x43, 0x1d, 0xac, 0xd8, 0xf8, 0x30 }, + { 0x9e, 0xb0, 0x5f, 0xfb, 0xa3, 0x9f, 0xd8, 0x59, + 0x6a, 0x45, 0x49, 0x3e, 0x18, 0xd2, 0x51, 0x0b, + 0xf3, 0xef, 0x06, 0x5c, 0x51, 0xd6, 0xe1, 0x3a, + 0xbe, 0x66, 0xaa, 0x57, 0xe0, 0x5c, 0xfd, 0xb7 }, + { 0x81, 0xdc, 0xc3, 0xa5, 0x05, 0xea, 0xce, 0x3f, + 0x87, 0x9d, 0x8f, 0x70, 0x27, 0x76, 0x77, 0x0f, + 0x9d, 0xf5, 0x0e, 0x52, 0x1d, 0x14, 0x28, 0xa8, + 0x5d, 0xaf, 0x04, 0xf9, 0xad, 0x21, 0x50, 0xe0 }, + { 0xe3, 0xe3, 0xc4, 0xaa, 0x3a, 0xcb, 0xbc, 0x85, + 0x33, 0x2a, 0xf9, 0xd5, 0x64, 0xbc, 0x24, 0x16, + 0x5e, 0x16, 0x87, 0xf6, 0xb1, 0xad, 0xcb, 0xfa, + 0xe7, 0x7a, 0x8f, 0x03, 0xc7, 0x2a, 0xc2, 0x8c }, + { 0x67, 0x46, 0xc8, 0x0b, 0x4e, 0xb5, 0x6a, 0xea, + 0x45, 0xe6, 0x4e, 0x72, 0x89, 0xbb, 0xa3, 0xed, + 0xbf, 0x45, 0xec, 0xf8, 0x20, 0x64, 0x81, 0xff, + 0x63, 0x02, 0x12, 0x29, 0x84, 0xcd, 0x52, 0x6a }, + { 0x2b, 0x62, 0x8e, 0x52, 0x76, 0x4d, 0x7d, 0x62, + 0xc0, 0x86, 0x8b, 0x21, 0x23, 0x57, 0xcd, 0xd1, + 0x2d, 0x91, 0x49, 0x82, 0x2f, 0x4e, 0x98, 0x45, + 0xd9, 0x18, 0xa0, 0x8d, 0x1a, 0xe9, 0x90, 0xc0 }, + { 0xe4, 0xbf, 0xe8, 0x0d, 0x58, 0xc9, 0x19, 0x94, + 0x61, 0x39, 0x09, 0xdc, 0x4b, 0x1a, 0x12, 0x49, + 0x68, 0x96, 0xc0, 0x04, 0xaf, 0x7b, 0x57, 0x01, + 0x48, 0x3d, 0xe4, 0x5d, 0x28, 0x23, 0xd7, 0x8e }, + { 0xeb, 0xb4, 0xba, 0x15, 0x0c, 0xef, 0x27, 0x34, + 0x34, 0x5b, 0x5d, 0x64, 0x1b, 0xbe, 0xd0, 0x3a, + 0x21, 0xea, 0xfa, 0xe9, 0x33, 0xc9, 0x9e, 0x00, + 0x92, 0x12, 0xef, 0x04, 0x57, 0x4a, 0x85, 0x30 }, + { 0x39, 0x66, 0xec, 0x73, 0xb1, 0x54, 0xac, 0xc6, + 0x97, 0xac, 0x5c, 0xf5, 0xb2, 0x4b, 0x40, 0xbd, + 0xb0, 0xdb, 0x9e, 0x39, 0x88, 0x36, 0xd7, 0x6d, + 0x4b, 0x88, 0x0e, 0x3b, 0x2a, 0xf1, 0xaa, 0x27 }, + { 0xef, 0x7e, 0x48, 0x31, 0xb3, 0xa8, 0x46, 0x36, + 0x51, 0x8d, 0x6e, 0x4b, 0xfc, 0xe6, 0x4a, 0x43, + 0xdb, 0x2a, 0x5d, 0xda, 0x9c, 0xca, 0x2b, 0x44, + 0xf3, 0x90, 0x33, 0xbd, 0xc4, 0x0d, 0x62, 0x43 }, + { 0x7a, 0xbf, 0x6a, 0xcf, 0x5c, 0x8e, 0x54, 0x9d, + 0xdb, 0xb1, 0x5a, 0xe8, 0xd8, 0xb3, 0x88, 0xc1, + 0xc1, 0x97, 0xe6, 0x98, 0x73, 0x7c, 0x97, 0x85, + 0x50, 0x1e, 0xd1, 0xf9, 0x49, 0x30, 0xb7, 0xd9 }, + { 0x88, 0x01, 0x8d, 0xed, 0x66, 0x81, 0x3f, 0x0c, + 0xa9, 0x5d, 0xef, 0x47, 0x4c, 0x63, 0x06, 0x92, + 0x01, 0x99, 0x67, 0xb9, 0xe3, 0x68, 0x88, 0xda, + 0xdd, 0x94, 0x12, 0x47, 0x19, 0xb6, 0x82, 0xf6 }, + { 0x39, 0x30, 0x87, 0x6b, 0x9f, 0xc7, 0x52, 0x90, + 0x36, 0xb0, 0x08, 0xb1, 0xb8, 0xbb, 0x99, 0x75, + 0x22, 0xa4, 0x41, 0x63, 0x5a, 0x0c, 0x25, 0xec, + 0x02, 0xfb, 0x6d, 0x90, 0x26, 0xe5, 0x5a, 0x97 }, + { 0x0a, 0x40, 0x49, 0xd5, 0x7e, 0x83, 0x3b, 0x56, + 0x95, 0xfa, 0xc9, 0x3d, 0xd1, 0xfb, 0xef, 0x31, + 0x66, 0xb4, 0x4b, 0x12, 0xad, 0x11, 0x24, 0x86, + 0x62, 0x38, 0x3a, 0xe0, 0x51, 0xe1, 0x58, 0x27 }, + { 0x81, 0xdc, 0xc0, 0x67, 0x8b, 0xb6, 0xa7, 0x65, + 0xe4, 0x8c, 0x32, 0x09, 0x65, 0x4f, 0xe9, 0x00, + 0x89, 0xce, 0x44, 0xff, 0x56, 0x18, 0x47, 0x7e, + 0x39, 0xab, 0x28, 0x64, 0x76, 0xdf, 0x05, 0x2b }, + { 0xe6, 0x9b, 0x3a, 0x36, 0xa4, 0x46, 0x19, 0x12, + 0xdc, 0x08, 0x34, 0x6b, 0x11, 0xdd, 0xcb, 0x9d, + 0xb7, 0x96, 0xf8, 0x85, 0xfd, 0x01, 0x93, 0x6e, + 0x66, 0x2f, 0xe2, 0x92, 0x97, 0xb0, 0x99, 0xa4 }, + { 0x5a, 0xc6, 0x50, 0x3b, 0x0d, 0x8d, 0xa6, 0x91, + 0x76, 0x46, 0xe6, 0xdc, 0xc8, 0x7e, 0xdc, 0x58, + 0xe9, 0x42, 0x45, 0x32, 0x4c, 0xc2, 0x04, 0xf4, + 0xdd, 0x4a, 0xf0, 0x15, 0x63, 0xac, 0xd4, 0x27 }, + { 0xdf, 0x6d, 0xda, 0x21, 0x35, 0x9a, 0x30, 0xbc, + 0x27, 0x17, 0x80, 0x97, 0x1c, 0x1a, 0xbd, 0x56, + 0xa6, 0xef, 0x16, 0x7e, 0x48, 0x08, 0x87, 0x88, + 0x8e, 0x73, 0xa8, 0x6d, 0x3b, 0xf6, 0x05, 0xe9 }, + { 0xe8, 0xe6, 0xe4, 0x70, 0x71, 0xe7, 0xb7, 0xdf, + 0x25, 0x80, 0xf2, 0x25, 0xcf, 0xbb, 0xed, 0xf8, + 0x4c, 0xe6, 0x77, 0x46, 0x62, 0x66, 0x28, 0xd3, + 0x30, 0x97, 0xe4, 0xb7, 0xdc, 0x57, 0x11, 0x07 }, + { 0x53, 0xe4, 0x0e, 0xad, 0x62, 0x05, 0x1e, 0x19, + 0xcb, 0x9b, 0xa8, 0x13, 0x3e, 0x3e, 0x5c, 0x1c, + 0xe0, 0x0d, 0xdc, 0xad, 0x8a, 0xcf, 0x34, 0x2a, + 0x22, 0x43, 0x60, 0xb0, 0xac, 0xc1, 0x47, 0x77 }, + { 0x9c, 0xcd, 0x53, 0xfe, 0x80, 0xbe, 0x78, 0x6a, + 0xa9, 0x84, 0x63, 0x84, 0x62, 0xfb, 0x28, 0xaf, + 0xdf, 0x12, 0x2b, 0x34, 0xd7, 0x8f, 0x46, 0x87, + 0xec, 0x63, 0x2b, 0xb1, 0x9d, 0xe2, 0x37, 0x1a }, + { 0xcb, 0xd4, 0x80, 0x52, 0xc4, 0x8d, 0x78, 0x84, + 0x66, 0xa3, 0xe8, 0x11, 0x8c, 0x56, 0xc9, 0x7f, + 0xe1, 0x46, 0xe5, 0x54, 0x6f, 0xaa, 0xf9, 0x3e, + 0x2b, 0xc3, 0xc4, 0x7e, 0x45, 0x93, 0x97, 0x53 }, + { 0x25, 0x68, 0x83, 0xb1, 0x4e, 0x2a, 0xf4, 0x4d, + 0xad, 0xb2, 0x8e, 0x1b, 0x34, 0xb2, 0xac, 0x0f, + 0x0f, 0x4c, 0x91, 0xc3, 0x4e, 0xc9, 0x16, 0x9e, + 0x29, 0x03, 0x61, 0x58, 0xac, 0xaa, 0x95, 0xb9 }, + { 0x44, 0x71, 0xb9, 0x1a, 0xb4, 0x2d, 0xb7, 0xc4, + 0xdd, 0x84, 0x90, 0xab, 0x95, 0xa2, 0xee, 0x8d, + 0x04, 0xe3, 0xef, 0x5c, 0x3d, 0x6f, 0xc7, 0x1a, + 0xc7, 0x4b, 0x2b, 0x26, 0x91, 0x4d, 0x16, 0x41 }, + { 0xa5, 0xeb, 0x08, 0x03, 0x8f, 0x8f, 0x11, 0x55, + 0xed, 0x86, 0xe6, 0x31, 0x90, 0x6f, 0xc1, 0x30, + 0x95, 0xf6, 0xbb, 0xa4, 0x1d, 0xe5, 0xd4, 0xe7, + 0x95, 0x75, 0x8e, 0xc8, 0xc8, 0xdf, 0x8a, 0xf1 }, + { 0xdc, 0x1d, 0xb6, 0x4e, 0xd8, 0xb4, 0x8a, 0x91, + 0x0e, 0x06, 0x0a, 0x6b, 0x86, 0x63, 0x74, 0xc5, + 0x78, 0x78, 0x4e, 0x9a, 0xc4, 0x9a, 0xb2, 0x77, + 0x40, 0x92, 0xac, 0x71, 0x50, 0x19, 0x34, 0xac }, + { 0x28, 0x54, 0x13, 0xb2, 0xf2, 0xee, 0x87, 0x3d, + 0x34, 0x31, 0x9e, 0xe0, 0xbb, 0xfb, 0xb9, 0x0f, + 0x32, 0xda, 0x43, 0x4c, 0xc8, 0x7e, 0x3d, 0xb5, + 0xed, 0x12, 0x1b, 0xb3, 0x98, 0xed, 0x96, 0x4b }, + { 0x02, 0x16, 0xe0, 0xf8, 0x1f, 0x75, 0x0f, 0x26, + 0xf1, 0x99, 0x8b, 0xc3, 0x93, 0x4e, 0x3e, 0x12, + 0x4c, 0x99, 0x45, 0xe6, 0x85, 0xa6, 0x0b, 0x25, + 0xe8, 0xfb, 0xd9, 0x62, 0x5a, 0xb6, 0xb5, 0x99 }, + { 0x38, 0xc4, 0x10, 0xf5, 0xb9, 0xd4, 0x07, 0x20, + 0x50, 0x75, 0x5b, 0x31, 0xdc, 0xa8, 0x9f, 0xd5, + 0x39, 0x5c, 0x67, 0x85, 0xee, 0xb3, 0xd7, 0x90, + 0xf3, 0x20, 0xff, 0x94, 0x1c, 0x5a, 0x93, 0xbf }, + { 0xf1, 0x84, 0x17, 0xb3, 0x9d, 0x61, 0x7a, 0xb1, + 0xc1, 0x8f, 0xdf, 0x91, 0xeb, 0xd0, 0xfc, 0x6d, + 0x55, 0x16, 0xbb, 0x34, 0xcf, 0x39, 0x36, 0x40, + 0x37, 0xbc, 0xe8, 0x1f, 0xa0, 0x4c, 0xec, 0xb1 }, + { 0x1f, 0xa8, 0x77, 0xde, 0x67, 0x25, 0x9d, 0x19, + 0x86, 0x3a, 0x2a, 0x34, 0xbc, 0xc6, 0x96, 0x2a, + 0x2b, 0x25, 0xfc, 0xbf, 0x5c, 0xbe, 0xcd, 0x7e, + 0xde, 0x8f, 0x1f, 0xa3, 0x66, 0x88, 0xa7, 0x96 }, + { 0x5b, 0xd1, 0x69, 0xe6, 0x7c, 0x82, 0xc2, 0xc2, + 0xe9, 0x8e, 0xf7, 0x00, 0x8b, 0xdf, 0x26, 0x1f, + 0x2d, 0xdf, 0x30, 0xb1, 0xc0, 0x0f, 0x9e, 0x7f, + 0x27, 0x5b, 0xb3, 0xe8, 0xa2, 0x8d, 0xc9, 0xa2 }, + { 0xc8, 0x0a, 0xbe, 0xeb, 0xb6, 0x69, 0xad, 0x5d, + 0xee, 0xb5, 0xf5, 0xec, 0x8e, 0xa6, 0xb7, 0xa0, + 0x5d, 0xdf, 0x7d, 0x31, 0xec, 0x4c, 0x0a, 0x2e, + 0xe2, 0x0b, 0x0b, 0x98, 0xca, 0xec, 0x67, 0x46 }, + { 0xe7, 0x6d, 0x3f, 0xbd, 0xa5, 0xba, 0x37, 0x4e, + 0x6b, 0xf8, 0xe5, 0x0f, 0xad, 0xc3, 0xbb, 0xb9, + 0xba, 0x5c, 0x20, 0x6e, 0xbd, 0xec, 0x89, 0xa3, + 0xa5, 0x4c, 0xf3, 0xdd, 0x84, 0xa0, 0x70, 0x16 }, + { 0x7b, 0xba, 0x9d, 0xc5, 0xb5, 0xdb, 0x20, 0x71, + 0xd1, 0x77, 0x52, 0xb1, 0x04, 0x4c, 0x1e, 0xce, + 0xd9, 0x6a, 0xaf, 0x2d, 0xd4, 0x6e, 0x9b, 0x43, + 0x37, 0x50, 0xe8, 0xea, 0x0d, 0xcc, 0x18, 0x70 }, + { 0xf2, 0x9b, 0x1b, 0x1a, 0xb9, 0xba, 0xb1, 0x63, + 0x01, 0x8e, 0xe3, 0xda, 0x15, 0x23, 0x2c, 0xca, + 0x78, 0xec, 0x52, 0xdb, 0xc3, 0x4e, 0xda, 0x5b, + 0x82, 0x2e, 0xc1, 0xd8, 0x0f, 0xc2, 0x1b, 0xd0 }, + { 0x9e, 0xe3, 0xe3, 0xe7, 0xe9, 0x00, 0xf1, 0xe1, + 0x1d, 0x30, 0x8c, 0x4b, 0x2b, 0x30, 0x76, 0xd2, + 0x72, 0xcf, 0x70, 0x12, 0x4f, 0x9f, 0x51, 0xe1, + 0xda, 0x60, 0xf3, 0x78, 0x46, 0xcd, 0xd2, 0xf4 }, + { 0x70, 0xea, 0x3b, 0x01, 0x76, 0x92, 0x7d, 0x90, + 0x96, 0xa1, 0x85, 0x08, 0xcd, 0x12, 0x3a, 0x29, + 0x03, 0x25, 0x92, 0x0a, 0x9d, 0x00, 0xa8, 0x9b, + 0x5d, 0xe0, 0x42, 0x73, 0xfb, 0xc7, 0x6b, 0x85 }, + { 0x67, 0xde, 0x25, 0xc0, 0x2a, 0x4a, 0xab, 0xa2, + 0x3b, 0xdc, 0x97, 0x3c, 0x8b, 0xb0, 0xb5, 0x79, + 0x6d, 0x47, 0xcc, 0x06, 0x59, 0xd4, 0x3d, 0xff, + 0x1f, 0x97, 0xde, 0x17, 0x49, 0x63, 0xb6, 0x8e }, + { 0xb2, 0x16, 0x8e, 0x4e, 0x0f, 0x18, 0xb0, 0xe6, + 0x41, 0x00, 0xb5, 0x17, 0xed, 0x95, 0x25, 0x7d, + 0x73, 0xf0, 0x62, 0x0d, 0xf8, 0x85, 0xc1, 0x3d, + 0x2e, 0xcf, 0x79, 0x36, 0x7b, 0x38, 0x4c, 0xee }, + { 0x2e, 0x7d, 0xec, 0x24, 0x28, 0x85, 0x3b, 0x2c, + 0x71, 0x76, 0x07, 0x45, 0x54, 0x1f, 0x7a, 0xfe, + 0x98, 0x25, 0xb5, 0xdd, 0x77, 0xdf, 0x06, 0x51, + 0x1d, 0x84, 0x41, 0xa9, 0x4b, 0xac, 0xc9, 0x27 }, + { 0xca, 0x9f, 0xfa, 0xc4, 0xc4, 0x3f, 0x0b, 0x48, + 0x46, 0x1d, 0xc5, 0xc2, 0x63, 0xbe, 0xa3, 0xf6, + 0xf0, 0x06, 0x11, 0xce, 0xac, 0xab, 0xf6, 0xf8, + 0x95, 0xba, 0x2b, 0x01, 0x01, 0xdb, 0xb6, 0x8d }, + { 0x74, 0x10, 0xd4, 0x2d, 0x8f, 0xd1, 0xd5, 0xe9, + 0xd2, 0xf5, 0x81, 0x5c, 0xb9, 0x34, 0x17, 0x99, + 0x88, 0x28, 0xef, 0x3c, 0x42, 0x30, 0xbf, 0xbd, + 0x41, 0x2d, 0xf0, 0xa4, 0xa7, 0xa2, 0x50, 0x7a }, + { 0x50, 0x10, 0xf6, 0x84, 0x51, 0x6d, 0xcc, 0xd0, + 0xb6, 0xee, 0x08, 0x52, 0xc2, 0x51, 0x2b, 0x4d, + 0xc0, 0x06, 0x6c, 0xf0, 0xd5, 0x6f, 0x35, 0x30, + 0x29, 0x78, 0xdb, 0x8a, 0xe3, 0x2c, 0x6a, 0x81 }, + { 0xac, 0xaa, 0xb5, 0x85, 0xf7, 0xb7, 0x9b, 0x71, + 0x99, 0x35, 0xce, 0xb8, 0x95, 0x23, 0xdd, 0xc5, + 0x48, 0x27, 0xf7, 0x5c, 0x56, 0x88, 0x38, 0x56, + 0x15, 0x4a, 0x56, 0xcd, 0xcd, 0x5e, 0xe9, 0x88 }, + { 0x66, 0x6d, 0xe5, 0xd1, 0x44, 0x0f, 0xee, 0x73, + 0x31, 0xaa, 0xf0, 0x12, 0x3a, 0x62, 0xef, 0x2d, + 0x8b, 0xa5, 0x74, 0x53, 0xa0, 0x76, 0x96, 0x35, + 0xac, 0x6c, 0xd0, 0x1e, 0x63, 0x3f, 0x77, 0x12 }, + { 0xa6, 0xf9, 0x86, 0x58, 0xf6, 0xea, 0xba, 0xf9, + 0x02, 0xd8, 0xb3, 0x87, 0x1a, 0x4b, 0x10, 0x1d, + 0x16, 0x19, 0x6e, 0x8a, 0x4b, 0x24, 0x1e, 0x15, + 0x58, 0xfe, 0x29, 0x96, 0x6e, 0x10, 0x3e, 0x8d }, + { 0x89, 0x15, 0x46, 0xa8, 0xb2, 0x9f, 0x30, 0x47, + 0xdd, 0xcf, 0xe5, 0xb0, 0x0e, 0x45, 0xfd, 0x55, + 0x75, 0x63, 0x73, 0x10, 0x5e, 0xa8, 0x63, 0x7d, + 0xfc, 0xff, 0x54, 0x7b, 0x6e, 0xa9, 0x53, 0x5f }, + { 0x18, 0xdf, 0xbc, 0x1a, 0xc5, 0xd2, 0x5b, 0x07, + 0x61, 0x13, 0x7d, 0xbd, 0x22, 0xc1, 0x7c, 0x82, + 0x9d, 0x0f, 0x0e, 0xf1, 0xd8, 0x23, 0x44, 0xe9, + 0xc8, 0x9c, 0x28, 0x66, 0x94, 0xda, 0x24, 0xe8 }, + { 0xb5, 0x4b, 0x9b, 0x67, 0xf8, 0xfe, 0xd5, 0x4b, + 0xbf, 0x5a, 0x26, 0x66, 0xdb, 0xdf, 0x4b, 0x23, + 0xcf, 0xf1, 0xd1, 0xb6, 0xf4, 0xaf, 0xc9, 0x85, + 0xb2, 0xe6, 0xd3, 0x30, 0x5a, 0x9f, 0xf8, 0x0f }, + { 0x7d, 0xb4, 0x42, 0xe1, 0x32, 0xba, 0x59, 0xbc, + 0x12, 0x89, 0xaa, 0x98, 0xb0, 0xd3, 0xe8, 0x06, + 0x00, 0x4f, 0x8e, 0xc1, 0x28, 0x11, 0xaf, 0x1e, + 0x2e, 0x33, 0xc6, 0x9b, 0xfd, 0xe7, 0x29, 0xe1 }, + { 0x25, 0x0f, 0x37, 0xcd, 0xc1, 0x5e, 0x81, 0x7d, + 0x2f, 0x16, 0x0d, 0x99, 0x56, 0xc7, 0x1f, 0xe3, + 0xeb, 0x5d, 0xb7, 0x45, 0x56, 0xe4, 0xad, 0xf9, + 0xa4, 0xff, 0xaf, 0xba, 0x74, 0x01, 0x03, 0x96 }, + { 0x4a, 0xb8, 0xa3, 0xdd, 0x1d, 0xdf, 0x8a, 0xd4, + 0x3d, 0xab, 0x13, 0xa2, 0x7f, 0x66, 0xa6, 0x54, + 0x4f, 0x29, 0x05, 0x97, 0xfa, 0x96, 0x04, 0x0e, + 0x0e, 0x1d, 0xb9, 0x26, 0x3a, 0xa4, 0x79, 0xf8 }, + { 0xee, 0x61, 0x72, 0x7a, 0x07, 0x66, 0xdf, 0x93, + 0x9c, 0xcd, 0xc8, 0x60, 0x33, 0x40, 0x44, 0xc7, + 0x9a, 0x3c, 0x9b, 0x15, 0x62, 0x00, 0xbc, 0x3a, + 0xa3, 0x29, 0x73, 0x48, 0x3d, 0x83, 0x41, 0xae }, + { 0x3f, 0x68, 0xc7, 0xec, 0x63, 0xac, 0x11, 0xeb, + 0xb9, 0x8f, 0x94, 0xb3, 0x39, 0xb0, 0x5c, 0x10, + 0x49, 0x84, 0xfd, 0xa5, 0x01, 0x03, 0x06, 0x01, + 0x44, 0xe5, 0xa2, 0xbf, 0xcc, 0xc9, 0xda, 0x95 }, + { 0x05, 0x6f, 0x29, 0x81, 0x6b, 0x8a, 0xf8, 0xf5, + 0x66, 0x82, 0xbc, 0x4d, 0x7c, 0xf0, 0x94, 0x11, + 0x1d, 0xa7, 0x73, 0x3e, 0x72, 0x6c, 0xd1, 0x3d, + 0x6b, 0x3e, 0x8e, 0xa0, 0x3e, 0x92, 0xa0, 0xd5 }, + { 0xf5, 0xec, 0x43, 0xa2, 0x8a, 0xcb, 0xef, 0xf1, + 0xf3, 0x31, 0x8a, 0x5b, 0xca, 0xc7, 0xc6, 0x6d, + 0xdb, 0x52, 0x30, 0xb7, 0x9d, 0xb2, 0xd1, 0x05, + 0xbc, 0xbe, 0x15, 0xf3, 0xc1, 0x14, 0x8d, 0x69 }, + { 0x2a, 0x69, 0x60, 0xad, 0x1d, 0x8d, 0xd5, 0x47, + 0x55, 0x5c, 0xfb, 0xd5, 0xe4, 0x60, 0x0f, 0x1e, + 0xaa, 0x1c, 0x8e, 0xda, 0x34, 0xde, 0x03, 0x74, + 0xec, 0x4a, 0x26, 0xea, 0xaa, 0xa3, 0x3b, 0x4e }, + { 0xdc, 0xc1, 0xea, 0x7b, 0xaa, 0xb9, 0x33, 0x84, + 0xf7, 0x6b, 0x79, 0x68, 0x66, 0x19, 0x97, 0x54, + 0x74, 0x2f, 0x7b, 0x96, 0xd6, 0xb4, 0xc1, 0x20, + 0x16, 0x5c, 0x04, 0xa6, 0xc4, 0xf5, 0xce, 0x10 }, + { 0x13, 0xd5, 0xdf, 0x17, 0x92, 0x21, 0x37, 0x9c, + 0x6a, 0x78, 0xc0, 0x7c, 0x79, 0x3f, 0xf5, 0x34, + 0x87, 0xca, 0xe6, 0xbf, 0x9f, 0xe8, 0x82, 0x54, + 0x1a, 0xb0, 0xe7, 0x35, 0xe3, 0xea, 0xda, 0x3b }, + { 0x8c, 0x59, 0xe4, 0x40, 0x76, 0x41, 0xa0, 0x1e, + 0x8f, 0xf9, 0x1f, 0x99, 0x80, 0xdc, 0x23, 0x6f, + 0x4e, 0xcd, 0x6f, 0xcf, 0x52, 0x58, 0x9a, 0x09, + 0x9a, 0x96, 0x16, 0x33, 0x96, 0x77, 0x14, 0xe1 }, + { 0x83, 0x3b, 0x1a, 0xc6, 0xa2, 0x51, 0xfd, 0x08, + 0xfd, 0x6d, 0x90, 0x8f, 0xea, 0x2a, 0x4e, 0xe1, + 0xe0, 0x40, 0xbc, 0xa9, 0x3f, 0xc1, 0xa3, 0x8e, + 0xc3, 0x82, 0x0e, 0x0c, 0x10, 0xbd, 0x82, 0xea }, + { 0xa2, 0x44, 0xf9, 0x27, 0xf3, 0xb4, 0x0b, 0x8f, + 0x6c, 0x39, 0x15, 0x70, 0xc7, 0x65, 0x41, 0x8f, + 0x2f, 0x6e, 0x70, 0x8e, 0xac, 0x90, 0x06, 0xc5, + 0x1a, 0x7f, 0xef, 0xf4, 0xaf, 0x3b, 0x2b, 0x9e }, + { 0x3d, 0x99, 0xed, 0x95, 0x50, 0xcf, 0x11, 0x96, + 0xe6, 0xc4, 0xd2, 0x0c, 0x25, 0x96, 0x20, 0xf8, + 0x58, 0xc3, 0xd7, 0x03, 0x37, 0x4c, 0x12, 0x8c, + 0xe7, 0xb5, 0x90, 0x31, 0x0c, 0x83, 0x04, 0x6d }, + { 0x2b, 0x35, 0xc4, 0x7d, 0x7b, 0x87, 0x76, 0x1f, + 0x0a, 0xe4, 0x3a, 0xc5, 0x6a, 0xc2, 0x7b, 0x9f, + 0x25, 0x83, 0x03, 0x67, 0xb5, 0x95, 0xbe, 0x8c, + 0x24, 0x0e, 0x94, 0x60, 0x0c, 0x6e, 0x33, 0x12 }, + { 0x5d, 0x11, 0xed, 0x37, 0xd2, 0x4d, 0xc7, 0x67, + 0x30, 0x5c, 0xb7, 0xe1, 0x46, 0x7d, 0x87, 0xc0, + 0x65, 0xac, 0x4b, 0xc8, 0xa4, 0x26, 0xde, 0x38, + 0x99, 0x1f, 0xf5, 0x9a, 0xa8, 0x73, 0x5d, 0x02 }, + { 0xb8, 0x36, 0x47, 0x8e, 0x1c, 0xa0, 0x64, 0x0d, + 0xce, 0x6f, 0xd9, 0x10, 0xa5, 0x09, 0x62, 0x72, + 0xc8, 0x33, 0x09, 0x90, 0xcd, 0x97, 0x86, 0x4a, + 0xc2, 0xbf, 0x14, 0xef, 0x6b, 0x23, 0x91, 0x4a }, + { 0x91, 0x00, 0xf9, 0x46, 0xd6, 0xcc, 0xde, 0x3a, + 0x59, 0x7f, 0x90, 0xd3, 0x9f, 0xc1, 0x21, 0x5b, + 0xad, 0xdc, 0x74, 0x13, 0x64, 0x3d, 0x85, 0xc2, + 0x1c, 0x3e, 0xee, 0x5d, 0x2d, 0xd3, 0x28, 0x94 }, + { 0xda, 0x70, 0xee, 0xdd, 0x23, 0xe6, 0x63, 0xaa, + 0x1a, 0x74, 0xb9, 0x76, 0x69, 0x35, 0xb4, 0x79, + 0x22, 0x2a, 0x72, 0xaf, 0xba, 0x5c, 0x79, 0x51, + 0x58, 0xda, 0xd4, 0x1a, 0x3b, 0xd7, 0x7e, 0x40 }, + { 0xf0, 0x67, 0xed, 0x6a, 0x0d, 0xbd, 0x43, 0xaa, + 0x0a, 0x92, 0x54, 0xe6, 0x9f, 0xd6, 0x6b, 0xdd, + 0x8a, 0xcb, 0x87, 0xde, 0x93, 0x6c, 0x25, 0x8c, + 0xfb, 0x02, 0x28, 0x5f, 0x2c, 0x11, 0xfa, 0x79 }, + { 0x71, 0x5c, 0x99, 0xc7, 0xd5, 0x75, 0x80, 0xcf, + 0x97, 0x53, 0xb4, 0xc1, 0xd7, 0x95, 0xe4, 0x5a, + 0x83, 0xfb, 0xb2, 0x28, 0xc0, 0xd3, 0x6f, 0xbe, + 0x20, 0xfa, 0xf3, 0x9b, 0xdd, 0x6d, 0x4e, 0x85 }, + { 0xe4, 0x57, 0xd6, 0xad, 0x1e, 0x67, 0xcb, 0x9b, + 0xbd, 0x17, 0xcb, 0xd6, 0x98, 0xfa, 0x6d, 0x7d, + 0xae, 0x0c, 0x9b, 0x7a, 0xd6, 0xcb, 0xd6, 0x53, + 0x96, 0x34, 0xe3, 0x2a, 0x71, 0x9c, 0x84, 0x92 }, + { 0xec, 0xe3, 0xea, 0x81, 0x03, 0xe0, 0x24, 0x83, + 0xc6, 0x4a, 0x70, 0xa4, 0xbd, 0xce, 0xe8, 0xce, + 0xb6, 0x27, 0x8f, 0x25, 0x33, 0xf3, 0xf4, 0x8d, + 0xbe, 0xed, 0xfb, 0xa9, 0x45, 0x31, 0xd4, 0xae }, + { 0x38, 0x8a, 0xa5, 0xd3, 0x66, 0x7a, 0x97, 0xc6, + 0x8d, 0x3d, 0x56, 0xf8, 0xf3, 0xee, 0x8d, 0x3d, + 0x36, 0x09, 0x1f, 0x17, 0xfe, 0x5d, 0x1b, 0x0d, + 0x5d, 0x84, 0xc9, 0x3b, 0x2f, 0xfe, 0x40, 0xbd }, + { 0x8b, 0x6b, 0x31, 0xb9, 0xad, 0x7c, 0x3d, 0x5c, + 0xd8, 0x4b, 0xf9, 0x89, 0x47, 0xb9, 0xcd, 0xb5, + 0x9d, 0xf8, 0xa2, 0x5f, 0xf7, 0x38, 0x10, 0x10, + 0x13, 0xbe, 0x4f, 0xd6, 0x5e, 0x1d, 0xd1, 0xa3 }, + { 0x06, 0x62, 0x91, 0xf6, 0xbb, 0xd2, 0x5f, 0x3c, + 0x85, 0x3d, 0xb7, 0xd8, 0xb9, 0x5c, 0x9a, 0x1c, + 0xfb, 0x9b, 0xf1, 0xc1, 0xc9, 0x9f, 0xb9, 0x5a, + 0x9b, 0x78, 0x69, 0xd9, 0x0f, 0x1c, 0x29, 0x03 }, + { 0xa7, 0x07, 0xef, 0xbc, 0xcd, 0xce, 0xed, 0x42, + 0x96, 0x7a, 0x66, 0xf5, 0x53, 0x9b, 0x93, 0xed, + 0x75, 0x60, 0xd4, 0x67, 0x30, 0x40, 0x16, 0xc4, + 0x78, 0x0d, 0x77, 0x55, 0xa5, 0x65, 0xd4, 0xc4 }, + { 0x38, 0xc5, 0x3d, 0xfb, 0x70, 0xbe, 0x7e, 0x79, + 0x2b, 0x07, 0xa6, 0xa3, 0x5b, 0x8a, 0x6a, 0x0a, + 0xba, 0x02, 0xc5, 0xc5, 0xf3, 0x8b, 0xaf, 0x5c, + 0x82, 0x3f, 0xdf, 0xd9, 0xe4, 0x2d, 0x65, 0x7e }, + { 0xf2, 0x91, 0x13, 0x86, 0x50, 0x1d, 0x9a, 0xb9, + 0xd7, 0x20, 0xcf, 0x8a, 0xd1, 0x05, 0x03, 0xd5, + 0x63, 0x4b, 0xf4, 0xb7, 0xd1, 0x2b, 0x56, 0xdf, + 0xb7, 0x4f, 0xec, 0xc6, 0xe4, 0x09, 0x3f, 0x68 }, + { 0xc6, 0xf2, 0xbd, 0xd5, 0x2b, 0x81, 0xe6, 0xe4, + 0xf6, 0x59, 0x5a, 0xbd, 0x4d, 0x7f, 0xb3, 0x1f, + 0x65, 0x11, 0x69, 0xd0, 0x0f, 0xf3, 0x26, 0x92, + 0x6b, 0x34, 0x94, 0x7b, 0x28, 0xa8, 0x39, 0x59 }, + { 0x29, 0x3d, 0x94, 0xb1, 0x8c, 0x98, 0xbb, 0x32, + 0x23, 0x36, 0x6b, 0x8c, 0xe7, 0x4c, 0x28, 0xfb, + 0xdf, 0x28, 0xe1, 0xf8, 0x4a, 0x33, 0x50, 0xb0, + 0xeb, 0x2d, 0x18, 0x04, 0xa5, 0x77, 0x57, 0x9b }, + { 0x2c, 0x2f, 0xa5, 0xc0, 0xb5, 0x15, 0x33, 0x16, + 0x5b, 0xc3, 0x75, 0xc2, 0x2e, 0x27, 0x81, 0x76, + 0x82, 0x70, 0xa3, 0x83, 0x98, 0x5d, 0x13, 0xbd, + 0x6b, 0x67, 0xb6, 0xfd, 0x67, 0xf8, 0x89, 0xeb }, + { 0xca, 0xa0, 0x9b, 0x82, 0xb7, 0x25, 0x62, 0xe4, + 0x3f, 0x4b, 0x22, 0x75, 0xc0, 0x91, 0x91, 0x8e, + 0x62, 0x4d, 0x91, 0x16, 0x61, 0xcc, 0x81, 0x1b, + 0xb5, 0xfa, 0xec, 0x51, 0xf6, 0x08, 0x8e, 0xf7 }, + { 0x24, 0x76, 0x1e, 0x45, 0xe6, 0x74, 0x39, 0x53, + 0x79, 0xfb, 0x17, 0x72, 0x9c, 0x78, 0xcb, 0x93, + 0x9e, 0x6f, 0x74, 0xc5, 0xdf, 0xfb, 0x9c, 0x96, + 0x1f, 0x49, 0x59, 0x82, 0xc3, 0xed, 0x1f, 0xe3 }, + { 0x55, 0xb7, 0x0a, 0x82, 0x13, 0x1e, 0xc9, 0x48, + 0x88, 0xd7, 0xab, 0x54, 0xa7, 0xc5, 0x15, 0x25, + 0x5c, 0x39, 0x38, 0xbb, 0x10, 0xbc, 0x78, 0x4d, + 0xc9, 0xb6, 0x7f, 0x07, 0x6e, 0x34, 0x1a, 0x73 }, + { 0x6a, 0xb9, 0x05, 0x7b, 0x97, 0x7e, 0xbc, 0x3c, + 0xa4, 0xd4, 0xce, 0x74, 0x50, 0x6c, 0x25, 0xcc, + 0xcd, 0xc5, 0x66, 0x49, 0x7c, 0x45, 0x0b, 0x54, + 0x15, 0xa3, 0x94, 0x86, 0xf8, 0x65, 0x7a, 0x03 }, + { 0x24, 0x06, 0x6d, 0xee, 0xe0, 0xec, 0xee, 0x15, + 0xa4, 0x5f, 0x0a, 0x32, 0x6d, 0x0f, 0x8d, 0xbc, + 0x79, 0x76, 0x1e, 0xbb, 0x93, 0xcf, 0x8c, 0x03, + 0x77, 0xaf, 0x44, 0x09, 0x78, 0xfc, 0xf9, 0x94 }, + { 0x20, 0x00, 0x0d, 0x3f, 0x66, 0xba, 0x76, 0x86, + 0x0d, 0x5a, 0x95, 0x06, 0x88, 0xb9, 0xaa, 0x0d, + 0x76, 0xcf, 0xea, 0x59, 0xb0, 0x05, 0xd8, 0x59, + 0x91, 0x4b, 0x1a, 0x46, 0x65, 0x3a, 0x93, 0x9b }, + { 0xb9, 0x2d, 0xaa, 0x79, 0x60, 0x3e, 0x3b, 0xdb, + 0xc3, 0xbf, 0xe0, 0xf4, 0x19, 0xe4, 0x09, 0xb2, + 0xea, 0x10, 0xdc, 0x43, 0x5b, 0xee, 0xfe, 0x29, + 0x59, 0xda, 0x16, 0x89, 0x5d, 0x5d, 0xca, 0x1c }, + { 0xe9, 0x47, 0x94, 0x87, 0x05, 0xb2, 0x06, 0xd5, + 0x72, 0xb0, 0xe8, 0xf6, 0x2f, 0x66, 0xa6, 0x55, + 0x1c, 0xbd, 0x6b, 0xc3, 0x05, 0xd2, 0x6c, 0xe7, + 0x53, 0x9a, 0x12, 0xf9, 0xaa, 0xdf, 0x75, 0x71 }, + { 0x3d, 0x67, 0xc1, 0xb3, 0xf9, 0xb2, 0x39, 0x10, + 0xe3, 0xd3, 0x5e, 0x6b, 0x0f, 0x2c, 0xcf, 0x44, + 0xa0, 0xb5, 0x40, 0xa4, 0x5c, 0x18, 0xba, 0x3c, + 0x36, 0x26, 0x4d, 0xd4, 0x8e, 0x96, 0xaf, 0x6a }, + { 0xc7, 0x55, 0x8b, 0xab, 0xda, 0x04, 0xbc, 0xcb, + 0x76, 0x4d, 0x0b, 0xbf, 0x33, 0x58, 0x42, 0x51, + 0x41, 0x90, 0x2d, 0x22, 0x39, 0x1d, 0x9f, 0x8c, + 0x59, 0x15, 0x9f, 0xec, 0x9e, 0x49, 0xb1, 0x51 }, + { 0x0b, 0x73, 0x2b, 0xb0, 0x35, 0x67, 0x5a, 0x50, + 0xff, 0x58, 0xf2, 0xc2, 0x42, 0xe4, 0x71, 0x0a, + 0xec, 0xe6, 0x46, 0x70, 0x07, 0x9c, 0x13, 0x04, + 0x4c, 0x79, 0xc9, 0xb7, 0x49, 0x1f, 0x70, 0x00 }, + { 0xd1, 0x20, 0xb5, 0xef, 0x6d, 0x57, 0xeb, 0xf0, + 0x6e, 0xaf, 0x96, 0xbc, 0x93, 0x3c, 0x96, 0x7b, + 0x16, 0xcb, 0xe6, 0xe2, 0xbf, 0x00, 0x74, 0x1c, + 0x30, 0xaa, 0x1c, 0x54, 0xba, 0x64, 0x80, 0x1f }, + { 0x58, 0xd2, 0x12, 0xad, 0x6f, 0x58, 0xae, 0xf0, + 0xf8, 0x01, 0x16, 0xb4, 0x41, 0xe5, 0x7f, 0x61, + 0x95, 0xbf, 0xef, 0x26, 0xb6, 0x14, 0x63, 0xed, + 0xec, 0x11, 0x83, 0xcd, 0xb0, 0x4f, 0xe7, 0x6d }, + { 0xb8, 0x83, 0x6f, 0x51, 0xd1, 0xe2, 0x9b, 0xdf, + 0xdb, 0xa3, 0x25, 0x56, 0x53, 0x60, 0x26, 0x8b, + 0x8f, 0xad, 0x62, 0x74, 0x73, 0xed, 0xec, 0xef, + 0x7e, 0xae, 0xfe, 0xe8, 0x37, 0xc7, 0x40, 0x03 }, + { 0xc5, 0x47, 0xa3, 0xc1, 0x24, 0xae, 0x56, 0x85, + 0xff, 0xa7, 0xb8, 0xed, 0xaf, 0x96, 0xec, 0x86, + 0xf8, 0xb2, 0xd0, 0xd5, 0x0c, 0xee, 0x8b, 0xe3, + 0xb1, 0xf0, 0xc7, 0x67, 0x63, 0x06, 0x9d, 0x9c }, + { 0x5d, 0x16, 0x8b, 0x76, 0x9a, 0x2f, 0x67, 0x85, + 0x3d, 0x62, 0x95, 0xf7, 0x56, 0x8b, 0xe4, 0x0b, + 0xb7, 0xa1, 0x6b, 0x8d, 0x65, 0xba, 0x87, 0x63, + 0x5d, 0x19, 0x78, 0xd2, 0xab, 0x11, 0xba, 0x2a }, + { 0xa2, 0xf6, 0x75, 0xdc, 0x73, 0x02, 0x63, 0x8c, + 0xb6, 0x02, 0x01, 0x06, 0x4c, 0xa5, 0x50, 0x77, + 0x71, 0x4d, 0x71, 0xfe, 0x09, 0x6a, 0x31, 0x5f, + 0x2f, 0xe7, 0x40, 0x12, 0x77, 0xca, 0xa5, 0xaf }, + { 0xc8, 0xaa, 0xb5, 0xcd, 0x01, 0x60, 0xae, 0x78, + 0xcd, 0x2e, 0x8a, 0xc5, 0xfb, 0x0e, 0x09, 0x3c, + 0xdb, 0x5c, 0x4b, 0x60, 0x52, 0xa0, 0xa9, 0x7b, + 0xb0, 0x42, 0x16, 0x82, 0x6f, 0xa7, 0xa4, 0x37 }, + { 0xff, 0x68, 0xca, 0x40, 0x35, 0xbf, 0xeb, 0x43, + 0xfb, 0xf1, 0x45, 0xfd, 0xdd, 0x5e, 0x43, 0xf1, + 0xce, 0xa5, 0x4f, 0x11, 0xf7, 0xbe, 0xe1, 0x30, + 0x58, 0xf0, 0x27, 0x32, 0x9a, 0x4a, 0x5f, 0xa4 }, + { 0x1d, 0x4e, 0x54, 0x87, 0xae, 0x3c, 0x74, 0x0f, + 0x2b, 0xa6, 0xe5, 0x41, 0xac, 0x91, 0xbc, 0x2b, + 0xfc, 0xd2, 0x99, 0x9c, 0x51, 0x8d, 0x80, 0x7b, + 0x42, 0x67, 0x48, 0x80, 0x3a, 0x35, 0x0f, 0xd4 }, + { 0x6d, 0x24, 0x4e, 0x1a, 0x06, 0xce, 0x4e, 0xf5, + 0x78, 0xdd, 0x0f, 0x63, 0xaf, 0xf0, 0x93, 0x67, + 0x06, 0x73, 0x51, 0x19, 0xca, 0x9c, 0x8d, 0x22, + 0xd8, 0x6c, 0x80, 0x14, 0x14, 0xab, 0x97, 0x41 }, + { 0xde, 0xcf, 0x73, 0x29, 0xdb, 0xcc, 0x82, 0x7b, + 0x8f, 0xc5, 0x24, 0xc9, 0x43, 0x1e, 0x89, 0x98, + 0x02, 0x9e, 0xce, 0x12, 0xce, 0x93, 0xb7, 0xb2, + 0xf3, 0xe7, 0x69, 0xa9, 0x41, 0xfb, 0x8c, 0xea }, + { 0x2f, 0xaf, 0xcc, 0x0f, 0x2e, 0x63, 0xcb, 0xd0, + 0x77, 0x55, 0xbe, 0x7b, 0x75, 0xec, 0xea, 0x0a, + 0xdf, 0xf9, 0xaa, 0x5e, 0xde, 0x2a, 0x52, 0xfd, + 0xab, 0x4d, 0xfd, 0x03, 0x74, 0xcd, 0x48, 0x3f }, + { 0xaa, 0x85, 0x01, 0x0d, 0xd4, 0x6a, 0x54, 0x6b, + 0x53, 0x5e, 0xf4, 0xcf, 0x5f, 0x07, 0xd6, 0x51, + 0x61, 0xe8, 0x98, 0x28, 0xf3, 0xa7, 0x7d, 0xb7, + 0xb9, 0xb5, 0x6f, 0x0d, 0xf5, 0x9a, 0xae, 0x45 }, + { 0x07, 0xe8, 0xe1, 0xee, 0x73, 0x2c, 0xb0, 0xd3, + 0x56, 0xc9, 0xc0, 0xd1, 0x06, 0x9c, 0x89, 0xd1, + 0x7a, 0xdf, 0x6a, 0x9a, 0x33, 0x4f, 0x74, 0x5e, + 0xc7, 0x86, 0x73, 0x32, 0x54, 0x8c, 0xa8, 0xe9 }, + { 0x0e, 0x01, 0xe8, 0x1c, 0xad, 0xa8, 0x16, 0x2b, + 0xfd, 0x5f, 0x8a, 0x8c, 0x81, 0x8a, 0x6c, 0x69, + 0xfe, 0xdf, 0x02, 0xce, 0xb5, 0x20, 0x85, 0x23, + 0xcb, 0xe5, 0x31, 0x3b, 0x89, 0xca, 0x10, 0x53 }, + { 0x6b, 0xb6, 0xc6, 0x47, 0x26, 0x55, 0x08, 0x43, + 0x99, 0x85, 0x2e, 0x00, 0x24, 0x9f, 0x8c, 0xb2, + 0x47, 0x89, 0x6d, 0x39, 0x2b, 0x02, 0xd7, 0x3b, + 0x7f, 0x0d, 0xd8, 0x18, 0xe1, 0xe2, 0x9b, 0x07 }, + { 0x42, 0xd4, 0x63, 0x6e, 0x20, 0x60, 0xf0, 0x8f, + 0x41, 0xc8, 0x82, 0xe7, 0x6b, 0x39, 0x6b, 0x11, + 0x2e, 0xf6, 0x27, 0xcc, 0x24, 0xc4, 0x3d, 0xd5, + 0xf8, 0x3a, 0x1d, 0x1a, 0x7e, 0xad, 0x71, 0x1a }, + { 0x48, 0x58, 0xc9, 0xa1, 0x88, 0xb0, 0x23, 0x4f, + 0xb9, 0xa8, 0xd4, 0x7d, 0x0b, 0x41, 0x33, 0x65, + 0x0a, 0x03, 0x0b, 0xd0, 0x61, 0x1b, 0x87, 0xc3, + 0x89, 0x2e, 0x94, 0x95, 0x1f, 0x8d, 0xf8, 0x52 }, + { 0x3f, 0xab, 0x3e, 0x36, 0x98, 0x8d, 0x44, 0x5a, + 0x51, 0xc8, 0x78, 0x3e, 0x53, 0x1b, 0xe3, 0xa0, + 0x2b, 0xe4, 0x0c, 0xd0, 0x47, 0x96, 0xcf, 0xb6, + 0x1d, 0x40, 0x34, 0x74, 0x42, 0xd3, 0xf7, 0x94 }, + { 0xeb, 0xab, 0xc4, 0x96, 0x36, 0xbd, 0x43, 0x3d, + 0x2e, 0xc8, 0xf0, 0xe5, 0x18, 0x73, 0x2e, 0xf8, + 0xfa, 0x21, 0xd4, 0xd0, 0x71, 0xcc, 0x3b, 0xc4, + 0x6c, 0xd7, 0x9f, 0xa3, 0x8a, 0x28, 0xb8, 0x10 }, + { 0xa1, 0xd0, 0x34, 0x35, 0x23, 0xb8, 0x93, 0xfc, + 0xa8, 0x4f, 0x47, 0xfe, 0xb4, 0xa6, 0x4d, 0x35, + 0x0a, 0x17, 0xd8, 0xee, 0xf5, 0x49, 0x7e, 0xce, + 0x69, 0x7d, 0x02, 0xd7, 0x91, 0x78, 0xb5, 0x91 }, + { 0x26, 0x2e, 0xbf, 0xd9, 0x13, 0x0b, 0x7d, 0x28, + 0x76, 0x0d, 0x08, 0xef, 0x8b, 0xfd, 0x3b, 0x86, + 0xcd, 0xd3, 0xb2, 0x11, 0x3d, 0x2c, 0xae, 0xf7, + 0xea, 0x95, 0x1a, 0x30, 0x3d, 0xfa, 0x38, 0x46 }, + { 0xf7, 0x61, 0x58, 0xed, 0xd5, 0x0a, 0x15, 0x4f, + 0xa7, 0x82, 0x03, 0xed, 0x23, 0x62, 0x93, 0x2f, + 0xcb, 0x82, 0x53, 0xaa, 0xe3, 0x78, 0x90, 0x3e, + 0xde, 0xd1, 0xe0, 0x3f, 0x70, 0x21, 0xa2, 0x57 }, + { 0x26, 0x17, 0x8e, 0x95, 0x0a, 0xc7, 0x22, 0xf6, + 0x7a, 0xe5, 0x6e, 0x57, 0x1b, 0x28, 0x4c, 0x02, + 0x07, 0x68, 0x4a, 0x63, 0x34, 0xa1, 0x77, 0x48, + 0xa9, 0x4d, 0x26, 0x0b, 0xc5, 0xf5, 0x52, 0x74 }, + { 0xc3, 0x78, 0xd1, 0xe4, 0x93, 0xb4, 0x0e, 0xf1, + 0x1f, 0xe6, 0xa1, 0x5d, 0x9c, 0x27, 0x37, 0xa3, + 0x78, 0x09, 0x63, 0x4c, 0x5a, 0xba, 0xd5, 0xb3, + 0x3d, 0x7e, 0x39, 0x3b, 0x4a, 0xe0, 0x5d, 0x03 }, + { 0x98, 0x4b, 0xd8, 0x37, 0x91, 0x01, 0xbe, 0x8f, + 0xd8, 0x06, 0x12, 0xd8, 0xea, 0x29, 0x59, 0xa7, + 0x86, 0x5e, 0xc9, 0x71, 0x85, 0x23, 0x55, 0x01, + 0x07, 0xae, 0x39, 0x38, 0xdf, 0x32, 0x01, 0x1b }, + { 0xc6, 0xf2, 0x5a, 0x81, 0x2a, 0x14, 0x48, 0x58, + 0xac, 0x5c, 0xed, 0x37, 0xa9, 0x3a, 0x9f, 0x47, + 0x59, 0xba, 0x0b, 0x1c, 0x0f, 0xdc, 0x43, 0x1d, + 0xce, 0x35, 0xf9, 0xec, 0x1f, 0x1f, 0x4a, 0x99 }, + { 0x92, 0x4c, 0x75, 0xc9, 0x44, 0x24, 0xff, 0x75, + 0xe7, 0x4b, 0x8b, 0x4e, 0x94, 0x35, 0x89, 0x58, + 0xb0, 0x27, 0xb1, 0x71, 0xdf, 0x5e, 0x57, 0x89, + 0x9a, 0xd0, 0xd4, 0xda, 0xc3, 0x73, 0x53, 0xb6 }, + { 0x0a, 0xf3, 0x58, 0x92, 0xa6, 0x3f, 0x45, 0x93, + 0x1f, 0x68, 0x46, 0xed, 0x19, 0x03, 0x61, 0xcd, + 0x07, 0x30, 0x89, 0xe0, 0x77, 0x16, 0x57, 0x14, + 0xb5, 0x0b, 0x81, 0xa2, 0xe3, 0xdd, 0x9b, 0xa1 }, + { 0xcc, 0x80, 0xce, 0xfb, 0x26, 0xc3, 0xb2, 0xb0, + 0xda, 0xef, 0x23, 0x3e, 0x60, 0x6d, 0x5f, 0xfc, + 0x80, 0xfa, 0x17, 0x42, 0x7d, 0x18, 0xe3, 0x04, + 0x89, 0x67, 0x3e, 0x06, 0xef, 0x4b, 0x87, 0xf7 }, + { 0xc2, 0xf8, 0xc8, 0x11, 0x74, 0x47, 0xf3, 0x97, + 0x8b, 0x08, 0x18, 0xdc, 0xf6, 0xf7, 0x01, 0x16, + 0xac, 0x56, 0xfd, 0x18, 0x4d, 0xd1, 0x27, 0x84, + 0x94, 0xe1, 0x03, 0xfc, 0x6d, 0x74, 0xa8, 0x87 }, + { 0xbd, 0xec, 0xf6, 0xbf, 0xc1, 0xba, 0x0d, 0xf6, + 0xe8, 0x62, 0xc8, 0x31, 0x99, 0x22, 0x07, 0x79, + 0x6a, 0xcc, 0x79, 0x79, 0x68, 0x35, 0x88, 0x28, + 0xc0, 0x6e, 0x7a, 0x51, 0xe0, 0x90, 0x09, 0x8f }, + { 0x24, 0xd1, 0xa2, 0x6e, 0x3d, 0xab, 0x02, 0xfe, + 0x45, 0x72, 0xd2, 0xaa, 0x7d, 0xbd, 0x3e, 0xc3, + 0x0f, 0x06, 0x93, 0xdb, 0x26, 0xf2, 0x73, 0xd0, + 0xab, 0x2c, 0xb0, 0xc1, 0x3b, 0x5e, 0x64, 0x51 }, + { 0xec, 0x56, 0xf5, 0x8b, 0x09, 0x29, 0x9a, 0x30, + 0x0b, 0x14, 0x05, 0x65, 0xd7, 0xd3, 0xe6, 0x87, + 0x82, 0xb6, 0xe2, 0xfb, 0xeb, 0x4b, 0x7e, 0xa9, + 0x7a, 0xc0, 0x57, 0x98, 0x90, 0x61, 0xdd, 0x3f }, + { 0x11, 0xa4, 0x37, 0xc1, 0xab, 0xa3, 0xc1, 0x19, + 0xdd, 0xfa, 0xb3, 0x1b, 0x3e, 0x8c, 0x84, 0x1d, + 0xee, 0xeb, 0x91, 0x3e, 0xf5, 0x7f, 0x7e, 0x48, + 0xf2, 0xc9, 0xcf, 0x5a, 0x28, 0xfa, 0x42, 0xbc }, + { 0x53, 0xc7, 0xe6, 0x11, 0x4b, 0x85, 0x0a, 0x2c, + 0xb4, 0x96, 0xc9, 0xb3, 0xc6, 0x9a, 0x62, 0x3e, + 0xae, 0xa2, 0xcb, 0x1d, 0x33, 0xdd, 0x81, 0x7e, + 0x47, 0x65, 0xed, 0xaa, 0x68, 0x23, 0xc2, 0x28 }, + { 0x15, 0x4c, 0x3e, 0x96, 0xfe, 0xe5, 0xdb, 0x14, + 0xf8, 0x77, 0x3e, 0x18, 0xaf, 0x14, 0x85, 0x79, + 0x13, 0x50, 0x9d, 0xa9, 0x99, 0xb4, 0x6c, 0xdd, + 0x3d, 0x4c, 0x16, 0x97, 0x60, 0xc8, 0x3a, 0xd2 }, + { 0x40, 0xb9, 0x91, 0x6f, 0x09, 0x3e, 0x02, 0x7a, + 0x87, 0x86, 0x64, 0x18, 0x18, 0x92, 0x06, 0x20, + 0x47, 0x2f, 0xbc, 0xf6, 0x8f, 0x70, 0x1d, 0x1b, + 0x68, 0x06, 0x32, 0xe6, 0x99, 0x6b, 0xde, 0xd3 }, + { 0x24, 0xc4, 0xcb, 0xba, 0x07, 0x11, 0x98, 0x31, + 0xa7, 0x26, 0xb0, 0x53, 0x05, 0xd9, 0x6d, 0xa0, + 0x2f, 0xf8, 0xb1, 0x48, 0xf0, 0xda, 0x44, 0x0f, + 0xe2, 0x33, 0xbc, 0xaa, 0x32, 0xc7, 0x2f, 0x6f }, + { 0x5d, 0x20, 0x15, 0x10, 0x25, 0x00, 0x20, 0xb7, + 0x83, 0x68, 0x96, 0x88, 0xab, 0xbf, 0x8e, 0xcf, + 0x25, 0x94, 0xa9, 0x6a, 0x08, 0xf2, 0xbf, 0xec, + 0x6c, 0xe0, 0x57, 0x44, 0x65, 0xdd, 0xed, 0x71 }, + { 0x04, 0x3b, 0x97, 0xe3, 0x36, 0xee, 0x6f, 0xdb, + 0xbe, 0x2b, 0x50, 0xf2, 0x2a, 0xf8, 0x32, 0x75, + 0xa4, 0x08, 0x48, 0x05, 0xd2, 0xd5, 0x64, 0x59, + 0x62, 0x45, 0x4b, 0x6c, 0x9b, 0x80, 0x53, 0xa0 }, + { 0x56, 0x48, 0x35, 0xcb, 0xae, 0xa7, 0x74, 0x94, + 0x85, 0x68, 0xbe, 0x36, 0xcf, 0x52, 0xfc, 0xdd, + 0x83, 0x93, 0x4e, 0xb0, 0xa2, 0x75, 0x12, 0xdb, + 0xe3, 0xe2, 0xdb, 0x47, 0xb9, 0xe6, 0x63, 0x5a }, + { 0xf2, 0x1c, 0x33, 0xf4, 0x7b, 0xde, 0x40, 0xa2, + 0xa1, 0x01, 0xc9, 0xcd, 0xe8, 0x02, 0x7a, 0xaf, + 0x61, 0xa3, 0x13, 0x7d, 0xe2, 0x42, 0x2b, 0x30, + 0x03, 0x5a, 0x04, 0xc2, 0x70, 0x89, 0x41, 0x83 }, + { 0x9d, 0xb0, 0xef, 0x74, 0xe6, 0x6c, 0xbb, 0x84, + 0x2e, 0xb0, 0xe0, 0x73, 0x43, 0xa0, 0x3c, 0x5c, + 0x56, 0x7e, 0x37, 0x2b, 0x3f, 0x23, 0xb9, 0x43, + 0xc7, 0x88, 0xa4, 0xf2, 0x50, 0xf6, 0x78, 0x91 }, + { 0xab, 0x8d, 0x08, 0x65, 0x5f, 0xf1, 0xd3, 0xfe, + 0x87, 0x58, 0xd5, 0x62, 0x23, 0x5f, 0xd2, 0x3e, + 0x7c, 0xf9, 0xdc, 0xaa, 0xd6, 0x58, 0x87, 0x2a, + 0x49, 0xe5, 0xd3, 0x18, 0x3b, 0x6c, 0xce, 0xbd }, + { 0x6f, 0x27, 0xf7, 0x7e, 0x7b, 0xcf, 0x46, 0xa1, + 0xe9, 0x63, 0xad, 0xe0, 0x30, 0x97, 0x33, 0x54, + 0x30, 0x31, 0xdc, 0xcd, 0xd4, 0x7c, 0xaa, 0xc1, + 0x74, 0xd7, 0xd2, 0x7c, 0xe8, 0x07, 0x7e, 0x8b }, + { 0xe3, 0xcd, 0x54, 0xda, 0x7e, 0x44, 0x4c, 0xaa, + 0x62, 0x07, 0x56, 0x95, 0x25, 0xa6, 0x70, 0xeb, + 0xae, 0x12, 0x78, 0xde, 0x4e, 0x3f, 0xe2, 0x68, + 0x4b, 0x3e, 0x33, 0xf5, 0xef, 0x90, 0xcc, 0x1b }, + { 0xb2, 0xc3, 0xe3, 0x3a, 0x51, 0xd2, 0x2c, 0x4c, + 0x08, 0xfc, 0x09, 0x89, 0xc8, 0x73, 0xc9, 0xcc, + 0x41, 0x50, 0x57, 0x9b, 0x1e, 0x61, 0x63, 0xfa, + 0x69, 0x4a, 0xd5, 0x1d, 0x53, 0xd7, 0x12, 0xdc }, + { 0xbe, 0x7f, 0xda, 0x98, 0x3e, 0x13, 0x18, 0x9b, + 0x4c, 0x77, 0xe0, 0xa8, 0x09, 0x20, 0xb6, 0xe0, + 0xe0, 0xea, 0x80, 0xc3, 0xb8, 0x4d, 0xbe, 0x7e, + 0x71, 0x17, 0xd2, 0x53, 0xf4, 0x81, 0x12, 0xf4 }, + { 0xb6, 0x00, 0x8c, 0x28, 0xfa, 0xe0, 0x8a, 0xa4, + 0x27, 0xe5, 0xbd, 0x3a, 0xad, 0x36, 0xf1, 0x00, + 0x21, 0xf1, 0x6c, 0x77, 0xcf, 0xea, 0xbe, 0xd0, + 0x7f, 0x97, 0xcc, 0x7d, 0xc1, 0xf1, 0x28, 0x4a }, + { 0x6e, 0x4e, 0x67, 0x60, 0xc5, 0x38, 0xf2, 0xe9, + 0x7b, 0x3a, 0xdb, 0xfb, 0xbc, 0xde, 0x57, 0xf8, + 0x96, 0x6b, 0x7e, 0xa8, 0xfc, 0xb5, 0xbf, 0x7e, + 0xfe, 0xc9, 0x13, 0xfd, 0x2a, 0x2b, 0x0c, 0x55 }, + { 0x4a, 0xe5, 0x1f, 0xd1, 0x83, 0x4a, 0xa5, 0xbd, + 0x9a, 0x6f, 0x7e, 0xc3, 0x9f, 0xc6, 0x63, 0x33, + 0x8d, 0xc5, 0xd2, 0xe2, 0x07, 0x61, 0x56, 0x6d, + 0x90, 0xcc, 0x68, 0xb1, 0xcb, 0x87, 0x5e, 0xd8 }, + { 0xb6, 0x73, 0xaa, 0xd7, 0x5a, 0xb1, 0xfd, 0xb5, + 0x40, 0x1a, 0xbf, 0xa1, 0xbf, 0x89, 0xf3, 0xad, + 0xd2, 0xeb, 0xc4, 0x68, 0xdf, 0x36, 0x24, 0xa4, + 0x78, 0xf4, 0xfe, 0x85, 0x9d, 0x8d, 0x55, 0xe2 }, + { 0x13, 0xc9, 0x47, 0x1a, 0x98, 0x55, 0x91, 0x35, + 0x39, 0x83, 0x66, 0x60, 0x39, 0x8d, 0xa0, 0xf3, + 0xf9, 0x9a, 0xda, 0x08, 0x47, 0x9c, 0x69, 0xd1, + 0xb7, 0xfc, 0xaa, 0x34, 0x61, 0xdd, 0x7e, 0x59 }, + { 0x2c, 0x11, 0xf4, 0xa7, 0xf9, 0x9a, 0x1d, 0x23, + 0xa5, 0x8b, 0xb6, 0x36, 0x35, 0x0f, 0xe8, 0x49, + 0xf2, 0x9c, 0xba, 0xc1, 0xb2, 0xa1, 0x11, 0x2d, + 0x9f, 0x1e, 0xd5, 0xbc, 0x5b, 0x31, 0x3c, 0xcd }, + { 0xc7, 0xd3, 0xc0, 0x70, 0x6b, 0x11, 0xae, 0x74, + 0x1c, 0x05, 0xa1, 0xef, 0x15, 0x0d, 0xd6, 0x5b, + 0x54, 0x94, 0xd6, 0xd5, 0x4c, 0x9a, 0x86, 0xe2, + 0x61, 0x78, 0x54, 0xe6, 0xae, 0xee, 0xbb, 0xd9 }, + { 0x19, 0x4e, 0x10, 0xc9, 0x38, 0x93, 0xaf, 0xa0, + 0x64, 0xc3, 0xac, 0x04, 0xc0, 0xdd, 0x80, 0x8d, + 0x79, 0x1c, 0x3d, 0x4b, 0x75, 0x56, 0xe8, 0x9d, + 0x8d, 0x9c, 0xb2, 0x25, 0xc4, 0xb3, 0x33, 0x39 }, + { 0x6f, 0xc4, 0x98, 0x8b, 0x8f, 0x78, 0x54, 0x6b, + 0x16, 0x88, 0x99, 0x18, 0x45, 0x90, 0x8f, 0x13, + 0x4b, 0x6a, 0x48, 0x2e, 0x69, 0x94, 0xb3, 0xd4, + 0x83, 0x17, 0xbf, 0x08, 0xdb, 0x29, 0x21, 0x85 }, + { 0x56, 0x65, 0xbe, 0xb8, 0xb0, 0x95, 0x55, 0x25, + 0x81, 0x3b, 0x59, 0x81, 0xcd, 0x14, 0x2e, 0xd4, + 0xd0, 0x3f, 0xba, 0x38, 0xa6, 0xf3, 0xe5, 0xad, + 0x26, 0x8e, 0x0c, 0xc2, 0x70, 0xd1, 0xcd, 0x11 }, + { 0xb8, 0x83, 0xd6, 0x8f, 0x5f, 0xe5, 0x19, 0x36, + 0x43, 0x1b, 0xa4, 0x25, 0x67, 0x38, 0x05, 0x3b, + 0x1d, 0x04, 0x26, 0xd4, 0xcb, 0x64, 0xb1, 0x6e, + 0x83, 0xba, 0xdc, 0x5e, 0x9f, 0xbe, 0x3b, 0x81 }, + { 0x53, 0xe7, 0xb2, 0x7e, 0xa5, 0x9c, 0x2f, 0x6d, + 0xbb, 0x50, 0x76, 0x9e, 0x43, 0x55, 0x4d, 0xf3, + 0x5a, 0xf8, 0x9f, 0x48, 0x22, 0xd0, 0x46, 0x6b, + 0x00, 0x7d, 0xd6, 0xf6, 0xde, 0xaf, 0xff, 0x02 }, + { 0x1f, 0x1a, 0x02, 0x29, 0xd4, 0x64, 0x0f, 0x01, + 0x90, 0x15, 0x88, 0xd9, 0xde, 0xc2, 0x2d, 0x13, + 0xfc, 0x3e, 0xb3, 0x4a, 0x61, 0xb3, 0x29, 0x38, + 0xef, 0xbf, 0x53, 0x34, 0xb2, 0x80, 0x0a, 0xfa }, + { 0xc2, 0xb4, 0x05, 0xaf, 0xa0, 0xfa, 0x66, 0x68, + 0x85, 0x2a, 0xee, 0x4d, 0x88, 0x04, 0x08, 0x53, + 0xfa, 0xb8, 0x00, 0xe7, 0x2b, 0x57, 0x58, 0x14, + 0x18, 0xe5, 0x50, 0x6f, 0x21, 0x4c, 0x7d, 0x1f }, + { 0xc0, 0x8a, 0xa1, 0xc2, 0x86, 0xd7, 0x09, 0xfd, + 0xc7, 0x47, 0x37, 0x44, 0x97, 0x71, 0x88, 0xc8, + 0x95, 0xba, 0x01, 0x10, 0x14, 0x24, 0x7e, 0x4e, + 0xfa, 0x8d, 0x07, 0xe7, 0x8f, 0xec, 0x69, 0x5c }, + { 0xf0, 0x3f, 0x57, 0x89, 0xd3, 0x33, 0x6b, 0x80, + 0xd0, 0x02, 0xd5, 0x9f, 0xdf, 0x91, 0x8b, 0xdb, + 0x77, 0x5b, 0x00, 0x95, 0x6e, 0xd5, 0x52, 0x8e, + 0x86, 0xaa, 0x99, 0x4a, 0xcb, 0x38, 0xfe, 0x2d } +}; + +static const u8 blake2s_keyed_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b, + 0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b, + 0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a, + 0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49 }, + { 0x40, 0xd1, 0x5f, 0xee, 0x7c, 0x32, 0x88, 0x30, + 0x16, 0x6a, 0xc3, 0xf9, 0x18, 0x65, 0x0f, 0x80, + 0x7e, 0x7e, 0x01, 0xe1, 0x77, 0x25, 0x8c, 0xdc, + 0x0a, 0x39, 0xb1, 0x1f, 0x59, 0x80, 0x66, 0xf1 }, + { 0x6b, 0xb7, 0x13, 0x00, 0x64, 0x4c, 0xd3, 0x99, + 0x1b, 0x26, 0xcc, 0xd4, 0xd2, 0x74, 0xac, 0xd1, + 0xad, 0xea, 0xb8, 0xb1, 0xd7, 0x91, 0x45, 0x46, + 0xc1, 0x19, 0x8b, 0xbe, 0x9f, 0xc9, 0xd8, 0x03 }, + { 0x1d, 0x22, 0x0d, 0xbe, 0x2e, 0xe1, 0x34, 0x66, + 0x1f, 0xdf, 0x6d, 0x9e, 0x74, 0xb4, 0x17, 0x04, + 0x71, 0x05, 0x56, 0xf2, 0xf6, 0xe5, 0xa0, 0x91, + 0xb2, 0x27, 0x69, 0x74, 0x45, 0xdb, 0xea, 0x6b }, + { 0xf6, 0xc3, 0xfb, 0xad, 0xb4, 0xcc, 0x68, 0x7a, + 0x00, 0x64, 0xa5, 0xbe, 0x6e, 0x79, 0x1b, 0xec, + 0x63, 0xb8, 0x68, 0xad, 0x62, 0xfb, 0xa6, 0x1b, + 0x37, 0x57, 0xef, 0x9c, 0xa5, 0x2e, 0x05, 0xb2 }, + { 0x49, 0xc1, 0xf2, 0x11, 0x88, 0xdf, 0xd7, 0x69, + 0xae, 0xa0, 0xe9, 0x11, 0xdd, 0x6b, 0x41, 0xf1, + 0x4d, 0xab, 0x10, 0x9d, 0x2b, 0x85, 0x97, 0x7a, + 0xa3, 0x08, 0x8b, 0x5c, 0x70, 0x7e, 0x85, 0x98 }, + { 0xfd, 0xd8, 0x99, 0x3d, 0xcd, 0x43, 0xf6, 0x96, + 0xd4, 0x4f, 0x3c, 0xea, 0x0f, 0xf3, 0x53, 0x45, + 0x23, 0x4e, 0xc8, 0xee, 0x08, 0x3e, 0xb3, 0xca, + 0xda, 0x01, 0x7c, 0x7f, 0x78, 0xc1, 0x71, 0x43 }, + { 0xe6, 0xc8, 0x12, 0x56, 0x37, 0x43, 0x8d, 0x09, + 0x05, 0xb7, 0x49, 0xf4, 0x65, 0x60, 0xac, 0x89, + 0xfd, 0x47, 0x1c, 0xf8, 0x69, 0x2e, 0x28, 0xfa, + 0xb9, 0x82, 0xf7, 0x3f, 0x01, 0x9b, 0x83, 0xa9 }, + { 0x19, 0xfc, 0x8c, 0xa6, 0x97, 0x9d, 0x60, 0xe6, + 0xed, 0xd3, 0xb4, 0x54, 0x1e, 0x2f, 0x96, 0x7c, + 0xed, 0x74, 0x0d, 0xf6, 0xec, 0x1e, 0xae, 0xbb, + 0xfe, 0x81, 0x38, 0x32, 0xe9, 0x6b, 0x29, 0x74 }, + { 0xa6, 0xad, 0x77, 0x7c, 0xe8, 0x81, 0xb5, 0x2b, + 0xb5, 0xa4, 0x42, 0x1a, 0xb6, 0xcd, 0xd2, 0xdf, + 0xba, 0x13, 0xe9, 0x63, 0x65, 0x2d, 0x4d, 0x6d, + 0x12, 0x2a, 0xee, 0x46, 0x54, 0x8c, 0x14, 0xa7 }, + { 0xf5, 0xc4, 0xb2, 0xba, 0x1a, 0x00, 0x78, 0x1b, + 0x13, 0xab, 0xa0, 0x42, 0x52, 0x42, 0xc6, 0x9c, + 0xb1, 0x55, 0x2f, 0x3f, 0x71, 0xa9, 0xa3, 0xbb, + 0x22, 0xb4, 0xa6, 0xb4, 0x27, 0x7b, 0x46, 0xdd }, + { 0xe3, 0x3c, 0x4c, 0x9b, 0xd0, 0xcc, 0x7e, 0x45, + 0xc8, 0x0e, 0x65, 0xc7, 0x7f, 0xa5, 0x99, 0x7f, + 0xec, 0x70, 0x02, 0x73, 0x85, 0x41, 0x50, 0x9e, + 0x68, 0xa9, 0x42, 0x38, 0x91, 0xe8, 0x22, 0xa3 }, + { 0xfb, 0xa1, 0x61, 0x69, 0xb2, 0xc3, 0xee, 0x10, + 0x5b, 0xe6, 0xe1, 0xe6, 0x50, 0xe5, 0xcb, 0xf4, + 0x07, 0x46, 0xb6, 0x75, 0x3d, 0x03, 0x6a, 0xb5, + 0x51, 0x79, 0x01, 0x4a, 0xd7, 0xef, 0x66, 0x51 }, + { 0xf5, 0xc4, 0xbe, 0xc6, 0xd6, 0x2f, 0xc6, 0x08, + 0xbf, 0x41, 0xcc, 0x11, 0x5f, 0x16, 0xd6, 0x1c, + 0x7e, 0xfd, 0x3f, 0xf6, 0xc6, 0x56, 0x92, 0xbb, + 0xe0, 0xaf, 0xff, 0xb1, 0xfe, 0xde, 0x74, 0x75 }, + { 0xa4, 0x86, 0x2e, 0x76, 0xdb, 0x84, 0x7f, 0x05, + 0xba, 0x17, 0xed, 0xe5, 0xda, 0x4e, 0x7f, 0x91, + 0xb5, 0x92, 0x5c, 0xf1, 0xad, 0x4b, 0xa1, 0x27, + 0x32, 0xc3, 0x99, 0x57, 0x42, 0xa5, 0xcd, 0x6e }, + { 0x65, 0xf4, 0xb8, 0x60, 0xcd, 0x15, 0xb3, 0x8e, + 0xf8, 0x14, 0xa1, 0xa8, 0x04, 0x31, 0x4a, 0x55, + 0xbe, 0x95, 0x3c, 0xaa, 0x65, 0xfd, 0x75, 0x8a, + 0xd9, 0x89, 0xff, 0x34, 0xa4, 0x1c, 0x1e, 0xea }, + { 0x19, 0xba, 0x23, 0x4f, 0x0a, 0x4f, 0x38, 0x63, + 0x7d, 0x18, 0x39, 0xf9, 0xd9, 0xf7, 0x6a, 0xd9, + 0x1c, 0x85, 0x22, 0x30, 0x71, 0x43, 0xc9, 0x7d, + 0x5f, 0x93, 0xf6, 0x92, 0x74, 0xce, 0xc9, 0xa7 }, + { 0x1a, 0x67, 0x18, 0x6c, 0xa4, 0xa5, 0xcb, 0x8e, + 0x65, 0xfc, 0xa0, 0xe2, 0xec, 0xbc, 0x5d, 0xdc, + 0x14, 0xae, 0x38, 0x1b, 0xb8, 0xbf, 0xfe, 0xb9, + 0xe0, 0xa1, 0x03, 0x44, 0x9e, 0x3e, 0xf0, 0x3c }, + { 0xaf, 0xbe, 0xa3, 0x17, 0xb5, 0xa2, 0xe8, 0x9c, + 0x0b, 0xd9, 0x0c, 0xcf, 0x5d, 0x7f, 0xd0, 0xed, + 0x57, 0xfe, 0x58, 0x5e, 0x4b, 0xe3, 0x27, 0x1b, + 0x0a, 0x6b, 0xf0, 0xf5, 0x78, 0x6b, 0x0f, 0x26 }, + { 0xf1, 0xb0, 0x15, 0x58, 0xce, 0x54, 0x12, 0x62, + 0xf5, 0xec, 0x34, 0x29, 0x9d, 0x6f, 0xb4, 0x09, + 0x00, 0x09, 0xe3, 0x43, 0x4b, 0xe2, 0xf4, 0x91, + 0x05, 0xcf, 0x46, 0xaf, 0x4d, 0x2d, 0x41, 0x24 }, + { 0x13, 0xa0, 0xa0, 0xc8, 0x63, 0x35, 0x63, 0x5e, + 0xaa, 0x74, 0xca, 0x2d, 0x5d, 0x48, 0x8c, 0x79, + 0x7b, 0xbb, 0x4f, 0x47, 0xdc, 0x07, 0x10, 0x50, + 0x15, 0xed, 0x6a, 0x1f, 0x33, 0x09, 0xef, 0xce }, + { 0x15, 0x80, 0xaf, 0xee, 0xbe, 0xbb, 0x34, 0x6f, + 0x94, 0xd5, 0x9f, 0xe6, 0x2d, 0xa0, 0xb7, 0x92, + 0x37, 0xea, 0xd7, 0xb1, 0x49, 0x1f, 0x56, 0x67, + 0xa9, 0x0e, 0x45, 0xed, 0xf6, 0xca, 0x8b, 0x03 }, + { 0x20, 0xbe, 0x1a, 0x87, 0x5b, 0x38, 0xc5, 0x73, + 0xdd, 0x7f, 0xaa, 0xa0, 0xde, 0x48, 0x9d, 0x65, + 0x5c, 0x11, 0xef, 0xb6, 0xa5, 0x52, 0x69, 0x8e, + 0x07, 0xa2, 0xd3, 0x31, 0xb5, 0xf6, 0x55, 0xc3 }, + { 0xbe, 0x1f, 0xe3, 0xc4, 0xc0, 0x40, 0x18, 0xc5, + 0x4c, 0x4a, 0x0f, 0x6b, 0x9a, 0x2e, 0xd3, 0xc5, + 0x3a, 0xbe, 0x3a, 0x9f, 0x76, 0xb4, 0xd2, 0x6d, + 0xe5, 0x6f, 0xc9, 0xae, 0x95, 0x05, 0x9a, 0x99 }, + { 0xe3, 0xe3, 0xac, 0xe5, 0x37, 0xeb, 0x3e, 0xdd, + 0x84, 0x63, 0xd9, 0xad, 0x35, 0x82, 0xe1, 0x3c, + 0xf8, 0x65, 0x33, 0xff, 0xde, 0x43, 0xd6, 0x68, + 0xdd, 0x2e, 0x93, 0xbb, 0xdb, 0xd7, 0x19, 0x5a }, + { 0x11, 0x0c, 0x50, 0xc0, 0xbf, 0x2c, 0x6e, 0x7a, + 0xeb, 0x7e, 0x43, 0x5d, 0x92, 0xd1, 0x32, 0xab, + 0x66, 0x55, 0x16, 0x8e, 0x78, 0xa2, 0xde, 0xcd, + 0xec, 0x33, 0x30, 0x77, 0x76, 0x84, 0xd9, 0xc1 }, + { 0xe9, 0xba, 0x8f, 0x50, 0x5c, 0x9c, 0x80, 0xc0, + 0x86, 0x66, 0xa7, 0x01, 0xf3, 0x36, 0x7e, 0x6c, + 0xc6, 0x65, 0xf3, 0x4b, 0x22, 0xe7, 0x3c, 0x3c, + 0x04, 0x17, 0xeb, 0x1c, 0x22, 0x06, 0x08, 0x2f }, + { 0x26, 0xcd, 0x66, 0xfc, 0xa0, 0x23, 0x79, 0xc7, + 0x6d, 0xf1, 0x23, 0x17, 0x05, 0x2b, 0xca, 0xfd, + 0x6c, 0xd8, 0xc3, 0xa7, 0xb8, 0x90, 0xd8, 0x05, + 0xf3, 0x6c, 0x49, 0x98, 0x97, 0x82, 0x43, 0x3a }, + { 0x21, 0x3f, 0x35, 0x96, 0xd6, 0xe3, 0xa5, 0xd0, + 0xe9, 0x93, 0x2c, 0xd2, 0x15, 0x91, 0x46, 0x01, + 0x5e, 0x2a, 0xbc, 0x94, 0x9f, 0x47, 0x29, 0xee, + 0x26, 0x32, 0xfe, 0x1e, 0xdb, 0x78, 0xd3, 0x37 }, + { 0x10, 0x15, 0xd7, 0x01, 0x08, 0xe0, 0x3b, 0xe1, + 0xc7, 0x02, 0xfe, 0x97, 0x25, 0x36, 0x07, 0xd1, + 0x4a, 0xee, 0x59, 0x1f, 0x24, 0x13, 0xea, 0x67, + 0x87, 0x42, 0x7b, 0x64, 0x59, 0xff, 0x21, 0x9a }, + { 0x3c, 0xa9, 0x89, 0xde, 0x10, 0xcf, 0xe6, 0x09, + 0x90, 0x94, 0x72, 0xc8, 0xd3, 0x56, 0x10, 0x80, + 0x5b, 0x2f, 0x97, 0x77, 0x34, 0xcf, 0x65, 0x2c, + 0xc6, 0x4b, 0x3b, 0xfc, 0x88, 0x2d, 0x5d, 0x89 }, + { 0xb6, 0x15, 0x6f, 0x72, 0xd3, 0x80, 0xee, 0x9e, + 0xa6, 0xac, 0xd1, 0x90, 0x46, 0x4f, 0x23, 0x07, + 0xa5, 0xc1, 0x79, 0xef, 0x01, 0xfd, 0x71, 0xf9, + 0x9f, 0x2d, 0x0f, 0x7a, 0x57, 0x36, 0x0a, 0xea }, + { 0xc0, 0x3b, 0xc6, 0x42, 0xb2, 0x09, 0x59, 0xcb, + 0xe1, 0x33, 0xa0, 0x30, 0x3e, 0x0c, 0x1a, 0xbf, + 0xf3, 0xe3, 0x1e, 0xc8, 0xe1, 0xa3, 0x28, 0xec, + 0x85, 0x65, 0xc3, 0x6d, 0xec, 0xff, 0x52, 0x65 }, + { 0x2c, 0x3e, 0x08, 0x17, 0x6f, 0x76, 0x0c, 0x62, + 0x64, 0xc3, 0xa2, 0xcd, 0x66, 0xfe, 0xc6, 0xc3, + 0xd7, 0x8d, 0xe4, 0x3f, 0xc1, 0x92, 0x45, 0x7b, + 0x2a, 0x4a, 0x66, 0x0a, 0x1e, 0x0e, 0xb2, 0x2b }, + { 0xf7, 0x38, 0xc0, 0x2f, 0x3c, 0x1b, 0x19, 0x0c, + 0x51, 0x2b, 0x1a, 0x32, 0xde, 0xab, 0xf3, 0x53, + 0x72, 0x8e, 0x0e, 0x9a, 0xb0, 0x34, 0x49, 0x0e, + 0x3c, 0x34, 0x09, 0x94, 0x6a, 0x97, 0xae, 0xec }, + { 0x8b, 0x18, 0x80, 0xdf, 0x30, 0x1c, 0xc9, 0x63, + 0x41, 0x88, 0x11, 0x08, 0x89, 0x64, 0x83, 0x92, + 0x87, 0xff, 0x7f, 0xe3, 0x1c, 0x49, 0xea, 0x6e, + 0xbd, 0x9e, 0x48, 0xbd, 0xee, 0xe4, 0x97, 0xc5 }, + { 0x1e, 0x75, 0xcb, 0x21, 0xc6, 0x09, 0x89, 0x02, + 0x03, 0x75, 0xf1, 0xa7, 0xa2, 0x42, 0x83, 0x9f, + 0x0b, 0x0b, 0x68, 0x97, 0x3a, 0x4c, 0x2a, 0x05, + 0xcf, 0x75, 0x55, 0xed, 0x5a, 0xae, 0xc4, 0xc1 }, + { 0x62, 0xbf, 0x8a, 0x9c, 0x32, 0xa5, 0xbc, 0xcf, + 0x29, 0x0b, 0x6c, 0x47, 0x4d, 0x75, 0xb2, 0xa2, + 0xa4, 0x09, 0x3f, 0x1a, 0x9e, 0x27, 0x13, 0x94, + 0x33, 0xa8, 0xf2, 0xb3, 0xbc, 0xe7, 0xb8, 0xd7 }, + { 0x16, 0x6c, 0x83, 0x50, 0xd3, 0x17, 0x3b, 0x5e, + 0x70, 0x2b, 0x78, 0x3d, 0xfd, 0x33, 0xc6, 0x6e, + 0xe0, 0x43, 0x27, 0x42, 0xe9, 0xb9, 0x2b, 0x99, + 0x7f, 0xd2, 0x3c, 0x60, 0xdc, 0x67, 0x56, 0xca }, + { 0x04, 0x4a, 0x14, 0xd8, 0x22, 0xa9, 0x0c, 0xac, + 0xf2, 0xf5, 0xa1, 0x01, 0x42, 0x8a, 0xdc, 0x8f, + 0x41, 0x09, 0x38, 0x6c, 0xcb, 0x15, 0x8b, 0xf9, + 0x05, 0xc8, 0x61, 0x8b, 0x8e, 0xe2, 0x4e, 0xc3 }, + { 0x38, 0x7d, 0x39, 0x7e, 0xa4, 0x3a, 0x99, 0x4b, + 0xe8, 0x4d, 0x2d, 0x54, 0x4a, 0xfb, 0xe4, 0x81, + 0xa2, 0x00, 0x0f, 0x55, 0x25, 0x26, 0x96, 0xbb, + 0xa2, 0xc5, 0x0c, 0x8e, 0xbd, 0x10, 0x13, 0x47 }, + { 0x56, 0xf8, 0xcc, 0xf1, 0xf8, 0x64, 0x09, 0xb4, + 0x6c, 0xe3, 0x61, 0x66, 0xae, 0x91, 0x65, 0x13, + 0x84, 0x41, 0x57, 0x75, 0x89, 0xdb, 0x08, 0xcb, + 0xc5, 0xf6, 0x6c, 0xa2, 0x97, 0x43, 0xb9, 0xfd }, + { 0x97, 0x06, 0xc0, 0x92, 0xb0, 0x4d, 0x91, 0xf5, + 0x3d, 0xff, 0x91, 0xfa, 0x37, 0xb7, 0x49, 0x3d, + 0x28, 0xb5, 0x76, 0xb5, 0xd7, 0x10, 0x46, 0x9d, + 0xf7, 0x94, 0x01, 0x66, 0x22, 0x36, 0xfc, 0x03 }, + { 0x87, 0x79, 0x68, 0x68, 0x6c, 0x06, 0x8c, 0xe2, + 0xf7, 0xe2, 0xad, 0xcf, 0xf6, 0x8b, 0xf8, 0x74, + 0x8e, 0xdf, 0x3c, 0xf8, 0x62, 0xcf, 0xb4, 0xd3, + 0x94, 0x7a, 0x31, 0x06, 0x95, 0x80, 0x54, 0xe3 }, + { 0x88, 0x17, 0xe5, 0x71, 0x98, 0x79, 0xac, 0xf7, + 0x02, 0x47, 0x87, 0xec, 0xcd, 0xb2, 0x71, 0x03, + 0x55, 0x66, 0xcf, 0xa3, 0x33, 0xe0, 0x49, 0x40, + 0x7c, 0x01, 0x78, 0xcc, 0xc5, 0x7a, 0x5b, 0x9f }, + { 0x89, 0x38, 0x24, 0x9e, 0x4b, 0x50, 0xca, 0xda, + 0xcc, 0xdf, 0x5b, 0x18, 0x62, 0x13, 0x26, 0xcb, + 0xb1, 0x52, 0x53, 0xe3, 0x3a, 0x20, 0xf5, 0x63, + 0x6e, 0x99, 0x5d, 0x72, 0x47, 0x8d, 0xe4, 0x72 }, + { 0xf1, 0x64, 0xab, 0xba, 0x49, 0x63, 0xa4, 0x4d, + 0x10, 0x72, 0x57, 0xe3, 0x23, 0x2d, 0x90, 0xac, + 0xa5, 0xe6, 0x6a, 0x14, 0x08, 0x24, 0x8c, 0x51, + 0x74, 0x1e, 0x99, 0x1d, 0xb5, 0x22, 0x77, 0x56 }, + { 0xd0, 0x55, 0x63, 0xe2, 0xb1, 0xcb, 0xa0, 0xc4, + 0xa2, 0xa1, 0xe8, 0xbd, 0xe3, 0xa1, 0xa0, 0xd9, + 0xf5, 0xb4, 0x0c, 0x85, 0xa0, 0x70, 0xd6, 0xf5, + 0xfb, 0x21, 0x06, 0x6e, 0xad, 0x5d, 0x06, 0x01 }, + { 0x03, 0xfb, 0xb1, 0x63, 0x84, 0xf0, 0xa3, 0x86, + 0x6f, 0x4c, 0x31, 0x17, 0x87, 0x76, 0x66, 0xef, + 0xbf, 0x12, 0x45, 0x97, 0x56, 0x4b, 0x29, 0x3d, + 0x4a, 0xab, 0x0d, 0x26, 0x9f, 0xab, 0xdd, 0xfa }, + { 0x5f, 0xa8, 0x48, 0x6a, 0xc0, 0xe5, 0x29, 0x64, + 0xd1, 0x88, 0x1b, 0xbe, 0x33, 0x8e, 0xb5, 0x4b, + 0xe2, 0xf7, 0x19, 0x54, 0x92, 0x24, 0x89, 0x20, + 0x57, 0xb4, 0xda, 0x04, 0xba, 0x8b, 0x34, 0x75 }, + { 0xcd, 0xfa, 0xbc, 0xee, 0x46, 0x91, 0x11, 0x11, + 0x23, 0x6a, 0x31, 0x70, 0x8b, 0x25, 0x39, 0xd7, + 0x1f, 0xc2, 0x11, 0xd9, 0xb0, 0x9c, 0x0d, 0x85, + 0x30, 0xa1, 0x1e, 0x1d, 0xbf, 0x6e, 0xed, 0x01 }, + { 0x4f, 0x82, 0xde, 0x03, 0xb9, 0x50, 0x47, 0x93, + 0xb8, 0x2a, 0x07, 0xa0, 0xbd, 0xcd, 0xff, 0x31, + 0x4d, 0x75, 0x9e, 0x7b, 0x62, 0xd2, 0x6b, 0x78, + 0x49, 0x46, 0xb0, 0xd3, 0x6f, 0x91, 0x6f, 0x52 }, + { 0x25, 0x9e, 0xc7, 0xf1, 0x73, 0xbc, 0xc7, 0x6a, + 0x09, 0x94, 0xc9, 0x67, 0xb4, 0xf5, 0xf0, 0x24, + 0xc5, 0x60, 0x57, 0xfb, 0x79, 0xc9, 0x65, 0xc4, + 0xfa, 0xe4, 0x18, 0x75, 0xf0, 0x6a, 0x0e, 0x4c }, + { 0x19, 0x3c, 0xc8, 0xe7, 0xc3, 0xe0, 0x8b, 0xb3, + 0x0f, 0x54, 0x37, 0xaa, 0x27, 0xad, 0xe1, 0xf1, + 0x42, 0x36, 0x9b, 0x24, 0x6a, 0x67, 0x5b, 0x23, + 0x83, 0xe6, 0xda, 0x9b, 0x49, 0xa9, 0x80, 0x9e }, + { 0x5c, 0x10, 0x89, 0x6f, 0x0e, 0x28, 0x56, 0xb2, + 0xa2, 0xee, 0xe0, 0xfe, 0x4a, 0x2c, 0x16, 0x33, + 0x56, 0x5d, 0x18, 0xf0, 0xe9, 0x3e, 0x1f, 0xab, + 0x26, 0xc3, 0x73, 0xe8, 0xf8, 0x29, 0x65, 0x4d }, + { 0xf1, 0x60, 0x12, 0xd9, 0x3f, 0x28, 0x85, 0x1a, + 0x1e, 0xb9, 0x89, 0xf5, 0xd0, 0xb4, 0x3f, 0x3f, + 0x39, 0xca, 0x73, 0xc9, 0xa6, 0x2d, 0x51, 0x81, + 0xbf, 0xf2, 0x37, 0x53, 0x6b, 0xd3, 0x48, 0xc3 }, + { 0x29, 0x66, 0xb3, 0xcf, 0xae, 0x1e, 0x44, 0xea, + 0x99, 0x6d, 0xc5, 0xd6, 0x86, 0xcf, 0x25, 0xfa, + 0x05, 0x3f, 0xb6, 0xf6, 0x72, 0x01, 0xb9, 0xe4, + 0x6e, 0xad, 0xe8, 0x5d, 0x0a, 0xd6, 0xb8, 0x06 }, + { 0xdd, 0xb8, 0x78, 0x24, 0x85, 0xe9, 0x00, 0xbc, + 0x60, 0xbc, 0xf4, 0xc3, 0x3a, 0x6f, 0xd5, 0x85, + 0x68, 0x0c, 0xc6, 0x83, 0xd5, 0x16, 0xef, 0xa0, + 0x3e, 0xb9, 0x98, 0x5f, 0xad, 0x87, 0x15, 0xfb }, + { 0x4c, 0x4d, 0x6e, 0x71, 0xae, 0xa0, 0x57, 0x86, + 0x41, 0x31, 0x48, 0xfc, 0x7a, 0x78, 0x6b, 0x0e, + 0xca, 0xf5, 0x82, 0xcf, 0xf1, 0x20, 0x9f, 0x5a, + 0x80, 0x9f, 0xba, 0x85, 0x04, 0xce, 0x66, 0x2c }, + { 0xfb, 0x4c, 0x5e, 0x86, 0xd7, 0xb2, 0x22, 0x9b, + 0x99, 0xb8, 0xba, 0x6d, 0x94, 0xc2, 0x47, 0xef, + 0x96, 0x4a, 0xa3, 0xa2, 0xba, 0xe8, 0xed, 0xc7, + 0x75, 0x69, 0xf2, 0x8d, 0xbb, 0xff, 0x2d, 0x4e }, + { 0xe9, 0x4f, 0x52, 0x6d, 0xe9, 0x01, 0x96, 0x33, + 0xec, 0xd5, 0x4a, 0xc6, 0x12, 0x0f, 0x23, 0x95, + 0x8d, 0x77, 0x18, 0xf1, 0xe7, 0x71, 0x7b, 0xf3, + 0x29, 0x21, 0x1a, 0x4f, 0xae, 0xed, 0x4e, 0x6d }, + { 0xcb, 0xd6, 0x66, 0x0a, 0x10, 0xdb, 0x3f, 0x23, + 0xf7, 0xa0, 0x3d, 0x4b, 0x9d, 0x40, 0x44, 0xc7, + 0x93, 0x2b, 0x28, 0x01, 0xac, 0x89, 0xd6, 0x0b, + 0xc9, 0xeb, 0x92, 0xd6, 0x5a, 0x46, 0xc2, 0xa0 }, + { 0x88, 0x18, 0xbb, 0xd3, 0xdb, 0x4d, 0xc1, 0x23, + 0xb2, 0x5c, 0xbb, 0xa5, 0xf5, 0x4c, 0x2b, 0xc4, + 0xb3, 0xfc, 0xf9, 0xbf, 0x7d, 0x7a, 0x77, 0x09, + 0xf4, 0xae, 0x58, 0x8b, 0x26, 0x7c, 0x4e, 0xce }, + { 0xc6, 0x53, 0x82, 0x51, 0x3f, 0x07, 0x46, 0x0d, + 0xa3, 0x98, 0x33, 0xcb, 0x66, 0x6c, 0x5e, 0xd8, + 0x2e, 0x61, 0xb9, 0xe9, 0x98, 0xf4, 0xb0, 0xc4, + 0x28, 0x7c, 0xee, 0x56, 0xc3, 0xcc, 0x9b, 0xcd }, + { 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66, + 0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26, + 0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab, + 0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4 }, + { 0x21, 0xfe, 0x0c, 0xeb, 0x00, 0x52, 0xbe, 0x7f, + 0xb0, 0xf0, 0x04, 0x18, 0x7c, 0xac, 0xd7, 0xde, + 0x67, 0xfa, 0x6e, 0xb0, 0x93, 0x8d, 0x92, 0x76, + 0x77, 0xf2, 0x39, 0x8c, 0x13, 0x23, 0x17, 0xa8 }, + { 0x2e, 0xf7, 0x3f, 0x3c, 0x26, 0xf1, 0x2d, 0x93, + 0x88, 0x9f, 0x3c, 0x78, 0xb6, 0xa6, 0x6c, 0x1d, + 0x52, 0xb6, 0x49, 0xdc, 0x9e, 0x85, 0x6e, 0x2c, + 0x17, 0x2e, 0xa7, 0xc5, 0x8a, 0xc2, 0xb5, 0xe3 }, + { 0x38, 0x8a, 0x3c, 0xd5, 0x6d, 0x73, 0x86, 0x7a, + 0xbb, 0x5f, 0x84, 0x01, 0x49, 0x2b, 0x6e, 0x26, + 0x81, 0xeb, 0x69, 0x85, 0x1e, 0x76, 0x7f, 0xd8, + 0x42, 0x10, 0xa5, 0x60, 0x76, 0xfb, 0x3d, 0xd3 }, + { 0xaf, 0x53, 0x3e, 0x02, 0x2f, 0xc9, 0x43, 0x9e, + 0x4e, 0x3c, 0xb8, 0x38, 0xec, 0xd1, 0x86, 0x92, + 0x23, 0x2a, 0xdf, 0x6f, 0xe9, 0x83, 0x95, 0x26, + 0xd3, 0xc3, 0xdd, 0x1b, 0x71, 0x91, 0x0b, 0x1a }, + { 0x75, 0x1c, 0x09, 0xd4, 0x1a, 0x93, 0x43, 0x88, + 0x2a, 0x81, 0xcd, 0x13, 0xee, 0x40, 0x81, 0x8d, + 0x12, 0xeb, 0x44, 0xc6, 0xc7, 0xf4, 0x0d, 0xf1, + 0x6e, 0x4a, 0xea, 0x8f, 0xab, 0x91, 0x97, 0x2a }, + { 0x5b, 0x73, 0xdd, 0xb6, 0x8d, 0x9d, 0x2b, 0x0a, + 0xa2, 0x65, 0xa0, 0x79, 0x88, 0xd6, 0xb8, 0x8a, + 0xe9, 0xaa, 0xc5, 0x82, 0xaf, 0x83, 0x03, 0x2f, + 0x8a, 0x9b, 0x21, 0xa2, 0xe1, 0xb7, 0xbf, 0x18 }, + { 0x3d, 0xa2, 0x91, 0x26, 0xc7, 0xc5, 0xd7, 0xf4, + 0x3e, 0x64, 0x24, 0x2a, 0x79, 0xfe, 0xaa, 0x4e, + 0xf3, 0x45, 0x9c, 0xde, 0xcc, 0xc8, 0x98, 0xed, + 0x59, 0xa9, 0x7f, 0x6e, 0xc9, 0x3b, 0x9d, 0xab }, + { 0x56, 0x6d, 0xc9, 0x20, 0x29, 0x3d, 0xa5, 0xcb, + 0x4f, 0xe0, 0xaa, 0x8a, 0xbd, 0xa8, 0xbb, 0xf5, + 0x6f, 0x55, 0x23, 0x13, 0xbf, 0xf1, 0x90, 0x46, + 0x64, 0x1e, 0x36, 0x15, 0xc1, 0xe3, 0xed, 0x3f }, + { 0x41, 0x15, 0xbe, 0xa0, 0x2f, 0x73, 0xf9, 0x7f, + 0x62, 0x9e, 0x5c, 0x55, 0x90, 0x72, 0x0c, 0x01, + 0xe7, 0xe4, 0x49, 0xae, 0x2a, 0x66, 0x97, 0xd4, + 0xd2, 0x78, 0x33, 0x21, 0x30, 0x36, 0x92, 0xf9 }, + { 0x4c, 0xe0, 0x8f, 0x47, 0x62, 0x46, 0x8a, 0x76, + 0x70, 0x01, 0x21, 0x64, 0x87, 0x8d, 0x68, 0x34, + 0x0c, 0x52, 0xa3, 0x5e, 0x66, 0xc1, 0x88, 0x4d, + 0x5c, 0x86, 0x48, 0x89, 0xab, 0xc9, 0x66, 0x77 }, + { 0x81, 0xea, 0x0b, 0x78, 0x04, 0x12, 0x4e, 0x0c, + 0x22, 0xea, 0x5f, 0xc7, 0x11, 0x04, 0xa2, 0xaf, + 0xcb, 0x52, 0xa1, 0xfa, 0x81, 0x6f, 0x3e, 0xcb, + 0x7d, 0xcb, 0x5d, 0x9d, 0xea, 0x17, 0x86, 0xd0 }, + { 0xfe, 0x36, 0x27, 0x33, 0xb0, 0x5f, 0x6b, 0xed, + 0xaf, 0x93, 0x79, 0xd7, 0xf7, 0x93, 0x6e, 0xde, + 0x20, 0x9b, 0x1f, 0x83, 0x23, 0xc3, 0x92, 0x25, + 0x49, 0xd9, 0xe7, 0x36, 0x81, 0xb5, 0xdb, 0x7b }, + { 0xef, 0xf3, 0x7d, 0x30, 0xdf, 0xd2, 0x03, 0x59, + 0xbe, 0x4e, 0x73, 0xfd, 0xf4, 0x0d, 0x27, 0x73, + 0x4b, 0x3d, 0xf9, 0x0a, 0x97, 0xa5, 0x5e, 0xd7, + 0x45, 0x29, 0x72, 0x94, 0xca, 0x85, 0xd0, 0x9f }, + { 0x17, 0x2f, 0xfc, 0x67, 0x15, 0x3d, 0x12, 0xe0, + 0xca, 0x76, 0xa8, 0xb6, 0xcd, 0x5d, 0x47, 0x31, + 0x88, 0x5b, 0x39, 0xce, 0x0c, 0xac, 0x93, 0xa8, + 0x97, 0x2a, 0x18, 0x00, 0x6c, 0x8b, 0x8b, 0xaf }, + { 0xc4, 0x79, 0x57, 0xf1, 0xcc, 0x88, 0xe8, 0x3e, + 0xf9, 0x44, 0x58, 0x39, 0x70, 0x9a, 0x48, 0x0a, + 0x03, 0x6b, 0xed, 0x5f, 0x88, 0xac, 0x0f, 0xcc, + 0x8e, 0x1e, 0x70, 0x3f, 0xfa, 0xac, 0x13, 0x2c }, + { 0x30, 0xf3, 0x54, 0x83, 0x70, 0xcf, 0xdc, 0xed, + 0xa5, 0xc3, 0x7b, 0x56, 0x9b, 0x61, 0x75, 0xe7, + 0x99, 0xee, 0xf1, 0xa6, 0x2a, 0xaa, 0x94, 0x32, + 0x45, 0xae, 0x76, 0x69, 0xc2, 0x27, 0xa7, 0xb5 }, + { 0xc9, 0x5d, 0xcb, 0x3c, 0xf1, 0xf2, 0x7d, 0x0e, + 0xef, 0x2f, 0x25, 0xd2, 0x41, 0x38, 0x70, 0x90, + 0x4a, 0x87, 0x7c, 0x4a, 0x56, 0xc2, 0xde, 0x1e, + 0x83, 0xe2, 0xbc, 0x2a, 0xe2, 0xe4, 0x68, 0x21 }, + { 0xd5, 0xd0, 0xb5, 0xd7, 0x05, 0x43, 0x4c, 0xd4, + 0x6b, 0x18, 0x57, 0x49, 0xf6, 0x6b, 0xfb, 0x58, + 0x36, 0xdc, 0xdf, 0x6e, 0xe5, 0x49, 0xa2, 0xb7, + 0xa4, 0xae, 0xe7, 0xf5, 0x80, 0x07, 0xca, 0xaf }, + { 0xbb, 0xc1, 0x24, 0xa7, 0x12, 0xf1, 0x5d, 0x07, + 0xc3, 0x00, 0xe0, 0x5b, 0x66, 0x83, 0x89, 0xa4, + 0x39, 0xc9, 0x17, 0x77, 0xf7, 0x21, 0xf8, 0x32, + 0x0c, 0x1c, 0x90, 0x78, 0x06, 0x6d, 0x2c, 0x7e }, + { 0xa4, 0x51, 0xb4, 0x8c, 0x35, 0xa6, 0xc7, 0x85, + 0x4c, 0xfa, 0xae, 0x60, 0x26, 0x2e, 0x76, 0x99, + 0x08, 0x16, 0x38, 0x2a, 0xc0, 0x66, 0x7e, 0x5a, + 0x5c, 0x9e, 0x1b, 0x46, 0xc4, 0x34, 0x2d, 0xdf }, + { 0xb0, 0xd1, 0x50, 0xfb, 0x55, 0xe7, 0x78, 0xd0, + 0x11, 0x47, 0xf0, 0xb5, 0xd8, 0x9d, 0x99, 0xec, + 0xb2, 0x0f, 0xf0, 0x7e, 0x5e, 0x67, 0x60, 0xd6, + 0xb6, 0x45, 0xeb, 0x5b, 0x65, 0x4c, 0x62, 0x2b }, + { 0x34, 0xf7, 0x37, 0xc0, 0xab, 0x21, 0x99, 0x51, + 0xee, 0xe8, 0x9a, 0x9f, 0x8d, 0xac, 0x29, 0x9c, + 0x9d, 0x4c, 0x38, 0xf3, 0x3f, 0xa4, 0x94, 0xc5, + 0xc6, 0xee, 0xfc, 0x92, 0xb6, 0xdb, 0x08, 0xbc }, + { 0x1a, 0x62, 0xcc, 0x3a, 0x00, 0x80, 0x0d, 0xcb, + 0xd9, 0x98, 0x91, 0x08, 0x0c, 0x1e, 0x09, 0x84, + 0x58, 0x19, 0x3a, 0x8c, 0xc9, 0xf9, 0x70, 0xea, + 0x99, 0xfb, 0xef, 0xf0, 0x03, 0x18, 0xc2, 0x89 }, + { 0xcf, 0xce, 0x55, 0xeb, 0xaf, 0xc8, 0x40, 0xd7, + 0xae, 0x48, 0x28, 0x1c, 0x7f, 0xd5, 0x7e, 0xc8, + 0xb4, 0x82, 0xd4, 0xb7, 0x04, 0x43, 0x74, 0x95, + 0x49, 0x5a, 0xc4, 0x14, 0xcf, 0x4a, 0x37, 0x4b }, + { 0x67, 0x46, 0xfa, 0xcf, 0x71, 0x14, 0x6d, 0x99, + 0x9d, 0xab, 0xd0, 0x5d, 0x09, 0x3a, 0xe5, 0x86, + 0x64, 0x8d, 0x1e, 0xe2, 0x8e, 0x72, 0x61, 0x7b, + 0x99, 0xd0, 0xf0, 0x08, 0x6e, 0x1e, 0x45, 0xbf }, + { 0x57, 0x1c, 0xed, 0x28, 0x3b, 0x3f, 0x23, 0xb4, + 0xe7, 0x50, 0xbf, 0x12, 0xa2, 0xca, 0xf1, 0x78, + 0x18, 0x47, 0xbd, 0x89, 0x0e, 0x43, 0x60, 0x3c, + 0xdc, 0x59, 0x76, 0x10, 0x2b, 0x7b, 0xb1, 0x1b }, + { 0xcf, 0xcb, 0x76, 0x5b, 0x04, 0x8e, 0x35, 0x02, + 0x2c, 0x5d, 0x08, 0x9d, 0x26, 0xe8, 0x5a, 0x36, + 0xb0, 0x05, 0xa2, 0xb8, 0x04, 0x93, 0xd0, 0x3a, + 0x14, 0x4e, 0x09, 0xf4, 0x09, 0xb6, 0xaf, 0xd1 }, + { 0x40, 0x50, 0xc7, 0xa2, 0x77, 0x05, 0xbb, 0x27, + 0xf4, 0x20, 0x89, 0xb2, 0x99, 0xf3, 0xcb, 0xe5, + 0x05, 0x4e, 0xad, 0x68, 0x72, 0x7e, 0x8e, 0xf9, + 0x31, 0x8c, 0xe6, 0xf2, 0x5c, 0xd6, 0xf3, 0x1d }, + { 0x18, 0x40, 0x70, 0xbd, 0x5d, 0x26, 0x5f, 0xbd, + 0xc1, 0x42, 0xcd, 0x1c, 0x5c, 0xd0, 0xd7, 0xe4, + 0x14, 0xe7, 0x03, 0x69, 0xa2, 0x66, 0xd6, 0x27, + 0xc8, 0xfb, 0xa8, 0x4f, 0xa5, 0xe8, 0x4c, 0x34 }, + { 0x9e, 0xdd, 0xa9, 0xa4, 0x44, 0x39, 0x02, 0xa9, + 0x58, 0x8c, 0x0d, 0x0c, 0xcc, 0x62, 0xb9, 0x30, + 0x21, 0x84, 0x79, 0xa6, 0x84, 0x1e, 0x6f, 0xe7, + 0xd4, 0x30, 0x03, 0xf0, 0x4b, 0x1f, 0xd6, 0x43 }, + { 0xe4, 0x12, 0xfe, 0xef, 0x79, 0x08, 0x32, 0x4a, + 0x6d, 0xa1, 0x84, 0x16, 0x29, 0xf3, 0x5d, 0x3d, + 0x35, 0x86, 0x42, 0x01, 0x93, 0x10, 0xec, 0x57, + 0xc6, 0x14, 0x83, 0x6b, 0x63, 0xd3, 0x07, 0x63 }, + { 0x1a, 0x2b, 0x8e, 0xdf, 0xf3, 0xf9, 0xac, 0xc1, + 0x55, 0x4f, 0xcb, 0xae, 0x3c, 0xf1, 0xd6, 0x29, + 0x8c, 0x64, 0x62, 0xe2, 0x2e, 0x5e, 0xb0, 0x25, + 0x96, 0x84, 0xf8, 0x35, 0x01, 0x2b, 0xd1, 0x3f }, + { 0x28, 0x8c, 0x4a, 0xd9, 0xb9, 0x40, 0x97, 0x62, + 0xea, 0x07, 0xc2, 0x4a, 0x41, 0xf0, 0x4f, 0x69, + 0xa7, 0xd7, 0x4b, 0xee, 0x2d, 0x95, 0x43, 0x53, + 0x74, 0xbd, 0xe9, 0x46, 0xd7, 0x24, 0x1c, 0x7b }, + { 0x80, 0x56, 0x91, 0xbb, 0x28, 0x67, 0x48, 0xcf, + 0xb5, 0x91, 0xd3, 0xae, 0xbe, 0x7e, 0x6f, 0x4e, + 0x4d, 0xc6, 0xe2, 0x80, 0x8c, 0x65, 0x14, 0x3c, + 0xc0, 0x04, 0xe4, 0xeb, 0x6f, 0xd0, 0x9d, 0x43 }, + { 0xd4, 0xac, 0x8d, 0x3a, 0x0a, 0xfc, 0x6c, 0xfa, + 0x7b, 0x46, 0x0a, 0xe3, 0x00, 0x1b, 0xae, 0xb3, + 0x6d, 0xad, 0xb3, 0x7d, 0xa0, 0x7d, 0x2e, 0x8a, + 0xc9, 0x18, 0x22, 0xdf, 0x34, 0x8a, 0xed, 0x3d }, + { 0xc3, 0x76, 0x61, 0x70, 0x14, 0xd2, 0x01, 0x58, + 0xbc, 0xed, 0x3d, 0x3b, 0xa5, 0x52, 0xb6, 0xec, + 0xcf, 0x84, 0xe6, 0x2a, 0xa3, 0xeb, 0x65, 0x0e, + 0x90, 0x02, 0x9c, 0x84, 0xd1, 0x3e, 0xea, 0x69 }, + { 0xc4, 0x1f, 0x09, 0xf4, 0x3c, 0xec, 0xae, 0x72, + 0x93, 0xd6, 0x00, 0x7c, 0xa0, 0xa3, 0x57, 0x08, + 0x7d, 0x5a, 0xe5, 0x9b, 0xe5, 0x00, 0xc1, 0xcd, + 0x5b, 0x28, 0x9e, 0xe8, 0x10, 0xc7, 0xb0, 0x82 }, + { 0x03, 0xd1, 0xce, 0xd1, 0xfb, 0xa5, 0xc3, 0x91, + 0x55, 0xc4, 0x4b, 0x77, 0x65, 0xcb, 0x76, 0x0c, + 0x78, 0x70, 0x8d, 0xcf, 0xc8, 0x0b, 0x0b, 0xd8, + 0xad, 0xe3, 0xa5, 0x6d, 0xa8, 0x83, 0x0b, 0x29 }, + { 0x09, 0xbd, 0xe6, 0xf1, 0x52, 0x21, 0x8d, 0xc9, + 0x2c, 0x41, 0xd7, 0xf4, 0x53, 0x87, 0xe6, 0x3e, + 0x58, 0x69, 0xd8, 0x07, 0xec, 0x70, 0xb8, 0x21, + 0x40, 0x5d, 0xbd, 0x88, 0x4b, 0x7f, 0xcf, 0x4b }, + { 0x71, 0xc9, 0x03, 0x6e, 0x18, 0x17, 0x9b, 0x90, + 0xb3, 0x7d, 0x39, 0xe9, 0xf0, 0x5e, 0xb8, 0x9c, + 0xc5, 0xfc, 0x34, 0x1f, 0xd7, 0xc4, 0x77, 0xd0, + 0xd7, 0x49, 0x32, 0x85, 0xfa, 0xca, 0x08, 0xa4 }, + { 0x59, 0x16, 0x83, 0x3e, 0xbb, 0x05, 0xcd, 0x91, + 0x9c, 0xa7, 0xfe, 0x83, 0xb6, 0x92, 0xd3, 0x20, + 0x5b, 0xef, 0x72, 0x39, 0x2b, 0x2c, 0xf6, 0xbb, + 0x0a, 0x6d, 0x43, 0xf9, 0x94, 0xf9, 0x5f, 0x11 }, + { 0xf6, 0x3a, 0xab, 0x3e, 0xc6, 0x41, 0xb3, 0xb0, + 0x24, 0x96, 0x4c, 0x2b, 0x43, 0x7c, 0x04, 0xf6, + 0x04, 0x3c, 0x4c, 0x7e, 0x02, 0x79, 0x23, 0x99, + 0x95, 0x40, 0x19, 0x58, 0xf8, 0x6b, 0xbe, 0x54 }, + { 0xf1, 0x72, 0xb1, 0x80, 0xbf, 0xb0, 0x97, 0x40, + 0x49, 0x31, 0x20, 0xb6, 0x32, 0x6c, 0xbd, 0xc5, + 0x61, 0xe4, 0x77, 0xde, 0xf9, 0xbb, 0xcf, 0xd2, + 0x8c, 0xc8, 0xc1, 0xc5, 0xe3, 0x37, 0x9a, 0x31 }, + { 0xcb, 0x9b, 0x89, 0xcc, 0x18, 0x38, 0x1d, 0xd9, + 0x14, 0x1a, 0xde, 0x58, 0x86, 0x54, 0xd4, 0xe6, + 0xa2, 0x31, 0xd5, 0xbf, 0x49, 0xd4, 0xd5, 0x9a, + 0xc2, 0x7d, 0x86, 0x9c, 0xbe, 0x10, 0x0c, 0xf3 }, + { 0x7b, 0xd8, 0x81, 0x50, 0x46, 0xfd, 0xd8, 0x10, + 0xa9, 0x23, 0xe1, 0x98, 0x4a, 0xae, 0xbd, 0xcd, + 0xf8, 0x4d, 0x87, 0xc8, 0x99, 0x2d, 0x68, 0xb5, + 0xee, 0xb4, 0x60, 0xf9, 0x3e, 0xb3, 0xc8, 0xd7 }, + { 0x60, 0x7b, 0xe6, 0x68, 0x62, 0xfd, 0x08, 0xee, + 0x5b, 0x19, 0xfa, 0xca, 0xc0, 0x9d, 0xfd, 0xbc, + 0xd4, 0x0c, 0x31, 0x21, 0x01, 0xd6, 0x6e, 0x6e, + 0xbd, 0x2b, 0x84, 0x1f, 0x1b, 0x9a, 0x93, 0x25 }, + { 0x9f, 0xe0, 0x3b, 0xbe, 0x69, 0xab, 0x18, 0x34, + 0xf5, 0x21, 0x9b, 0x0d, 0xa8, 0x8a, 0x08, 0xb3, + 0x0a, 0x66, 0xc5, 0x91, 0x3f, 0x01, 0x51, 0x96, + 0x3c, 0x36, 0x05, 0x60, 0xdb, 0x03, 0x87, 0xb3 }, + { 0x90, 0xa8, 0x35, 0x85, 0x71, 0x7b, 0x75, 0xf0, + 0xe9, 0xb7, 0x25, 0xe0, 0x55, 0xee, 0xee, 0xb9, + 0xe7, 0xa0, 0x28, 0xea, 0x7e, 0x6c, 0xbc, 0x07, + 0xb2, 0x09, 0x17, 0xec, 0x03, 0x63, 0xe3, 0x8c }, + { 0x33, 0x6e, 0xa0, 0x53, 0x0f, 0x4a, 0x74, 0x69, + 0x12, 0x6e, 0x02, 0x18, 0x58, 0x7e, 0xbb, 0xde, + 0x33, 0x58, 0xa0, 0xb3, 0x1c, 0x29, 0xd2, 0x00, + 0xf7, 0xdc, 0x7e, 0xb1, 0x5c, 0x6a, 0xad, 0xd8 }, + { 0xa7, 0x9e, 0x76, 0xdc, 0x0a, 0xbc, 0xa4, 0x39, + 0x6f, 0x07, 0x47, 0xcd, 0x7b, 0x74, 0x8d, 0xf9, + 0x13, 0x00, 0x76, 0x26, 0xb1, 0xd6, 0x59, 0xda, + 0x0c, 0x1f, 0x78, 0xb9, 0x30, 0x3d, 0x01, 0xa3 }, + { 0x44, 0xe7, 0x8a, 0x77, 0x37, 0x56, 0xe0, 0x95, + 0x15, 0x19, 0x50, 0x4d, 0x70, 0x38, 0xd2, 0x8d, + 0x02, 0x13, 0xa3, 0x7e, 0x0c, 0xe3, 0x75, 0x37, + 0x17, 0x57, 0xbc, 0x99, 0x63, 0x11, 0xe3, 0xb8 }, + { 0x77, 0xac, 0x01, 0x2a, 0x3f, 0x75, 0x4d, 0xcf, + 0xea, 0xb5, 0xeb, 0x99, 0x6b, 0xe9, 0xcd, 0x2d, + 0x1f, 0x96, 0x11, 0x1b, 0x6e, 0x49, 0xf3, 0x99, + 0x4d, 0xf1, 0x81, 0xf2, 0x85, 0x69, 0xd8, 0x25 }, + { 0xce, 0x5a, 0x10, 0xdb, 0x6f, 0xcc, 0xda, 0xf1, + 0x40, 0xaa, 0xa4, 0xde, 0xd6, 0x25, 0x0a, 0x9c, + 0x06, 0xe9, 0x22, 0x2b, 0xc9, 0xf9, 0xf3, 0x65, + 0x8a, 0x4a, 0xff, 0x93, 0x5f, 0x2b, 0x9f, 0x3a }, + { 0xec, 0xc2, 0x03, 0xa7, 0xfe, 0x2b, 0xe4, 0xab, + 0xd5, 0x5b, 0xb5, 0x3e, 0x6e, 0x67, 0x35, 0x72, + 0xe0, 0x07, 0x8d, 0xa8, 0xcd, 0x37, 0x5e, 0xf4, + 0x30, 0xcc, 0x97, 0xf9, 0xf8, 0x00, 0x83, 0xaf }, + { 0x14, 0xa5, 0x18, 0x6d, 0xe9, 0xd7, 0xa1, 0x8b, + 0x04, 0x12, 0xb8, 0x56, 0x3e, 0x51, 0xcc, 0x54, + 0x33, 0x84, 0x0b, 0x4a, 0x12, 0x9a, 0x8f, 0xf9, + 0x63, 0xb3, 0x3a, 0x3c, 0x4a, 0xfe, 0x8e, 0xbb }, + { 0x13, 0xf8, 0xef, 0x95, 0xcb, 0x86, 0xe6, 0xa6, + 0x38, 0x93, 0x1c, 0x8e, 0x10, 0x76, 0x73, 0xeb, + 0x76, 0xba, 0x10, 0xd7, 0xc2, 0xcd, 0x70, 0xb9, + 0xd9, 0x92, 0x0b, 0xbe, 0xed, 0x92, 0x94, 0x09 }, + { 0x0b, 0x33, 0x8f, 0x4e, 0xe1, 0x2f, 0x2d, 0xfc, + 0xb7, 0x87, 0x13, 0x37, 0x79, 0x41, 0xe0, 0xb0, + 0x63, 0x21, 0x52, 0x58, 0x1d, 0x13, 0x32, 0x51, + 0x6e, 0x4a, 0x2c, 0xab, 0x19, 0x42, 0xcc, 0xa4 }, + { 0xea, 0xab, 0x0e, 0xc3, 0x7b, 0x3b, 0x8a, 0xb7, + 0x96, 0xe9, 0xf5, 0x72, 0x38, 0xde, 0x14, 0xa2, + 0x64, 0xa0, 0x76, 0xf3, 0x88, 0x7d, 0x86, 0xe2, + 0x9b, 0xb5, 0x90, 0x6d, 0xb5, 0xa0, 0x0e, 0x02 }, + { 0x23, 0xcb, 0x68, 0xb8, 0xc0, 0xe6, 0xdc, 0x26, + 0xdc, 0x27, 0x76, 0x6d, 0xdc, 0x0a, 0x13, 0xa9, + 0x94, 0x38, 0xfd, 0x55, 0x61, 0x7a, 0xa4, 0x09, + 0x5d, 0x8f, 0x96, 0x97, 0x20, 0xc8, 0x72, 0xdf }, + { 0x09, 0x1d, 0x8e, 0xe3, 0x0d, 0x6f, 0x29, 0x68, + 0xd4, 0x6b, 0x68, 0x7d, 0xd6, 0x52, 0x92, 0x66, + 0x57, 0x42, 0xde, 0x0b, 0xb8, 0x3d, 0xcc, 0x00, + 0x04, 0xc7, 0x2c, 0xe1, 0x00, 0x07, 0xa5, 0x49 }, + { 0x7f, 0x50, 0x7a, 0xbc, 0x6d, 0x19, 0xba, 0x00, + 0xc0, 0x65, 0xa8, 0x76, 0xec, 0x56, 0x57, 0x86, + 0x88, 0x82, 0xd1, 0x8a, 0x22, 0x1b, 0xc4, 0x6c, + 0x7a, 0x69, 0x12, 0x54, 0x1f, 0x5b, 0xc7, 0xba }, + { 0xa0, 0x60, 0x7c, 0x24, 0xe1, 0x4e, 0x8c, 0x22, + 0x3d, 0xb0, 0xd7, 0x0b, 0x4d, 0x30, 0xee, 0x88, + 0x01, 0x4d, 0x60, 0x3f, 0x43, 0x7e, 0x9e, 0x02, + 0xaa, 0x7d, 0xaf, 0xa3, 0xcd, 0xfb, 0xad, 0x94 }, + { 0xdd, 0xbf, 0xea, 0x75, 0xcc, 0x46, 0x78, 0x82, + 0xeb, 0x34, 0x83, 0xce, 0x5e, 0x2e, 0x75, 0x6a, + 0x4f, 0x47, 0x01, 0xb7, 0x6b, 0x44, 0x55, 0x19, + 0xe8, 0x9f, 0x22, 0xd6, 0x0f, 0xa8, 0x6e, 0x06 }, + { 0x0c, 0x31, 0x1f, 0x38, 0xc3, 0x5a, 0x4f, 0xb9, + 0x0d, 0x65, 0x1c, 0x28, 0x9d, 0x48, 0x68, 0x56, + 0xcd, 0x14, 0x13, 0xdf, 0x9b, 0x06, 0x77, 0xf5, + 0x3e, 0xce, 0x2c, 0xd9, 0xe4, 0x77, 0xc6, 0x0a }, + { 0x46, 0xa7, 0x3a, 0x8d, 0xd3, 0xe7, 0x0f, 0x59, + 0xd3, 0x94, 0x2c, 0x01, 0xdf, 0x59, 0x9d, 0xef, + 0x78, 0x3c, 0x9d, 0xa8, 0x2f, 0xd8, 0x32, 0x22, + 0xcd, 0x66, 0x2b, 0x53, 0xdc, 0xe7, 0xdb, 0xdf }, + { 0xad, 0x03, 0x8f, 0xf9, 0xb1, 0x4d, 0xe8, 0x4a, + 0x80, 0x1e, 0x4e, 0x62, 0x1c, 0xe5, 0xdf, 0x02, + 0x9d, 0xd9, 0x35, 0x20, 0xd0, 0xc2, 0xfa, 0x38, + 0xbf, 0xf1, 0x76, 0xa8, 0xb1, 0xd1, 0x69, 0x8c }, + { 0xab, 0x70, 0xc5, 0xdf, 0xbd, 0x1e, 0xa8, 0x17, + 0xfe, 0xd0, 0xcd, 0x06, 0x72, 0x93, 0xab, 0xf3, + 0x19, 0xe5, 0xd7, 0x90, 0x1c, 0x21, 0x41, 0xd5, + 0xd9, 0x9b, 0x23, 0xf0, 0x3a, 0x38, 0xe7, 0x48 }, + { 0x1f, 0xff, 0xda, 0x67, 0x93, 0x2b, 0x73, 0xc8, + 0xec, 0xaf, 0x00, 0x9a, 0x34, 0x91, 0xa0, 0x26, + 0x95, 0x3b, 0xab, 0xfe, 0x1f, 0x66, 0x3b, 0x06, + 0x97, 0xc3, 0xc4, 0xae, 0x8b, 0x2e, 0x7d, 0xcb }, + { 0xb0, 0xd2, 0xcc, 0x19, 0x47, 0x2d, 0xd5, 0x7f, + 0x2b, 0x17, 0xef, 0xc0, 0x3c, 0x8d, 0x58, 0xc2, + 0x28, 0x3d, 0xbb, 0x19, 0xda, 0x57, 0x2f, 0x77, + 0x55, 0x85, 0x5a, 0xa9, 0x79, 0x43, 0x17, 0xa0 }, + { 0xa0, 0xd1, 0x9a, 0x6e, 0xe3, 0x39, 0x79, 0xc3, + 0x25, 0x51, 0x0e, 0x27, 0x66, 0x22, 0xdf, 0x41, + 0xf7, 0x15, 0x83, 0xd0, 0x75, 0x01, 0xb8, 0x70, + 0x71, 0x12, 0x9a, 0x0a, 0xd9, 0x47, 0x32, 0xa5 }, + { 0x72, 0x46, 0x42, 0xa7, 0x03, 0x2d, 0x10, 0x62, + 0xb8, 0x9e, 0x52, 0xbe, 0xa3, 0x4b, 0x75, 0xdf, + 0x7d, 0x8f, 0xe7, 0x72, 0xd9, 0xfe, 0x3c, 0x93, + 0xdd, 0xf3, 0xc4, 0x54, 0x5a, 0xb5, 0xa9, 0x9b }, + { 0xad, 0xe5, 0xea, 0xa7, 0xe6, 0x1f, 0x67, 0x2d, + 0x58, 0x7e, 0xa0, 0x3d, 0xae, 0x7d, 0x7b, 0x55, + 0x22, 0x9c, 0x01, 0xd0, 0x6b, 0xc0, 0xa5, 0x70, + 0x14, 0x36, 0xcb, 0xd1, 0x83, 0x66, 0xa6, 0x26 }, + { 0x01, 0x3b, 0x31, 0xeb, 0xd2, 0x28, 0xfc, 0xdd, + 0xa5, 0x1f, 0xab, 0xb0, 0x3b, 0xb0, 0x2d, 0x60, + 0xac, 0x20, 0xca, 0x21, 0x5a, 0xaf, 0xa8, 0x3b, + 0xdd, 0x85, 0x5e, 0x37, 0x55, 0xa3, 0x5f, 0x0b }, + { 0x33, 0x2e, 0xd4, 0x0b, 0xb1, 0x0d, 0xde, 0x3c, + 0x95, 0x4a, 0x75, 0xd7, 0xb8, 0x99, 0x9d, 0x4b, + 0x26, 0xa1, 0xc0, 0x63, 0xc1, 0xdc, 0x6e, 0x32, + 0xc1, 0xd9, 0x1b, 0xab, 0x7b, 0xbb, 0x7d, 0x16 }, + { 0xc7, 0xa1, 0x97, 0xb3, 0xa0, 0x5b, 0x56, 0x6b, + 0xcc, 0x9f, 0xac, 0xd2, 0x0e, 0x44, 0x1d, 0x6f, + 0x6c, 0x28, 0x60, 0xac, 0x96, 0x51, 0xcd, 0x51, + 0xd6, 0xb9, 0xd2, 0xcd, 0xee, 0xea, 0x03, 0x90 }, + { 0xbd, 0x9c, 0xf6, 0x4e, 0xa8, 0x95, 0x3c, 0x03, + 0x71, 0x08, 0xe6, 0xf6, 0x54, 0x91, 0x4f, 0x39, + 0x58, 0xb6, 0x8e, 0x29, 0xc1, 0x67, 0x00, 0xdc, + 0x18, 0x4d, 0x94, 0xa2, 0x17, 0x08, 0xff, 0x60 }, + { 0x88, 0x35, 0xb0, 0xac, 0x02, 0x11, 0x51, 0xdf, + 0x71, 0x64, 0x74, 0xce, 0x27, 0xce, 0x4d, 0x3c, + 0x15, 0xf0, 0xb2, 0xda, 0xb4, 0x80, 0x03, 0xcf, + 0x3f, 0x3e, 0xfd, 0x09, 0x45, 0x10, 0x6b, 0x9a }, + { 0x3b, 0xfe, 0xfa, 0x33, 0x01, 0xaa, 0x55, 0xc0, + 0x80, 0x19, 0x0c, 0xff, 0xda, 0x8e, 0xae, 0x51, + 0xd9, 0xaf, 0x48, 0x8b, 0x4c, 0x1f, 0x24, 0xc3, + 0xd9, 0xa7, 0x52, 0x42, 0xfd, 0x8e, 0xa0, 0x1d }, + { 0x08, 0x28, 0x4d, 0x14, 0x99, 0x3c, 0xd4, 0x7d, + 0x53, 0xeb, 0xae, 0xcf, 0x0d, 0xf0, 0x47, 0x8c, + 0xc1, 0x82, 0xc8, 0x9c, 0x00, 0xe1, 0x85, 0x9c, + 0x84, 0x85, 0x16, 0x86, 0xdd, 0xf2, 0xc1, 0xb7 }, + { 0x1e, 0xd7, 0xef, 0x9f, 0x04, 0xc2, 0xac, 0x8d, + 0xb6, 0xa8, 0x64, 0xdb, 0x13, 0x10, 0x87, 0xf2, + 0x70, 0x65, 0x09, 0x8e, 0x69, 0xc3, 0xfe, 0x78, + 0x71, 0x8d, 0x9b, 0x94, 0x7f, 0x4a, 0x39, 0xd0 }, + { 0xc1, 0x61, 0xf2, 0xdc, 0xd5, 0x7e, 0x9c, 0x14, + 0x39, 0xb3, 0x1a, 0x9d, 0xd4, 0x3d, 0x8f, 0x3d, + 0x7d, 0xd8, 0xf0, 0xeb, 0x7c, 0xfa, 0xc6, 0xfb, + 0x25, 0xa0, 0xf2, 0x8e, 0x30, 0x6f, 0x06, 0x61 }, + { 0xc0, 0x19, 0x69, 0xad, 0x34, 0xc5, 0x2c, 0xaf, + 0x3d, 0xc4, 0xd8, 0x0d, 0x19, 0x73, 0x5c, 0x29, + 0x73, 0x1a, 0xc6, 0xe7, 0xa9, 0x20, 0x85, 0xab, + 0x92, 0x50, 0xc4, 0x8d, 0xea, 0x48, 0xa3, 0xfc }, + { 0x17, 0x20, 0xb3, 0x65, 0x56, 0x19, 0xd2, 0xa5, + 0x2b, 0x35, 0x21, 0xae, 0x0e, 0x49, 0xe3, 0x45, + 0xcb, 0x33, 0x89, 0xeb, 0xd6, 0x20, 0x8a, 0xca, + 0xf9, 0xf1, 0x3f, 0xda, 0xcc, 0xa8, 0xbe, 0x49 }, + { 0x75, 0x62, 0x88, 0x36, 0x1c, 0x83, 0xe2, 0x4c, + 0x61, 0x7c, 0xf9, 0x5c, 0x90, 0x5b, 0x22, 0xd0, + 0x17, 0xcd, 0xc8, 0x6f, 0x0b, 0xf1, 0xd6, 0x58, + 0xf4, 0x75, 0x6c, 0x73, 0x79, 0x87, 0x3b, 0x7f }, + { 0xe7, 0xd0, 0xed, 0xa3, 0x45, 0x26, 0x93, 0xb7, + 0x52, 0xab, 0xcd, 0xa1, 0xb5, 0x5e, 0x27, 0x6f, + 0x82, 0x69, 0x8f, 0x5f, 0x16, 0x05, 0x40, 0x3e, + 0xff, 0x83, 0x0b, 0xea, 0x00, 0x71, 0xa3, 0x94 }, + { 0x2c, 0x82, 0xec, 0xaa, 0x6b, 0x84, 0x80, 0x3e, + 0x04, 0x4a, 0xf6, 0x31, 0x18, 0xaf, 0xe5, 0x44, + 0x68, 0x7c, 0xb6, 0xe6, 0xc7, 0xdf, 0x49, 0xed, + 0x76, 0x2d, 0xfd, 0x7c, 0x86, 0x93, 0xa1, 0xbc }, + { 0x61, 0x36, 0xcb, 0xf4, 0xb4, 0x41, 0x05, 0x6f, + 0xa1, 0xe2, 0x72, 0x24, 0x98, 0x12, 0x5d, 0x6d, + 0xed, 0x45, 0xe1, 0x7b, 0x52, 0x14, 0x39, 0x59, + 0xc7, 0xf4, 0xd4, 0xe3, 0x95, 0x21, 0x8a, 0xc2 }, + { 0x72, 0x1d, 0x32, 0x45, 0xaa, 0xfe, 0xf2, 0x7f, + 0x6a, 0x62, 0x4f, 0x47, 0x95, 0x4b, 0x6c, 0x25, + 0x50, 0x79, 0x52, 0x6f, 0xfa, 0x25, 0xe9, 0xff, + 0x77, 0xe5, 0xdc, 0xff, 0x47, 0x3b, 0x15, 0x97 }, + { 0x9d, 0xd2, 0xfb, 0xd8, 0xce, 0xf1, 0x6c, 0x35, + 0x3c, 0x0a, 0xc2, 0x11, 0x91, 0xd5, 0x09, 0xeb, + 0x28, 0xdd, 0x9e, 0x3e, 0x0d, 0x8c, 0xea, 0x5d, + 0x26, 0xca, 0x83, 0x93, 0x93, 0x85, 0x1c, 0x3a }, + { 0xb2, 0x39, 0x4c, 0xea, 0xcd, 0xeb, 0xf2, 0x1b, + 0xf9, 0xdf, 0x2c, 0xed, 0x98, 0xe5, 0x8f, 0x1c, + 0x3a, 0x4b, 0xbb, 0xff, 0x66, 0x0d, 0xd9, 0x00, + 0xf6, 0x22, 0x02, 0xd6, 0x78, 0x5c, 0xc4, 0x6e }, + { 0x57, 0x08, 0x9f, 0x22, 0x27, 0x49, 0xad, 0x78, + 0x71, 0x76, 0x5f, 0x06, 0x2b, 0x11, 0x4f, 0x43, + 0xba, 0x20, 0xec, 0x56, 0x42, 0x2a, 0x8b, 0x1e, + 0x3f, 0x87, 0x19, 0x2c, 0x0e, 0xa7, 0x18, 0xc6 }, + { 0xe4, 0x9a, 0x94, 0x59, 0x96, 0x1c, 0xd3, 0x3c, + 0xdf, 0x4a, 0xae, 0x1b, 0x10, 0x78, 0xa5, 0xde, + 0xa7, 0xc0, 0x40, 0xe0, 0xfe, 0xa3, 0x40, 0xc9, + 0x3a, 0x72, 0x48, 0x72, 0xfc, 0x4a, 0xf8, 0x06 }, + { 0xed, 0xe6, 0x7f, 0x72, 0x0e, 0xff, 0xd2, 0xca, + 0x9c, 0x88, 0x99, 0x41, 0x52, 0xd0, 0x20, 0x1d, + 0xee, 0x6b, 0x0a, 0x2d, 0x2c, 0x07, 0x7a, 0xca, + 0x6d, 0xae, 0x29, 0xf7, 0x3f, 0x8b, 0x63, 0x09 }, + { 0xe0, 0xf4, 0x34, 0xbf, 0x22, 0xe3, 0x08, 0x80, + 0x39, 0xc2, 0x1f, 0x71, 0x9f, 0xfc, 0x67, 0xf0, + 0xf2, 0xcb, 0x5e, 0x98, 0xa7, 0xa0, 0x19, 0x4c, + 0x76, 0xe9, 0x6b, 0xf4, 0xe8, 0xe1, 0x7e, 0x61 }, + { 0x27, 0x7c, 0x04, 0xe2, 0x85, 0x34, 0x84, 0xa4, + 0xeb, 0xa9, 0x10, 0xad, 0x33, 0x6d, 0x01, 0xb4, + 0x77, 0xb6, 0x7c, 0xc2, 0x00, 0xc5, 0x9f, 0x3c, + 0x8d, 0x77, 0xee, 0xf8, 0x49, 0x4f, 0x29, 0xcd }, + { 0x15, 0x6d, 0x57, 0x47, 0xd0, 0xc9, 0x9c, 0x7f, + 0x27, 0x09, 0x7d, 0x7b, 0x7e, 0x00, 0x2b, 0x2e, + 0x18, 0x5c, 0xb7, 0x2d, 0x8d, 0xd7, 0xeb, 0x42, + 0x4a, 0x03, 0x21, 0x52, 0x81, 0x61, 0x21, 0x9f }, + { 0x20, 0xdd, 0xd1, 0xed, 0x9b, 0x1c, 0xa8, 0x03, + 0x94, 0x6d, 0x64, 0xa8, 0x3a, 0xe4, 0x65, 0x9d, + 0xa6, 0x7f, 0xba, 0x7a, 0x1a, 0x3e, 0xdd, 0xb1, + 0xe1, 0x03, 0xc0, 0xf5, 0xe0, 0x3e, 0x3a, 0x2c }, + { 0xf0, 0xaf, 0x60, 0x4d, 0x3d, 0xab, 0xbf, 0x9a, + 0x0f, 0x2a, 0x7d, 0x3d, 0xda, 0x6b, 0xd3, 0x8b, + 0xba, 0x72, 0xc6, 0xd0, 0x9b, 0xe4, 0x94, 0xfc, + 0xef, 0x71, 0x3f, 0xf1, 0x01, 0x89, 0xb6, 0xe6 }, + { 0x98, 0x02, 0xbb, 0x87, 0xde, 0xf4, 0xcc, 0x10, + 0xc4, 0xa5, 0xfd, 0x49, 0xaa, 0x58, 0xdf, 0xe2, + 0xf3, 0xfd, 0xdb, 0x46, 0xb4, 0x70, 0x88, 0x14, + 0xea, 0xd8, 0x1d, 0x23, 0xba, 0x95, 0x13, 0x9b }, + { 0x4f, 0x8c, 0xe1, 0xe5, 0x1d, 0x2f, 0xe7, 0xf2, + 0x40, 0x43, 0xa9, 0x04, 0xd8, 0x98, 0xeb, 0xfc, + 0x91, 0x97, 0x54, 0x18, 0x75, 0x34, 0x13, 0xaa, + 0x09, 0x9b, 0x79, 0x5e, 0xcb, 0x35, 0xce, 0xdb }, + { 0xbd, 0xdc, 0x65, 0x14, 0xd7, 0xee, 0x6a, 0xce, + 0x0a, 0x4a, 0xc1, 0xd0, 0xe0, 0x68, 0x11, 0x22, + 0x88, 0xcb, 0xcf, 0x56, 0x04, 0x54, 0x64, 0x27, + 0x05, 0x63, 0x01, 0x77, 0xcb, 0xa6, 0x08, 0xbd }, + { 0xd6, 0x35, 0x99, 0x4f, 0x62, 0x91, 0x51, 0x7b, + 0x02, 0x81, 0xff, 0xdd, 0x49, 0x6a, 0xfa, 0x86, + 0x27, 0x12, 0xe5, 0xb3, 0xc4, 0xe5, 0x2e, 0x4c, + 0xd5, 0xfd, 0xae, 0x8c, 0x0e, 0x72, 0xfb, 0x08 }, + { 0x87, 0x8d, 0x9c, 0xa6, 0x00, 0xcf, 0x87, 0xe7, + 0x69, 0xcc, 0x30, 0x5c, 0x1b, 0x35, 0x25, 0x51, + 0x86, 0x61, 0x5a, 0x73, 0xa0, 0xda, 0x61, 0x3b, + 0x5f, 0x1c, 0x98, 0xdb, 0xf8, 0x12, 0x83, 0xea }, + { 0xa6, 0x4e, 0xbe, 0x5d, 0xc1, 0x85, 0xde, 0x9f, + 0xdd, 0xe7, 0x60, 0x7b, 0x69, 0x98, 0x70, 0x2e, + 0xb2, 0x34, 0x56, 0x18, 0x49, 0x57, 0x30, 0x7d, + 0x2f, 0xa7, 0x2e, 0x87, 0xa4, 0x77, 0x02, 0xd6 }, + { 0xce, 0x50, 0xea, 0xb7, 0xb5, 0xeb, 0x52, 0xbd, + 0xc9, 0xad, 0x8e, 0x5a, 0x48, 0x0a, 0xb7, 0x80, + 0xca, 0x93, 0x20, 0xe4, 0x43, 0x60, 0xb1, 0xfe, + 0x37, 0xe0, 0x3f, 0x2f, 0x7a, 0xd7, 0xde, 0x01 }, + { 0xee, 0xdd, 0xb7, 0xc0, 0xdb, 0x6e, 0x30, 0xab, + 0xe6, 0x6d, 0x79, 0xe3, 0x27, 0x51, 0x1e, 0x61, + 0xfc, 0xeb, 0xbc, 0x29, 0xf1, 0x59, 0xb4, 0x0a, + 0x86, 0xb0, 0x46, 0xec, 0xf0, 0x51, 0x38, 0x23 }, + { 0x78, 0x7f, 0xc9, 0x34, 0x40, 0xc1, 0xec, 0x96, + 0xb5, 0xad, 0x01, 0xc1, 0x6c, 0xf7, 0x79, 0x16, + 0xa1, 0x40, 0x5f, 0x94, 0x26, 0x35, 0x6e, 0xc9, + 0x21, 0xd8, 0xdf, 0xf3, 0xea, 0x63, 0xb7, 0xe0 }, + { 0x7f, 0x0d, 0x5e, 0xab, 0x47, 0xee, 0xfd, 0xa6, + 0x96, 0xc0, 0xbf, 0x0f, 0xbf, 0x86, 0xab, 0x21, + 0x6f, 0xce, 0x46, 0x1e, 0x93, 0x03, 0xab, 0xa6, + 0xac, 0x37, 0x41, 0x20, 0xe8, 0x90, 0xe8, 0xdf }, + { 0xb6, 0x80, 0x04, 0xb4, 0x2f, 0x14, 0xad, 0x02, + 0x9f, 0x4c, 0x2e, 0x03, 0xb1, 0xd5, 0xeb, 0x76, + 0xd5, 0x71, 0x60, 0xe2, 0x64, 0x76, 0xd2, 0x11, + 0x31, 0xbe, 0xf2, 0x0a, 0xda, 0x7d, 0x27, 0xf4 }, + { 0xb0, 0xc4, 0xeb, 0x18, 0xae, 0x25, 0x0b, 0x51, + 0xa4, 0x13, 0x82, 0xea, 0xd9, 0x2d, 0x0d, 0xc7, + 0x45, 0x5f, 0x93, 0x79, 0xfc, 0x98, 0x84, 0x42, + 0x8e, 0x47, 0x70, 0x60, 0x8d, 0xb0, 0xfa, 0xec }, + { 0xf9, 0x2b, 0x7a, 0x87, 0x0c, 0x05, 0x9f, 0x4d, + 0x46, 0x46, 0x4c, 0x82, 0x4e, 0xc9, 0x63, 0x55, + 0x14, 0x0b, 0xdc, 0xe6, 0x81, 0x32, 0x2c, 0xc3, + 0xa9, 0x92, 0xff, 0x10, 0x3e, 0x3f, 0xea, 0x52 }, + { 0x53, 0x64, 0x31, 0x26, 0x14, 0x81, 0x33, 0x98, + 0xcc, 0x52, 0x5d, 0x4c, 0x4e, 0x14, 0x6e, 0xde, + 0xb3, 0x71, 0x26, 0x5f, 0xba, 0x19, 0x13, 0x3a, + 0x2c, 0x3d, 0x21, 0x59, 0x29, 0x8a, 0x17, 0x42 }, + { 0xf6, 0x62, 0x0e, 0x68, 0xd3, 0x7f, 0xb2, 0xaf, + 0x50, 0x00, 0xfc, 0x28, 0xe2, 0x3b, 0x83, 0x22, + 0x97, 0xec, 0xd8, 0xbc, 0xe9, 0x9e, 0x8b, 0xe4, + 0xd0, 0x4e, 0x85, 0x30, 0x9e, 0x3d, 0x33, 0x74 }, + { 0x53, 0x16, 0xa2, 0x79, 0x69, 0xd7, 0xfe, 0x04, + 0xff, 0x27, 0xb2, 0x83, 0x96, 0x1b, 0xff, 0xc3, + 0xbf, 0x5d, 0xfb, 0x32, 0xfb, 0x6a, 0x89, 0xd1, + 0x01, 0xc6, 0xc3, 0xb1, 0x93, 0x7c, 0x28, 0x71 }, + { 0x81, 0xd1, 0x66, 0x4f, 0xdf, 0x3c, 0xb3, 0x3c, + 0x24, 0xee, 0xba, 0xc0, 0xbd, 0x64, 0x24, 0x4b, + 0x77, 0xc4, 0xab, 0xea, 0x90, 0xbb, 0xe8, 0xb5, + 0xee, 0x0b, 0x2a, 0xaf, 0xcf, 0x2d, 0x6a, 0x53 }, + { 0x34, 0x57, 0x82, 0xf2, 0x95, 0xb0, 0x88, 0x03, + 0x52, 0xe9, 0x24, 0xa0, 0x46, 0x7b, 0x5f, 0xbc, + 0x3e, 0x8f, 0x3b, 0xfb, 0xc3, 0xc7, 0xe4, 0x8b, + 0x67, 0x09, 0x1f, 0xb5, 0xe8, 0x0a, 0x94, 0x42 }, + { 0x79, 0x41, 0x11, 0xea, 0x6c, 0xd6, 0x5e, 0x31, + 0x1f, 0x74, 0xee, 0x41, 0xd4, 0x76, 0xcb, 0x63, + 0x2c, 0xe1, 0xe4, 0xb0, 0x51, 0xdc, 0x1d, 0x9e, + 0x9d, 0x06, 0x1a, 0x19, 0xe1, 0xd0, 0xbb, 0x49 }, + { 0x2a, 0x85, 0xda, 0xf6, 0x13, 0x88, 0x16, 0xb9, + 0x9b, 0xf8, 0xd0, 0x8b, 0xa2, 0x11, 0x4b, 0x7a, + 0xb0, 0x79, 0x75, 0xa7, 0x84, 0x20, 0xc1, 0xa3, + 0xb0, 0x6a, 0x77, 0x7c, 0x22, 0xdd, 0x8b, 0xcb }, + { 0x89, 0xb0, 0xd5, 0xf2, 0x89, 0xec, 0x16, 0x40, + 0x1a, 0x06, 0x9a, 0x96, 0x0d, 0x0b, 0x09, 0x3e, + 0x62, 0x5d, 0xa3, 0xcf, 0x41, 0xee, 0x29, 0xb5, + 0x9b, 0x93, 0x0c, 0x58, 0x20, 0x14, 0x54, 0x55 }, + { 0xd0, 0xfd, 0xcb, 0x54, 0x39, 0x43, 0xfc, 0x27, + 0xd2, 0x08, 0x64, 0xf5, 0x21, 0x81, 0x47, 0x1b, + 0x94, 0x2c, 0xc7, 0x7c, 0xa6, 0x75, 0xbc, 0xb3, + 0x0d, 0xf3, 0x1d, 0x35, 0x8e, 0xf7, 0xb1, 0xeb }, + { 0xb1, 0x7e, 0xa8, 0xd7, 0x70, 0x63, 0xc7, 0x09, + 0xd4, 0xdc, 0x6b, 0x87, 0x94, 0x13, 0xc3, 0x43, + 0xe3, 0x79, 0x0e, 0x9e, 0x62, 0xca, 0x85, 0xb7, + 0x90, 0x0b, 0x08, 0x6f, 0x6b, 0x75, 0xc6, 0x72 }, + { 0xe7, 0x1a, 0x3e, 0x2c, 0x27, 0x4d, 0xb8, 0x42, + 0xd9, 0x21, 0x14, 0xf2, 0x17, 0xe2, 0xc0, 0xea, + 0xc8, 0xb4, 0x50, 0x93, 0xfd, 0xfd, 0x9d, 0xf4, + 0xca, 0x71, 0x62, 0x39, 0x48, 0x62, 0xd5, 0x01 }, + { 0xc0, 0x47, 0x67, 0x59, 0xab, 0x7a, 0xa3, 0x33, + 0x23, 0x4f, 0x6b, 0x44, 0xf5, 0xfd, 0x85, 0x83, + 0x90, 0xec, 0x23, 0x69, 0x4c, 0x62, 0x2c, 0xb9, + 0x86, 0xe7, 0x69, 0xc7, 0x8e, 0xdd, 0x73, 0x3e }, + { 0x9a, 0xb8, 0xea, 0xbb, 0x14, 0x16, 0x43, 0x4d, + 0x85, 0x39, 0x13, 0x41, 0xd5, 0x69, 0x93, 0xc5, + 0x54, 0x58, 0x16, 0x7d, 0x44, 0x18, 0xb1, 0x9a, + 0x0f, 0x2a, 0xd8, 0xb7, 0x9a, 0x83, 0xa7, 0x5b }, + { 0x79, 0x92, 0xd0, 0xbb, 0xb1, 0x5e, 0x23, 0x82, + 0x6f, 0x44, 0x3e, 0x00, 0x50, 0x5d, 0x68, 0xd3, + 0xed, 0x73, 0x72, 0x99, 0x5a, 0x5c, 0x3e, 0x49, + 0x86, 0x54, 0x10, 0x2f, 0xbc, 0xd0, 0x96, 0x4e }, + { 0xc0, 0x21, 0xb3, 0x00, 0x85, 0x15, 0x14, 0x35, + 0xdf, 0x33, 0xb0, 0x07, 0xcc, 0xec, 0xc6, 0x9d, + 0xf1, 0x26, 0x9f, 0x39, 0xba, 0x25, 0x09, 0x2b, + 0xed, 0x59, 0xd9, 0x32, 0xac, 0x0f, 0xdc, 0x28 }, + { 0x91, 0xa2, 0x5e, 0xc0, 0xec, 0x0d, 0x9a, 0x56, + 0x7f, 0x89, 0xc4, 0xbf, 0xe1, 0xa6, 0x5a, 0x0e, + 0x43, 0x2d, 0x07, 0x06, 0x4b, 0x41, 0x90, 0xe2, + 0x7d, 0xfb, 0x81, 0x90, 0x1f, 0xd3, 0x13, 0x9b }, + { 0x59, 0x50, 0xd3, 0x9a, 0x23, 0xe1, 0x54, 0x5f, + 0x30, 0x12, 0x70, 0xaa, 0x1a, 0x12, 0xf2, 0xe6, + 0xc4, 0x53, 0x77, 0x6e, 0x4d, 0x63, 0x55, 0xde, + 0x42, 0x5c, 0xc1, 0x53, 0xf9, 0x81, 0x88, 0x67 }, + { 0xd7, 0x9f, 0x14, 0x72, 0x0c, 0x61, 0x0a, 0xf1, + 0x79, 0xa3, 0x76, 0x5d, 0x4b, 0x7c, 0x09, 0x68, + 0xf9, 0x77, 0x96, 0x2d, 0xbf, 0x65, 0x5b, 0x52, + 0x12, 0x72, 0xb6, 0xf1, 0xe1, 0x94, 0x48, 0x8e }, + { 0xe9, 0x53, 0x1b, 0xfc, 0x8b, 0x02, 0x99, 0x5a, + 0xea, 0xa7, 0x5b, 0xa2, 0x70, 0x31, 0xfa, 0xdb, + 0xcb, 0xf4, 0xa0, 0xda, 0xb8, 0x96, 0x1d, 0x92, + 0x96, 0xcd, 0x7e, 0x84, 0xd2, 0x5d, 0x60, 0x06 }, + { 0x34, 0xe9, 0xc2, 0x6a, 0x01, 0xd7, 0xf1, 0x61, + 0x81, 0xb4, 0x54, 0xa9, 0xd1, 0x62, 0x3c, 0x23, + 0x3c, 0xb9, 0x9d, 0x31, 0xc6, 0x94, 0x65, 0x6e, + 0x94, 0x13, 0xac, 0xa3, 0xe9, 0x18, 0x69, 0x2f }, + { 0xd9, 0xd7, 0x42, 0x2f, 0x43, 0x7b, 0xd4, 0x39, + 0xdd, 0xd4, 0xd8, 0x83, 0xda, 0xe2, 0xa0, 0x83, + 0x50, 0x17, 0x34, 0x14, 0xbe, 0x78, 0x15, 0x51, + 0x33, 0xff, 0xf1, 0x96, 0x4c, 0x3d, 0x79, 0x72 }, + { 0x4a, 0xee, 0x0c, 0x7a, 0xaf, 0x07, 0x54, 0x14, + 0xff, 0x17, 0x93, 0xea, 0xd7, 0xea, 0xca, 0x60, + 0x17, 0x75, 0xc6, 0x15, 0xdb, 0xd6, 0x0b, 0x64, + 0x0b, 0x0a, 0x9f, 0x0c, 0xe5, 0x05, 0xd4, 0x35 }, + { 0x6b, 0xfd, 0xd1, 0x54, 0x59, 0xc8, 0x3b, 0x99, + 0xf0, 0x96, 0xbf, 0xb4, 0x9e, 0xe8, 0x7b, 0x06, + 0x3d, 0x69, 0xc1, 0x97, 0x4c, 0x69, 0x28, 0xac, + 0xfc, 0xfb, 0x40, 0x99, 0xf8, 0xc4, 0xef, 0x67 }, + { 0x9f, 0xd1, 0xc4, 0x08, 0xfd, 0x75, 0xc3, 0x36, + 0x19, 0x3a, 0x2a, 0x14, 0xd9, 0x4f, 0x6a, 0xf5, + 0xad, 0xf0, 0x50, 0xb8, 0x03, 0x87, 0xb4, 0xb0, + 0x10, 0xfb, 0x29, 0xf4, 0xcc, 0x72, 0x70, 0x7c }, + { 0x13, 0xc8, 0x84, 0x80, 0xa5, 0xd0, 0x0d, 0x6c, + 0x8c, 0x7a, 0xd2, 0x11, 0x0d, 0x76, 0xa8, 0x2d, + 0x9b, 0x70, 0xf4, 0xfa, 0x66, 0x96, 0xd4, 0xe5, + 0xdd, 0x42, 0xa0, 0x66, 0xdc, 0xaf, 0x99, 0x20 }, + { 0x82, 0x0e, 0x72, 0x5e, 0xe2, 0x5f, 0xe8, 0xfd, + 0x3a, 0x8d, 0x5a, 0xbe, 0x4c, 0x46, 0xc3, 0xba, + 0x88, 0x9d, 0xe6, 0xfa, 0x91, 0x91, 0xaa, 0x22, + 0xba, 0x67, 0xd5, 0x70, 0x54, 0x21, 0x54, 0x2b }, + { 0x32, 0xd9, 0x3a, 0x0e, 0xb0, 0x2f, 0x42, 0xfb, + 0xbc, 0xaf, 0x2b, 0xad, 0x00, 0x85, 0xb2, 0x82, + 0xe4, 0x60, 0x46, 0xa4, 0xdf, 0x7a, 0xd1, 0x06, + 0x57, 0xc9, 0xd6, 0x47, 0x63, 0x75, 0xb9, 0x3e }, + { 0xad, 0xc5, 0x18, 0x79, 0x05, 0xb1, 0x66, 0x9c, + 0xd8, 0xec, 0x9c, 0x72, 0x1e, 0x19, 0x53, 0x78, + 0x6b, 0x9d, 0x89, 0xa9, 0xba, 0xe3, 0x07, 0x80, + 0xf1, 0xe1, 0xea, 0xb2, 0x4a, 0x00, 0x52, 0x3c }, + { 0xe9, 0x07, 0x56, 0xff, 0x7f, 0x9a, 0xd8, 0x10, + 0xb2, 0x39, 0xa1, 0x0c, 0xed, 0x2c, 0xf9, 0xb2, + 0x28, 0x43, 0x54, 0xc1, 0xf8, 0xc7, 0xe0, 0xac, + 0xcc, 0x24, 0x61, 0xdc, 0x79, 0x6d, 0x6e, 0x89 }, + { 0x12, 0x51, 0xf7, 0x6e, 0x56, 0x97, 0x84, 0x81, + 0x87, 0x53, 0x59, 0x80, 0x1d, 0xb5, 0x89, 0xa0, + 0xb2, 0x2f, 0x86, 0xd8, 0xd6, 0x34, 0xdc, 0x04, + 0x50, 0x6f, 0x32, 0x2e, 0xd7, 0x8f, 0x17, 0xe8 }, + { 0x3a, 0xfa, 0x89, 0x9f, 0xd9, 0x80, 0xe7, 0x3e, + 0xcb, 0x7f, 0x4d, 0x8b, 0x8f, 0x29, 0x1d, 0xc9, + 0xaf, 0x79, 0x6b, 0xc6, 0x5d, 0x27, 0xf9, 0x74, + 0xc6, 0xf1, 0x93, 0xc9, 0x19, 0x1a, 0x09, 0xfd }, + { 0xaa, 0x30, 0x5b, 0xe2, 0x6e, 0x5d, 0xed, 0xdc, + 0x3c, 0x10, 0x10, 0xcb, 0xc2, 0x13, 0xf9, 0x5f, + 0x05, 0x1c, 0x78, 0x5c, 0x5b, 0x43, 0x1e, 0x6a, + 0x7c, 0xd0, 0x48, 0xf1, 0x61, 0x78, 0x75, 0x28 }, + { 0x8e, 0xa1, 0x88, 0x4f, 0xf3, 0x2e, 0x9d, 0x10, + 0xf0, 0x39, 0xb4, 0x07, 0xd0, 0xd4, 0x4e, 0x7e, + 0x67, 0x0a, 0xbd, 0x88, 0x4a, 0xee, 0xe0, 0xfb, + 0x75, 0x7a, 0xe9, 0x4e, 0xaa, 0x97, 0x37, 0x3d }, + { 0xd4, 0x82, 0xb2, 0x15, 0x5d, 0x4d, 0xec, 0x6b, + 0x47, 0x36, 0xa1, 0xf1, 0x61, 0x7b, 0x53, 0xaa, + 0xa3, 0x73, 0x10, 0x27, 0x7d, 0x3f, 0xef, 0x0c, + 0x37, 0xad, 0x41, 0x76, 0x8f, 0xc2, 0x35, 0xb4 }, + { 0x4d, 0x41, 0x39, 0x71, 0x38, 0x7e, 0x7a, 0x88, + 0x98, 0xa8, 0xdc, 0x2a, 0x27, 0x50, 0x07, 0x78, + 0x53, 0x9e, 0xa2, 0x14, 0xa2, 0xdf, 0xe9, 0xb3, + 0xd7, 0xe8, 0xeb, 0xdc, 0xe5, 0xcf, 0x3d, 0xb3 }, + { 0x69, 0x6e, 0x5d, 0x46, 0xe6, 0xc5, 0x7e, 0x87, + 0x96, 0xe4, 0x73, 0x5d, 0x08, 0x91, 0x6e, 0x0b, + 0x79, 0x29, 0xb3, 0xcf, 0x29, 0x8c, 0x29, 0x6d, + 0x22, 0xe9, 0xd3, 0x01, 0x96, 0x53, 0x37, 0x1c }, + { 0x1f, 0x56, 0x47, 0xc1, 0xd3, 0xb0, 0x88, 0x22, + 0x88, 0x85, 0x86, 0x5c, 0x89, 0x40, 0x90, 0x8b, + 0xf4, 0x0d, 0x1a, 0x82, 0x72, 0x82, 0x19, 0x73, + 0xb1, 0x60, 0x00, 0x8e, 0x7a, 0x3c, 0xe2, 0xeb }, + { 0xb6, 0xe7, 0x6c, 0x33, 0x0f, 0x02, 0x1a, 0x5b, + 0xda, 0x65, 0x87, 0x50, 0x10, 0xb0, 0xed, 0xf0, + 0x91, 0x26, 0xc0, 0xf5, 0x10, 0xea, 0x84, 0x90, + 0x48, 0x19, 0x20, 0x03, 0xae, 0xf4, 0xc6, 0x1c }, + { 0x3c, 0xd9, 0x52, 0xa0, 0xbe, 0xad, 0xa4, 0x1a, + 0xbb, 0x42, 0x4c, 0xe4, 0x7f, 0x94, 0xb4, 0x2b, + 0xe6, 0x4e, 0x1f, 0xfb, 0x0f, 0xd0, 0x78, 0x22, + 0x76, 0x80, 0x79, 0x46, 0xd0, 0xd0, 0xbc, 0x55 }, + { 0x98, 0xd9, 0x26, 0x77, 0x43, 0x9b, 0x41, 0xb7, + 0xbb, 0x51, 0x33, 0x12, 0xaf, 0xb9, 0x2b, 0xcc, + 0x8e, 0xe9, 0x68, 0xb2, 0xe3, 0xb2, 0x38, 0xce, + 0xcb, 0x9b, 0x0f, 0x34, 0xc9, 0xbb, 0x63, 0xd0 }, + { 0xec, 0xbc, 0xa2, 0xcf, 0x08, 0xae, 0x57, 0xd5, + 0x17, 0xad, 0x16, 0x15, 0x8a, 0x32, 0xbf, 0xa7, + 0xdc, 0x03, 0x82, 0xea, 0xed, 0xa1, 0x28, 0xe9, + 0x18, 0x86, 0x73, 0x4c, 0x24, 0xa0, 0xb2, 0x9d }, + { 0x94, 0x2c, 0xc7, 0xc0, 0xb5, 0x2e, 0x2b, 0x16, + 0xa4, 0xb8, 0x9f, 0xa4, 0xfc, 0x7e, 0x0b, 0xf6, + 0x09, 0xe2, 0x9a, 0x08, 0xc1, 0xa8, 0x54, 0x34, + 0x52, 0xb7, 0x7c, 0x7b, 0xfd, 0x11, 0xbb, 0x28 }, + { 0x8a, 0x06, 0x5d, 0x8b, 0x61, 0xa0, 0xdf, 0xfb, + 0x17, 0x0d, 0x56, 0x27, 0x73, 0x5a, 0x76, 0xb0, + 0xe9, 0x50, 0x60, 0x37, 0x80, 0x8c, 0xba, 0x16, + 0xc3, 0x45, 0x00, 0x7c, 0x9f, 0x79, 0xcf, 0x8f }, + { 0x1b, 0x9f, 0xa1, 0x97, 0x14, 0x65, 0x9c, 0x78, + 0xff, 0x41, 0x38, 0x71, 0x84, 0x92, 0x15, 0x36, + 0x10, 0x29, 0xac, 0x80, 0x2b, 0x1c, 0xbc, 0xd5, + 0x4e, 0x40, 0x8b, 0xd8, 0x72, 0x87, 0xf8, 0x1f }, + { 0x8d, 0xab, 0x07, 0x1b, 0xcd, 0x6c, 0x72, 0x92, + 0xa9, 0xef, 0x72, 0x7b, 0x4a, 0xe0, 0xd8, 0x67, + 0x13, 0x30, 0x1d, 0xa8, 0x61, 0x8d, 0x9a, 0x48, + 0xad, 0xce, 0x55, 0xf3, 0x03, 0xa8, 0x69, 0xa1 }, + { 0x82, 0x53, 0xe3, 0xe7, 0xc7, 0xb6, 0x84, 0xb9, + 0xcb, 0x2b, 0xeb, 0x01, 0x4c, 0xe3, 0x30, 0xff, + 0x3d, 0x99, 0xd1, 0x7a, 0xbb, 0xdb, 0xab, 0xe4, + 0xf4, 0xd6, 0x74, 0xde, 0xd5, 0x3f, 0xfc, 0x6b }, + { 0xf1, 0x95, 0xf3, 0x21, 0xe9, 0xe3, 0xd6, 0xbd, + 0x7d, 0x07, 0x45, 0x04, 0xdd, 0x2a, 0xb0, 0xe6, + 0x24, 0x1f, 0x92, 0xe7, 0x84, 0xb1, 0xaa, 0x27, + 0x1f, 0xf6, 0x48, 0xb1, 0xca, 0xb6, 0xd7, 0xf6 }, + { 0x27, 0xe4, 0xcc, 0x72, 0x09, 0x0f, 0x24, 0x12, + 0x66, 0x47, 0x6a, 0x7c, 0x09, 0x49, 0x5f, 0x2d, + 0xb1, 0x53, 0xd5, 0xbc, 0xbd, 0x76, 0x19, 0x03, + 0xef, 0x79, 0x27, 0x5e, 0xc5, 0x6b, 0x2e, 0xd8 }, + { 0x89, 0x9c, 0x24, 0x05, 0x78, 0x8e, 0x25, 0xb9, + 0x9a, 0x18, 0x46, 0x35, 0x5e, 0x64, 0x6d, 0x77, + 0xcf, 0x40, 0x00, 0x83, 0x41, 0x5f, 0x7d, 0xc5, + 0xaf, 0xe6, 0x9d, 0x6e, 0x17, 0xc0, 0x00, 0x23 }, + { 0xa5, 0x9b, 0x78, 0xc4, 0x90, 0x57, 0x44, 0x07, + 0x6b, 0xfe, 0xe8, 0x94, 0xde, 0x70, 0x7d, 0x4f, + 0x12, 0x0b, 0x5c, 0x68, 0x93, 0xea, 0x04, 0x00, + 0x29, 0x7d, 0x0b, 0xb8, 0x34, 0x72, 0x76, 0x32 }, + { 0x59, 0xdc, 0x78, 0xb1, 0x05, 0x64, 0x97, 0x07, + 0xa2, 0xbb, 0x44, 0x19, 0xc4, 0x8f, 0x00, 0x54, + 0x00, 0xd3, 0x97, 0x3d, 0xe3, 0x73, 0x66, 0x10, + 0x23, 0x04, 0x35, 0xb1, 0x04, 0x24, 0xb2, 0x4f }, + { 0xc0, 0x14, 0x9d, 0x1d, 0x7e, 0x7a, 0x63, 0x53, + 0xa6, 0xd9, 0x06, 0xef, 0xe7, 0x28, 0xf2, 0xf3, + 0x29, 0xfe, 0x14, 0xa4, 0x14, 0x9a, 0x3e, 0xa7, + 0x76, 0x09, 0xbc, 0x42, 0xb9, 0x75, 0xdd, 0xfa }, + { 0xa3, 0x2f, 0x24, 0x14, 0x74, 0xa6, 0xc1, 0x69, + 0x32, 0xe9, 0x24, 0x3b, 0xe0, 0xcf, 0x09, 0xbc, + 0xdc, 0x7e, 0x0c, 0xa0, 0xe7, 0xa6, 0xa1, 0xb9, + 0xb1, 0xa0, 0xf0, 0x1e, 0x41, 0x50, 0x23, 0x77 }, + { 0xb2, 0x39, 0xb2, 0xe4, 0xf8, 0x18, 0x41, 0x36, + 0x1c, 0x13, 0x39, 0xf6, 0x8e, 0x2c, 0x35, 0x9f, + 0x92, 0x9a, 0xf9, 0xad, 0x9f, 0x34, 0xe0, 0x1a, + 0xab, 0x46, 0x31, 0xad, 0x6d, 0x55, 0x00, 0xb0 }, + { 0x85, 0xfb, 0x41, 0x9c, 0x70, 0x02, 0xa3, 0xe0, + 0xb4, 0xb6, 0xea, 0x09, 0x3b, 0x4c, 0x1a, 0xc6, + 0x93, 0x66, 0x45, 0xb6, 0x5d, 0xac, 0x5a, 0xc1, + 0x5a, 0x85, 0x28, 0xb7, 0xb9, 0x4c, 0x17, 0x54 }, + { 0x96, 0x19, 0x72, 0x06, 0x25, 0xf1, 0x90, 0xb9, + 0x3a, 0x3f, 0xad, 0x18, 0x6a, 0xb3, 0x14, 0x18, + 0x96, 0x33, 0xc0, 0xd3, 0xa0, 0x1e, 0x6f, 0x9b, + 0xc8, 0xc4, 0xa8, 0xf8, 0x2f, 0x38, 0x3d, 0xbf }, + { 0x7d, 0x62, 0x0d, 0x90, 0xfe, 0x69, 0xfa, 0x46, + 0x9a, 0x65, 0x38, 0x38, 0x89, 0x70, 0xa1, 0xaa, + 0x09, 0xbb, 0x48, 0xa2, 0xd5, 0x9b, 0x34, 0x7b, + 0x97, 0xe8, 0xce, 0x71, 0xf4, 0x8c, 0x7f, 0x46 }, + { 0x29, 0x43, 0x83, 0x56, 0x85, 0x96, 0xfb, 0x37, + 0xc7, 0x5b, 0xba, 0xcd, 0x97, 0x9c, 0x5f, 0xf6, + 0xf2, 0x0a, 0x55, 0x6b, 0xf8, 0x87, 0x9c, 0xc7, + 0x29, 0x24, 0x85, 0x5d, 0xf9, 0xb8, 0x24, 0x0e }, + { 0x16, 0xb1, 0x8a, 0xb3, 0x14, 0x35, 0x9c, 0x2b, + 0x83, 0x3c, 0x1c, 0x69, 0x86, 0xd4, 0x8c, 0x55, + 0xa9, 0xfc, 0x97, 0xcd, 0xe9, 0xa3, 0xc1, 0xf1, + 0x0a, 0x31, 0x77, 0x14, 0x0f, 0x73, 0xf7, 0x38 }, + { 0x8c, 0xbb, 0xdd, 0x14, 0xbc, 0x33, 0xf0, 0x4c, + 0xf4, 0x58, 0x13, 0xe4, 0xa1, 0x53, 0xa2, 0x73, + 0xd3, 0x6a, 0xda, 0xd5, 0xce, 0x71, 0xf4, 0x99, + 0xee, 0xb8, 0x7f, 0xb8, 0xac, 0x63, 0xb7, 0x29 }, + { 0x69, 0xc9, 0xa4, 0x98, 0xdb, 0x17, 0x4e, 0xca, + 0xef, 0xcc, 0x5a, 0x3a, 0xc9, 0xfd, 0xed, 0xf0, + 0xf8, 0x13, 0xa5, 0xbe, 0xc7, 0x27, 0xf1, 0xe7, + 0x75, 0xba, 0xbd, 0xec, 0x77, 0x18, 0x81, 0x6e }, + { 0xb4, 0x62, 0xc3, 0xbe, 0x40, 0x44, 0x8f, 0x1d, + 0x4f, 0x80, 0x62, 0x62, 0x54, 0xe5, 0x35, 0xb0, + 0x8b, 0xc9, 0xcd, 0xcf, 0xf5, 0x99, 0xa7, 0x68, + 0x57, 0x8d, 0x4b, 0x28, 0x81, 0xa8, 0xe3, 0xf0 }, + { 0x55, 0x3e, 0x9d, 0x9c, 0x5f, 0x36, 0x0a, 0xc0, + 0xb7, 0x4a, 0x7d, 0x44, 0xe5, 0xa3, 0x91, 0xda, + 0xd4, 0xce, 0xd0, 0x3e, 0x0c, 0x24, 0x18, 0x3b, + 0x7e, 0x8e, 0xca, 0xbd, 0xf1, 0x71, 0x5a, 0x64 }, + { 0x7a, 0x7c, 0x55, 0xa5, 0x6f, 0xa9, 0xae, 0x51, + 0xe6, 0x55, 0xe0, 0x19, 0x75, 0xd8, 0xa6, 0xff, + 0x4a, 0xe9, 0xe4, 0xb4, 0x86, 0xfc, 0xbe, 0x4e, + 0xac, 0x04, 0x45, 0x88, 0xf2, 0x45, 0xeb, 0xea }, + { 0x2a, 0xfd, 0xf3, 0xc8, 0x2a, 0xbc, 0x48, 0x67, + 0xf5, 0xde, 0x11, 0x12, 0x86, 0xc2, 0xb3, 0xbe, + 0x7d, 0x6e, 0x48, 0x65, 0x7b, 0xa9, 0x23, 0xcf, + 0xbf, 0x10, 0x1a, 0x6d, 0xfc, 0xf9, 0xdb, 0x9a }, + { 0x41, 0x03, 0x7d, 0x2e, 0xdc, 0xdc, 0xe0, 0xc4, + 0x9b, 0x7f, 0xb4, 0xa6, 0xaa, 0x09, 0x99, 0xca, + 0x66, 0x97, 0x6c, 0x74, 0x83, 0xaf, 0xe6, 0x31, + 0xd4, 0xed, 0xa2, 0x83, 0x14, 0x4f, 0x6d, 0xfc }, + { 0xc4, 0x46, 0x6f, 0x84, 0x97, 0xca, 0x2e, 0xeb, + 0x45, 0x83, 0xa0, 0xb0, 0x8e, 0x9d, 0x9a, 0xc7, + 0x43, 0x95, 0x70, 0x9f, 0xda, 0x10, 0x9d, 0x24, + 0xf2, 0xe4, 0x46, 0x21, 0x96, 0x77, 0x9c, 0x5d }, + { 0x75, 0xf6, 0x09, 0x33, 0x8a, 0xa6, 0x7d, 0x96, + 0x9a, 0x2a, 0xe2, 0xa2, 0x36, 0x2b, 0x2d, 0xa9, + 0xd7, 0x7c, 0x69, 0x5d, 0xfd, 0x1d, 0xf7, 0x22, + 0x4a, 0x69, 0x01, 0xdb, 0x93, 0x2c, 0x33, 0x64 }, + { 0x68, 0x60, 0x6c, 0xeb, 0x98, 0x9d, 0x54, 0x88, + 0xfc, 0x7c, 0xf6, 0x49, 0xf3, 0xd7, 0xc2, 0x72, + 0xef, 0x05, 0x5d, 0xa1, 0xa9, 0x3f, 0xae, 0xcd, + 0x55, 0xfe, 0x06, 0xf6, 0x96, 0x70, 0x98, 0xca }, + { 0x44, 0x34, 0x6b, 0xde, 0xb7, 0xe0, 0x52, 0xf6, + 0x25, 0x50, 0x48, 0xf0, 0xd9, 0xb4, 0x2c, 0x42, + 0x5b, 0xab, 0x9c, 0x3d, 0xd2, 0x41, 0x68, 0x21, + 0x2c, 0x3e, 0xcf, 0x1e, 0xbf, 0x34, 0xe6, 0xae }, + { 0x8e, 0x9c, 0xf6, 0xe1, 0xf3, 0x66, 0x47, 0x1f, + 0x2a, 0xc7, 0xd2, 0xee, 0x9b, 0x5e, 0x62, 0x66, + 0xfd, 0xa7, 0x1f, 0x8f, 0x2e, 0x41, 0x09, 0xf2, + 0x23, 0x7e, 0xd5, 0xf8, 0x81, 0x3f, 0xc7, 0x18 }, + { 0x84, 0xbb, 0xeb, 0x84, 0x06, 0xd2, 0x50, 0x95, + 0x1f, 0x8c, 0x1b, 0x3e, 0x86, 0xa7, 0xc0, 0x10, + 0x08, 0x29, 0x21, 0x83, 0x3d, 0xfd, 0x95, 0x55, + 0xa2, 0xf9, 0x09, 0xb1, 0x08, 0x6e, 0xb4, 0xb8 }, + { 0xee, 0x66, 0x6f, 0x3e, 0xef, 0x0f, 0x7e, 0x2a, + 0x9c, 0x22, 0x29, 0x58, 0xc9, 0x7e, 0xaf, 0x35, + 0xf5, 0x1c, 0xed, 0x39, 0x3d, 0x71, 0x44, 0x85, + 0xab, 0x09, 0xa0, 0x69, 0x34, 0x0f, 0xdf, 0x88 }, + { 0xc1, 0x53, 0xd3, 0x4a, 0x65, 0xc4, 0x7b, 0x4a, + 0x62, 0xc5, 0xca, 0xcf, 0x24, 0x01, 0x09, 0x75, + 0xd0, 0x35, 0x6b, 0x2f, 0x32, 0xc8, 0xf5, 0xda, + 0x53, 0x0d, 0x33, 0x88, 0x16, 0xad, 0x5d, 0xe6 }, + { 0x9f, 0xc5, 0x45, 0x01, 0x09, 0xe1, 0xb7, 0x79, + 0xf6, 0xc7, 0xae, 0x79, 0xd5, 0x6c, 0x27, 0x63, + 0x5c, 0x8d, 0xd4, 0x26, 0xc5, 0xa9, 0xd5, 0x4e, + 0x25, 0x78, 0xdb, 0x98, 0x9b, 0x8c, 0x3b, 0x4e }, + { 0xd1, 0x2b, 0xf3, 0x73, 0x2e, 0xf4, 0xaf, 0x5c, + 0x22, 0xfa, 0x90, 0x35, 0x6a, 0xf8, 0xfc, 0x50, + 0xfc, 0xb4, 0x0f, 0x8f, 0x2e, 0xa5, 0xc8, 0x59, + 0x47, 0x37, 0xa3, 0xb3, 0xd5, 0xab, 0xdb, 0xd7 }, + { 0x11, 0x03, 0x0b, 0x92, 0x89, 0xbb, 0xa5, 0xaf, + 0x65, 0x26, 0x06, 0x72, 0xab, 0x6f, 0xee, 0x88, + 0xb8, 0x74, 0x20, 0xac, 0xef, 0x4a, 0x17, 0x89, + 0xa2, 0x07, 0x3b, 0x7e, 0xc2, 0xf2, 0xa0, 0x9e }, + { 0x69, 0xcb, 0x19, 0x2b, 0x84, 0x44, 0x00, 0x5c, + 0x8c, 0x0c, 0xeb, 0x12, 0xc8, 0x46, 0x86, 0x07, + 0x68, 0x18, 0x8c, 0xda, 0x0a, 0xec, 0x27, 0xa9, + 0xc8, 0xa5, 0x5c, 0xde, 0xe2, 0x12, 0x36, 0x32 }, + { 0xdb, 0x44, 0x4c, 0x15, 0x59, 0x7b, 0x5f, 0x1a, + 0x03, 0xd1, 0xf9, 0xed, 0xd1, 0x6e, 0x4a, 0x9f, + 0x43, 0xa6, 0x67, 0xcc, 0x27, 0x51, 0x75, 0xdf, + 0xa2, 0xb7, 0x04, 0xe3, 0xbb, 0x1a, 0x9b, 0x83 }, + { 0x3f, 0xb7, 0x35, 0x06, 0x1a, 0xbc, 0x51, 0x9d, + 0xfe, 0x97, 0x9e, 0x54, 0xc1, 0xee, 0x5b, 0xfa, + 0xd0, 0xa9, 0xd8, 0x58, 0xb3, 0x31, 0x5b, 0xad, + 0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd } +}; + +static bool __init blake2s_selftest(void) +{ + u8 key[BLAKE2S_KEY_SIZE]; + u8 buf[ARRAY_SIZE(blake2s_testvecs)]; + u8 hash[BLAKE2S_HASH_SIZE]; + size_t i; + bool success = true; + + for (i = 0; i < BLAKE2S_KEY_SIZE; ++i) + key[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) + buf[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_keyed_testvecs); ++i) { + blake2s(hash, buf, key, BLAKE2S_HASH_SIZE, i, BLAKE2S_KEY_SIZE); + if (memcmp(hash, blake2s_keyed_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s keyed self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) { + blake2s(hash, buf, NULL, BLAKE2S_HASH_SIZE, i, 0); + if (memcmp(hash, blake2s_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s unkeyed self-test %zu: FAIL\n", i + i); + success = false; + } + } + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20.c b/net/wireguard/crypto/zinc/selftest/chacha20.c new file mode 100644 index 0000000000000..1a2390aaf6c23 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20.c @@ -0,0 +1,2698 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20_testvec { + const u8 *input, *output, *key; + u64 nonce; + size_t ilen; +}; + +struct hchacha20_testvec { + u8 key[HCHACHA20_KEY_SIZE]; + u8 nonce[HCHACHA20_NONCE_SIZE]; + u8 output[CHACHA20_KEY_SIZE]; +}; + +/* These test vectors are generated by reference implementations and are + * designed to check chacha20 implementation block handling, as well as from + * the draft-arciszewski-xchacha-01 document. + */ + +static const u8 input01[] __initconst = { }; +static const u8 output01[] __initconst = { }; +static const u8 key01[] __initconst = { + 0x09, 0xf4, 0xe8, 0x57, 0x10, 0xf2, 0x12, 0xc3, + 0xc6, 0x91, 0xc4, 0x09, 0x97, 0x46, 0xef, 0xfe, + 0x02, 0x00, 0xe4, 0x5c, 0x82, 0xed, 0x16, 0xf3, + 0x32, 0xbe, 0xec, 0x7a, 0xe6, 0x68, 0x12, 0x26 +}; +enum { nonce01 = 0x3834e2afca3c66d3ULL }; + +static const u8 input02[] __initconst = { + 0x9d +}; +static const u8 output02[] __initconst = { + 0x94 +}; +static const u8 key02[] __initconst = { + 0x8c, 0x01, 0xac, 0xaf, 0x62, 0x63, 0x56, 0x7a, + 0xad, 0x23, 0x4c, 0x58, 0x29, 0x29, 0xbe, 0xab, + 0xe9, 0xf8, 0xdf, 0x6c, 0x8c, 0x74, 0x4d, 0x7d, + 0x13, 0x94, 0x10, 0x02, 0x3d, 0x8e, 0x9f, 0x94 +}; +enum { nonce02 = 0x5d1b3bfdedd9f73aULL }; + +static const u8 input03[] __initconst = { + 0x04, 0x16 +}; +static const u8 output03[] __initconst = { + 0x92, 0x07 +}; +static const u8 key03[] __initconst = { + 0x22, 0x0c, 0x79, 0x2c, 0x38, 0x51, 0xbe, 0x99, + 0xa9, 0x59, 0x24, 0x50, 0xef, 0x87, 0x38, 0xa6, + 0xa0, 0x97, 0x20, 0xcb, 0xb4, 0x0c, 0x94, 0x67, + 0x1f, 0x98, 0xdc, 0xc4, 0x83, 0xbc, 0x35, 0x4d +}; +enum { nonce03 = 0x7a3353ad720a3e2eULL }; + +static const u8 input04[] __initconst = { + 0xc7, 0xcc, 0xd0 +}; +static const u8 output04[] __initconst = { + 0xd8, 0x41, 0x80 +}; +static const u8 key04[] __initconst = { + 0x81, 0x5e, 0x12, 0x01, 0xc4, 0x36, 0x15, 0x03, + 0x11, 0xa0, 0xe9, 0x86, 0xbb, 0x5a, 0xdc, 0x45, + 0x7d, 0x5e, 0x98, 0xf8, 0x06, 0x76, 0x1c, 0xec, + 0xc0, 0xf7, 0xca, 0x4e, 0x99, 0xd9, 0x42, 0x38 +}; +enum { nonce04 = 0x6816e2fc66176da2ULL }; + +static const u8 input05[] __initconst = { + 0x48, 0xf1, 0x31, 0x5f +}; +static const u8 output05[] __initconst = { + 0x48, 0xf7, 0x13, 0x67 +}; +static const u8 key05[] __initconst = { + 0x3f, 0xd6, 0xb6, 0x5e, 0x2f, 0xda, 0x82, 0x39, + 0x97, 0x06, 0xd3, 0x62, 0x4f, 0xbd, 0xcb, 0x9b, + 0x1d, 0xe6, 0x4a, 0x76, 0xab, 0xdd, 0x14, 0x50, + 0x59, 0x21, 0xe3, 0xb2, 0xc7, 0x95, 0xbc, 0x45 +}; +enum { nonce05 = 0xc41a7490e228cc42ULL }; + +static const u8 input06[] __initconst = { + 0xae, 0xa2, 0x85, 0x1d, 0xc8 +}; +static const u8 output06[] __initconst = { + 0xfa, 0xff, 0x45, 0x6b, 0x6f +}; +static const u8 key06[] __initconst = { + 0x04, 0x8d, 0xea, 0x67, 0x20, 0x78, 0xfb, 0x8f, + 0x49, 0x80, 0x35, 0xb5, 0x7b, 0xe4, 0x31, 0x74, + 0x57, 0x43, 0x3a, 0x64, 0x64, 0xb9, 0xe6, 0x23, + 0x4d, 0xfe, 0xb8, 0x7b, 0x71, 0x4d, 0x9d, 0x21 +}; +enum { nonce06 = 0x251366db50b10903ULL }; + +static const u8 input07[] __initconst = { + 0x1a, 0x32, 0x85, 0xb6, 0xe8, 0x52 +}; +static const u8 output07[] __initconst = { + 0xd3, 0x5f, 0xf0, 0x07, 0x69, 0xec +}; +static const u8 key07[] __initconst = { + 0xbf, 0x2d, 0x42, 0x99, 0x97, 0x76, 0x04, 0xad, + 0xd3, 0x8f, 0x6e, 0x6a, 0x34, 0x85, 0xaf, 0x81, + 0xef, 0x36, 0x33, 0xd5, 0x43, 0xa2, 0xaa, 0x08, + 0x0f, 0x77, 0x42, 0x83, 0x58, 0xc5, 0x42, 0x2a +}; +enum { nonce07 = 0xe0796da17dba9b58ULL }; + +static const u8 input08[] __initconst = { + 0x40, 0xae, 0xcd, 0xe4, 0x3d, 0x22, 0xe0 +}; +static const u8 output08[] __initconst = { + 0xfd, 0x8a, 0x9f, 0x3d, 0x05, 0xc9, 0xd3 +}; +static const u8 key08[] __initconst = { + 0xdc, 0x3f, 0x41, 0xe3, 0x23, 0x2a, 0x8d, 0xf6, + 0x41, 0x2a, 0xa7, 0x66, 0x05, 0x68, 0xe4, 0x7b, + 0xc4, 0x58, 0xd6, 0xcc, 0xdf, 0x0d, 0xc6, 0x25, + 0x1b, 0x61, 0x32, 0x12, 0x4e, 0xf1, 0xe6, 0x29 +}; +enum { nonce08 = 0xb1d2536d9e159832ULL }; + +static const u8 input09[] __initconst = { + 0xba, 0x1d, 0x14, 0x16, 0x9f, 0x83, 0x67, 0x24 +}; +static const u8 output09[] __initconst = { + 0x7c, 0xe3, 0x78, 0x1d, 0xa2, 0xe7, 0xe9, 0x39 +}; +static const u8 key09[] __initconst = { + 0x17, 0x55, 0x90, 0x52, 0xa4, 0xce, 0x12, 0xae, + 0xd4, 0xfd, 0xd4, 0xfb, 0xd5, 0x18, 0x59, 0x50, + 0x4e, 0x51, 0x99, 0x32, 0x09, 0x31, 0xfc, 0xf7, + 0x27, 0x10, 0x8e, 0xa2, 0x4b, 0xa5, 0xf5, 0x62 +}; +enum { nonce09 = 0x495fc269536d003ULL }; + +static const u8 input10[] __initconst = { + 0x09, 0xfd, 0x3c, 0x0b, 0x3d, 0x0e, 0xf3, 0x9d, + 0x27 +}; +static const u8 output10[] __initconst = { + 0xdc, 0xe4, 0x33, 0x60, 0x0c, 0x07, 0xcb, 0x51, + 0x6b +}; +static const u8 key10[] __initconst = { + 0x4e, 0x00, 0x72, 0x37, 0x0f, 0x52, 0x4d, 0x6f, + 0x37, 0x50, 0x3c, 0xb3, 0x51, 0x81, 0x49, 0x16, + 0x7e, 0xfd, 0xb1, 0x51, 0x72, 0x2e, 0xe4, 0x16, + 0x68, 0x5c, 0x5b, 0x8a, 0xc3, 0x90, 0x70, 0x04 +}; +enum { nonce10 = 0x1ad9d1114d88cbbdULL }; + +static const u8 input11[] __initconst = { + 0x70, 0x18, 0x52, 0x85, 0xba, 0x66, 0xff, 0x2c, + 0x9a, 0x46 +}; +static const u8 output11[] __initconst = { + 0xf5, 0x2a, 0x7a, 0xfd, 0x31, 0x7c, 0x91, 0x41, + 0xb1, 0xcf +}; +static const u8 key11[] __initconst = { + 0x48, 0xb4, 0xd0, 0x7c, 0x88, 0xd1, 0x96, 0x0d, + 0x80, 0x33, 0xb4, 0xd5, 0x31, 0x9a, 0x88, 0xca, + 0x14, 0xdc, 0xf0, 0xa8, 0xf3, 0xac, 0xb8, 0x47, + 0x75, 0x86, 0x7c, 0x88, 0x50, 0x11, 0x43, 0x40 +}; +enum { nonce11 = 0x47c35dd1f4f8aa4fULL }; + +static const u8 input12[] __initconst = { + 0x9e, 0x8e, 0x3d, 0x2a, 0x05, 0xfd, 0xe4, 0x90, + 0x24, 0x1c, 0xd3 +}; +static const u8 output12[] __initconst = { + 0x97, 0x72, 0x40, 0x9f, 0xc0, 0x6b, 0x05, 0x33, + 0x42, 0x7e, 0x28 +}; +static const u8 key12[] __initconst = { + 0xee, 0xff, 0x33, 0x33, 0xe0, 0x28, 0xdf, 0xa2, + 0xb6, 0x5e, 0x25, 0x09, 0x52, 0xde, 0xa5, 0x9c, + 0x8f, 0x95, 0xa9, 0x03, 0x77, 0x0f, 0xbe, 0xa1, + 0xd0, 0x7d, 0x73, 0x2f, 0xf8, 0x7e, 0x51, 0x44 +}; +enum { nonce12 = 0xc22d044dc6ea4af3ULL }; + +static const u8 input13[] __initconst = { + 0x9c, 0x16, 0xa2, 0x22, 0x4d, 0xbe, 0x04, 0x9a, + 0xb3, 0xb5, 0xc6, 0x58 +}; +static const u8 output13[] __initconst = { + 0xf0, 0x81, 0xdb, 0x6d, 0xa3, 0xe9, 0xb2, 0xc6, + 0x32, 0x50, 0x16, 0x9f +}; +static const u8 key13[] __initconst = { + 0x96, 0xb3, 0x01, 0xd2, 0x7a, 0x8c, 0x94, 0x09, + 0x4f, 0x58, 0xbe, 0x80, 0xcc, 0xa9, 0x7e, 0x2d, + 0xad, 0x58, 0x3b, 0x63, 0xb8, 0x5c, 0x17, 0xce, + 0xbf, 0x43, 0x33, 0x7a, 0x7b, 0x82, 0x28, 0x2f +}; +enum { nonce13 = 0x2a5d05d88cd7b0daULL }; + +static const u8 input14[] __initconst = { + 0x57, 0x4f, 0xaa, 0x30, 0xe6, 0x23, 0x50, 0x86, + 0x91, 0xa5, 0x60, 0x96, 0x2b +}; +static const u8 output14[] __initconst = { + 0x6c, 0x1f, 0x3b, 0x42, 0xb6, 0x2f, 0xf0, 0xbd, + 0x76, 0x60, 0xc7, 0x7e, 0x8d +}; +static const u8 key14[] __initconst = { + 0x22, 0x85, 0xaf, 0x8f, 0xa3, 0x53, 0xa0, 0xc4, + 0xb5, 0x75, 0xc0, 0xba, 0x30, 0x92, 0xc3, 0x32, + 0x20, 0x5a, 0x8f, 0x7e, 0x93, 0xda, 0x65, 0x18, + 0xd1, 0xf6, 0x9a, 0x9b, 0x8f, 0x85, 0x30, 0xe6 +}; +enum { nonce14 = 0xf9946c166aa4475fULL }; + +static const u8 input15[] __initconst = { + 0x89, 0x81, 0xc7, 0xe2, 0x00, 0xac, 0x52, 0x70, + 0xa4, 0x79, 0xab, 0xeb, 0x74, 0xf7 +}; +static const u8 output15[] __initconst = { + 0xb4, 0xd0, 0xa9, 0x9d, 0x15, 0x5f, 0x48, 0xd6, + 0x00, 0x7e, 0x4c, 0x77, 0x5a, 0x46 +}; +static const u8 key15[] __initconst = { + 0x0a, 0x66, 0x36, 0xca, 0x5d, 0x82, 0x23, 0xb6, + 0xe4, 0x9b, 0xad, 0x5e, 0xd0, 0x7f, 0xf6, 0x7a, + 0x7b, 0x03, 0xa7, 0x4c, 0xfd, 0xec, 0xd5, 0xa1, + 0xfc, 0x25, 0x54, 0xda, 0x5a, 0x5c, 0xf0, 0x2c +}; +enum { nonce15 = 0x9ab2b87a35e772c8ULL }; + +static const u8 input16[] __initconst = { + 0x5f, 0x09, 0xc0, 0x8b, 0x1e, 0xde, 0xca, 0xd9, + 0xb7, 0x5c, 0x23, 0xc9, 0x55, 0x1e, 0xcf +}; +static const u8 output16[] __initconst = { + 0x76, 0x9b, 0x53, 0xf3, 0x66, 0x88, 0x28, 0x60, + 0x98, 0x80, 0x2c, 0xa8, 0x80, 0xa6, 0x48 +}; +static const u8 key16[] __initconst = { + 0x80, 0xb5, 0x51, 0xdf, 0x17, 0x5b, 0xb0, 0xef, + 0x8b, 0x5b, 0x2e, 0x3e, 0xc5, 0xe3, 0xa5, 0x86, + 0xac, 0x0d, 0x8e, 0x32, 0x90, 0x9d, 0x82, 0x27, + 0xf1, 0x23, 0x26, 0xc3, 0xea, 0x55, 0xb6, 0x63 +}; +enum { nonce16 = 0xa82e9d39e4d02ef5ULL }; + +static const u8 input17[] __initconst = { + 0x87, 0x0b, 0x36, 0x71, 0x7c, 0xb9, 0x0b, 0x80, + 0x4d, 0x77, 0x5c, 0x4f, 0xf5, 0x51, 0x0e, 0x1a +}; +static const u8 output17[] __initconst = { + 0xf1, 0x12, 0x4a, 0x8a, 0xd9, 0xd0, 0x08, 0x67, + 0x66, 0xd7, 0x34, 0xea, 0x32, 0x3b, 0x54, 0x0e +}; +static const u8 key17[] __initconst = { + 0xfb, 0x71, 0x5f, 0x3f, 0x7a, 0xc0, 0x9a, 0xc8, + 0xc8, 0xcf, 0xe8, 0xbc, 0xfb, 0x09, 0xbf, 0x89, + 0x6a, 0xef, 0xd5, 0xe5, 0x36, 0x87, 0x14, 0x76, + 0x00, 0xb9, 0x32, 0x28, 0xb2, 0x00, 0x42, 0x53 +}; +enum { nonce17 = 0x229b87e73d557b96ULL }; + +static const u8 input18[] __initconst = { + 0x38, 0x42, 0xb5, 0x37, 0xb4, 0x3d, 0xfe, 0x59, + 0x38, 0x68, 0x88, 0xfa, 0x89, 0x8a, 0x5f, 0x90, + 0x3c +}; +static const u8 output18[] __initconst = { + 0xac, 0xad, 0x14, 0xe8, 0x7e, 0xd7, 0xce, 0x96, + 0x3d, 0xb3, 0x78, 0x85, 0x22, 0x5a, 0xcb, 0x39, + 0xd4 +}; +static const u8 key18[] __initconst = { + 0xe1, 0xc1, 0xa8, 0xe0, 0x91, 0xe7, 0x38, 0x66, + 0x80, 0x17, 0x12, 0x3c, 0x5e, 0x2d, 0xbb, 0xea, + 0xeb, 0x6c, 0x8b, 0xc8, 0x1b, 0x6f, 0x7c, 0xea, + 0x50, 0x57, 0x23, 0x1e, 0x65, 0x6f, 0x6d, 0x81 +}; +enum { nonce18 = 0xfaf5fcf8f30e57a9ULL }; + +static const u8 input19[] __initconst = { + 0x1c, 0x4a, 0x30, 0x26, 0xef, 0x9a, 0x32, 0xa7, + 0x8f, 0xe5, 0xc0, 0x0f, 0x30, 0x3a, 0xbf, 0x38, + 0x54, 0xba +}; +static const u8 output19[] __initconst = { + 0x57, 0x67, 0x54, 0x4f, 0x31, 0xd6, 0xef, 0x35, + 0x0b, 0xd9, 0x52, 0xa7, 0x46, 0x7d, 0x12, 0x17, + 0x1e, 0xe3 +}; +static const u8 key19[] __initconst = { + 0x5a, 0x79, 0xc1, 0xea, 0x33, 0xb3, 0xc7, 0x21, + 0xec, 0xf8, 0xcb, 0xd2, 0x58, 0x96, 0x23, 0xd6, + 0x4d, 0xed, 0x2f, 0xdf, 0x8a, 0x79, 0xe6, 0x8b, + 0x38, 0xa3, 0xc3, 0x7a, 0x33, 0xda, 0x02, 0xc7 +}; +enum { nonce19 = 0x2b23b61840429604ULL }; + +static const u8 input20[] __initconst = { + 0xab, 0xe9, 0x32, 0xbb, 0x35, 0x17, 0xe0, 0x60, + 0x80, 0xb1, 0x27, 0xdc, 0xe6, 0x62, 0x9e, 0x0c, + 0x77, 0xf4, 0x50 +}; +static const u8 output20[] __initconst = { + 0x54, 0x6d, 0xaa, 0xfc, 0x08, 0xfb, 0x71, 0xa8, + 0xd6, 0x1d, 0x7d, 0xf3, 0x45, 0x10, 0xb5, 0x4c, + 0xcc, 0x4b, 0x45 +}; +static const u8 key20[] __initconst = { + 0xa3, 0xfd, 0x3d, 0xa9, 0xeb, 0xea, 0x2c, 0x69, + 0xcf, 0x59, 0x38, 0x13, 0x5b, 0xa7, 0x53, 0x8f, + 0x5e, 0xa2, 0x33, 0x86, 0x4c, 0x75, 0x26, 0xaf, + 0x35, 0x12, 0x09, 0x71, 0x81, 0xea, 0x88, 0x66 +}; +enum { nonce20 = 0x7459667a8fadff58ULL }; + +static const u8 input21[] __initconst = { + 0xa6, 0x82, 0x21, 0x23, 0xad, 0x27, 0x3f, 0xc6, + 0xd7, 0x16, 0x0d, 0x6d, 0x24, 0x15, 0x54, 0xc5, + 0x96, 0x72, 0x59, 0x8a +}; +static const u8 output21[] __initconst = { + 0x5f, 0x34, 0x32, 0xea, 0x06, 0xd4, 0x9e, 0x01, + 0xdc, 0x32, 0x32, 0x40, 0x66, 0x73, 0x6d, 0x4a, + 0x6b, 0x12, 0x20, 0xe8 +}; +static const u8 key21[] __initconst = { + 0x96, 0xfd, 0x13, 0x23, 0xa9, 0x89, 0x04, 0xe6, + 0x31, 0xa5, 0x2c, 0xc1, 0x40, 0xd5, 0x69, 0x5c, + 0x32, 0x79, 0x56, 0xe0, 0x29, 0x93, 0x8f, 0xe8, + 0x5f, 0x65, 0x53, 0x7f, 0xc1, 0xe9, 0xaf, 0xaf +}; +enum { nonce21 = 0xba8defee9d8e13b5ULL }; + +static const u8 input22[] __initconst = { + 0xb8, 0x32, 0x1a, 0x81, 0xd8, 0x38, 0x89, 0x5a, + 0xb0, 0x05, 0xbe, 0xf4, 0xd2, 0x08, 0xc6, 0xee, + 0x79, 0x7b, 0x3a, 0x76, 0x59 +}; +static const u8 output22[] __initconst = { + 0xb7, 0xba, 0xae, 0x80, 0xe4, 0x9f, 0x79, 0x84, + 0x5a, 0x48, 0x50, 0x6d, 0xcb, 0xd0, 0x06, 0x0c, + 0x15, 0x63, 0xa7, 0x5e, 0xbd +}; +static const u8 key22[] __initconst = { + 0x0f, 0x35, 0x3d, 0xeb, 0x5f, 0x0a, 0x82, 0x0d, + 0x24, 0x59, 0x71, 0xd8, 0xe6, 0x2d, 0x5f, 0xe1, + 0x7e, 0x0c, 0xae, 0xf6, 0xdc, 0x2c, 0xc5, 0x4a, + 0x38, 0x88, 0xf2, 0xde, 0xd9, 0x5f, 0x76, 0x7c +}; +enum { nonce22 = 0xe77f1760e9f5e192ULL }; + +static const u8 input23[] __initconst = { + 0x4b, 0x1e, 0x79, 0x99, 0xcf, 0xef, 0x64, 0x4b, + 0xb0, 0x66, 0xae, 0x99, 0x2e, 0x68, 0x97, 0xf5, + 0x5d, 0x9b, 0x3f, 0x7a, 0xa9, 0xd9 +}; +static const u8 output23[] __initconst = { + 0x5f, 0xa4, 0x08, 0x39, 0xca, 0xfa, 0x2b, 0x83, + 0x5d, 0x95, 0x70, 0x7c, 0x2e, 0xd4, 0xae, 0xfa, + 0x45, 0x4a, 0x77, 0x7f, 0xa7, 0x65 +}; +static const u8 key23[] __initconst = { + 0x4a, 0x06, 0x83, 0x64, 0xaa, 0xe3, 0x38, 0x32, + 0x28, 0x5d, 0xa4, 0xb2, 0x5a, 0xee, 0xcf, 0x8e, + 0x19, 0x67, 0xf1, 0x09, 0xe8, 0xc9, 0xf6, 0x40, + 0x02, 0x6d, 0x0b, 0xde, 0xfa, 0x81, 0x03, 0xb1 +}; +enum { nonce23 = 0x9b3f349158709849ULL }; + +static const u8 input24[] __initconst = { + 0xc6, 0xfc, 0x47, 0x5e, 0xd8, 0xed, 0xa9, 0xe5, + 0x4f, 0x82, 0x79, 0x35, 0xee, 0x3e, 0x7e, 0x3e, + 0x35, 0x70, 0x6e, 0xfa, 0x6d, 0x08, 0xe8 +}; +static const u8 output24[] __initconst = { + 0x3b, 0xc5, 0xf8, 0xc2, 0xbf, 0x2b, 0x90, 0x33, + 0xa6, 0xae, 0xf5, 0x5a, 0x65, 0xb3, 0x3d, 0xe1, + 0xcd, 0x5f, 0x55, 0xfa, 0xe7, 0xa5, 0x4a +}; +static const u8 key24[] __initconst = { + 0x00, 0x24, 0xc3, 0x65, 0x5f, 0xe6, 0x31, 0xbb, + 0x6d, 0xfc, 0x20, 0x7b, 0x1b, 0xa8, 0x96, 0x26, + 0x55, 0x21, 0x62, 0x25, 0x7e, 0xba, 0x23, 0x97, + 0xc9, 0xb8, 0x53, 0xa8, 0xef, 0xab, 0xad, 0x61 +}; +enum { nonce24 = 0x13ee0b8f526177c3ULL }; + +static const u8 input25[] __initconst = { + 0x33, 0x07, 0x16, 0xb1, 0x34, 0x33, 0x67, 0x04, + 0x9b, 0x0a, 0xce, 0x1b, 0xe9, 0xde, 0x1a, 0xec, + 0xd0, 0x55, 0xfb, 0xc6, 0x33, 0xaf, 0x2d, 0xe3 +}; +static const u8 output25[] __initconst = { + 0x05, 0x93, 0x10, 0xd1, 0x58, 0x6f, 0x68, 0x62, + 0x45, 0xdb, 0x91, 0xae, 0x70, 0xcf, 0xd4, 0x5f, + 0xee, 0xdf, 0xd5, 0xba, 0x9e, 0xde, 0x68, 0xe6 +}; +static const u8 key25[] __initconst = { + 0x83, 0xa9, 0x4f, 0x5d, 0x74, 0xd5, 0x91, 0xb3, + 0xc9, 0x97, 0x19, 0x15, 0xdb, 0x0d, 0x0b, 0x4a, + 0x3d, 0x55, 0xcf, 0xab, 0xb2, 0x05, 0x21, 0x35, + 0x45, 0x50, 0xeb, 0xf8, 0xf5, 0xbf, 0x36, 0x35 +}; +enum { nonce25 = 0x7c6f459e49ebfebcULL }; + +static const u8 input26[] __initconst = { + 0xc2, 0xd4, 0x7a, 0xa3, 0x92, 0xe1, 0xac, 0x46, + 0x1a, 0x15, 0x38, 0xc9, 0xb5, 0xfd, 0xdf, 0x84, + 0x38, 0xbc, 0x6b, 0x1d, 0xb0, 0x83, 0x43, 0x04, + 0x39 +}; +static const u8 output26[] __initconst = { + 0x7f, 0xde, 0xd6, 0x87, 0xcc, 0x34, 0xf4, 0x12, + 0xae, 0x55, 0xa5, 0x89, 0x95, 0x29, 0xfc, 0x18, + 0xd8, 0xc7, 0x7c, 0xd3, 0xcb, 0x85, 0x95, 0x21, + 0xd2 +}; +static const u8 key26[] __initconst = { + 0xe4, 0xd0, 0x54, 0x1d, 0x7d, 0x47, 0xa8, 0xc1, + 0x08, 0xca, 0xe2, 0x42, 0x52, 0x95, 0x16, 0x43, + 0xa3, 0x01, 0x23, 0x03, 0xcc, 0x3b, 0x81, 0x78, + 0x23, 0xcc, 0xa7, 0x36, 0xd7, 0xa0, 0x97, 0x8d +}; +enum { nonce26 = 0x524401012231683ULL }; + +static const u8 input27[] __initconst = { + 0x0d, 0xb0, 0xcf, 0xec, 0xfc, 0x38, 0x9d, 0x9d, + 0x89, 0x00, 0x96, 0xf2, 0x79, 0x8a, 0xa1, 0x8d, + 0x32, 0x5e, 0xc6, 0x12, 0x22, 0xec, 0xf6, 0x52, + 0xc1, 0x0b +}; +static const u8 output27[] __initconst = { + 0xef, 0xe1, 0xf2, 0x67, 0x8e, 0x2c, 0x00, 0x9f, + 0x1d, 0x4c, 0x66, 0x1f, 0x94, 0x58, 0xdc, 0xbb, + 0xb9, 0x11, 0x8f, 0x74, 0xfd, 0x0e, 0x14, 0x01, + 0xa8, 0x21 +}; +static const u8 key27[] __initconst = { + 0x78, 0x71, 0xa4, 0xe6, 0xb2, 0x95, 0x44, 0x12, + 0x81, 0xaa, 0x7e, 0x94, 0xa7, 0x8d, 0x44, 0xea, + 0xc4, 0xbc, 0x01, 0xb7, 0x9e, 0xf7, 0x82, 0x9e, + 0x3b, 0x23, 0x9f, 0x31, 0xdd, 0xb8, 0x0d, 0x18 +}; +enum { nonce27 = 0xd58fe0e58fb254d6ULL }; + +static const u8 input28[] __initconst = { + 0xaa, 0xb7, 0xaa, 0xd9, 0xa8, 0x91, 0xd7, 0x8a, + 0x97, 0x9b, 0xdb, 0x7c, 0x47, 0x2b, 0xdb, 0xd2, + 0xda, 0x77, 0xb1, 0xfa, 0x2d, 0x12, 0xe3, 0xe9, + 0xc4, 0x7f, 0x54 +}; +static const u8 output28[] __initconst = { + 0x87, 0x84, 0xa9, 0xa6, 0xad, 0x8f, 0xe6, 0x0f, + 0x69, 0xf8, 0x21, 0xc3, 0x54, 0x95, 0x0f, 0xb0, + 0x4e, 0xc7, 0x02, 0xe4, 0x04, 0xb0, 0x6c, 0x42, + 0x8c, 0x63, 0xe3 +}; +static const u8 key28[] __initconst = { + 0x12, 0x23, 0x37, 0x95, 0x04, 0xb4, 0x21, 0xe8, + 0xbc, 0x65, 0x46, 0x7a, 0xf4, 0x01, 0x05, 0x3f, + 0xb1, 0x34, 0x73, 0xd2, 0x49, 0xbf, 0x6f, 0x20, + 0xbd, 0x23, 0x58, 0x5f, 0xd1, 0x73, 0x57, 0xa6 +}; +enum { nonce28 = 0x3a04d51491eb4e07ULL }; + +static const u8 input29[] __initconst = { + 0x55, 0xd0, 0xd4, 0x4b, 0x17, 0xc8, 0xc4, 0x2b, + 0xc0, 0x28, 0xbd, 0x9d, 0x65, 0x4d, 0xaf, 0x77, + 0x72, 0x7c, 0x36, 0x68, 0xa7, 0xb6, 0x87, 0x4d, + 0xb9, 0x27, 0x25, 0x6c +}; +static const u8 output29[] __initconst = { + 0x0e, 0xac, 0x4c, 0xf5, 0x12, 0xb5, 0x56, 0xa5, + 0x00, 0x9a, 0xd6, 0xe5, 0x1a, 0x59, 0x2c, 0xf6, + 0x42, 0x22, 0xcf, 0x23, 0x98, 0x34, 0x29, 0xac, + 0x6e, 0xe3, 0x37, 0x6d +}; +static const u8 key29[] __initconst = { + 0xda, 0x9d, 0x05, 0x0c, 0x0c, 0xba, 0x75, 0xb9, + 0x9e, 0xb1, 0x8d, 0xd9, 0x73, 0x26, 0x2c, 0xa9, + 0x3a, 0xb5, 0xcb, 0x19, 0x49, 0xa7, 0x4f, 0xf7, + 0x64, 0x35, 0x23, 0x20, 0x2a, 0x45, 0x78, 0xc7 +}; +enum { nonce29 = 0xc25ac9982431cbfULL }; + +static const u8 input30[] __initconst = { + 0x4e, 0xd6, 0x85, 0xbb, 0xe7, 0x99, 0xfa, 0x04, + 0x33, 0x24, 0xfd, 0x75, 0x18, 0xe3, 0xd3, 0x25, + 0xcd, 0xca, 0xae, 0x00, 0xbe, 0x52, 0x56, 0x4a, + 0x31, 0xe9, 0x4f, 0xae, 0x8a +}; +static const u8 output30[] __initconst = { + 0x30, 0x36, 0x32, 0xa2, 0x3c, 0xb6, 0xf9, 0xf9, + 0x76, 0x70, 0xad, 0xa6, 0x10, 0x41, 0x00, 0x4a, + 0xfa, 0xce, 0x1b, 0x86, 0x05, 0xdb, 0x77, 0x96, + 0xb3, 0xb7, 0x8f, 0x61, 0x24 +}; +static const u8 key30[] __initconst = { + 0x49, 0x35, 0x4c, 0x15, 0x98, 0xfb, 0xc6, 0x57, + 0x62, 0x6d, 0x06, 0xc3, 0xd4, 0x79, 0x20, 0x96, + 0x05, 0x2a, 0x31, 0x63, 0xc0, 0x44, 0x42, 0x09, + 0x13, 0x13, 0xff, 0x1b, 0xc8, 0x63, 0x1f, 0x0b +}; +enum { nonce30 = 0x4967f9c08e41568bULL }; + +static const u8 input31[] __initconst = { + 0x91, 0x04, 0x20, 0x47, 0x59, 0xee, 0xa6, 0x0f, + 0x04, 0x75, 0xc8, 0x18, 0x95, 0x44, 0x01, 0x28, + 0x20, 0x6f, 0x73, 0x68, 0x66, 0xb5, 0x03, 0xb3, + 0x58, 0x27, 0x6e, 0x7a, 0x76, 0xb8 +}; +static const u8 output31[] __initconst = { + 0xe8, 0x03, 0x78, 0x9d, 0x13, 0x15, 0x98, 0xef, + 0x64, 0x68, 0x12, 0x41, 0xb0, 0x29, 0x94, 0x0c, + 0x83, 0x35, 0x46, 0xa9, 0x74, 0xe1, 0x75, 0xf0, + 0xb6, 0x96, 0xc3, 0x6f, 0xd7, 0x70 +}; +static const u8 key31[] __initconst = { + 0xef, 0xcd, 0x5a, 0x4a, 0xf4, 0x7e, 0x6a, 0x3a, + 0x11, 0x88, 0x72, 0x94, 0xb8, 0xae, 0x84, 0xc3, + 0x66, 0xe0, 0xde, 0x4b, 0x00, 0xa5, 0xd6, 0x2d, + 0x50, 0xb7, 0x28, 0xff, 0x76, 0x57, 0x18, 0x1f +}; +enum { nonce31 = 0xcb6f428fa4192e19ULL }; + +static const u8 input32[] __initconst = { + 0x90, 0x06, 0x50, 0x4b, 0x98, 0x14, 0x30, 0xf1, + 0xb8, 0xd7, 0xf0, 0xa4, 0x3e, 0x4e, 0xd8, 0x00, + 0xea, 0xdb, 0x4f, 0x93, 0x05, 0xef, 0x02, 0x71, + 0x1a, 0xcd, 0xa3, 0xb1, 0xae, 0xd3, 0x18 +}; +static const u8 output32[] __initconst = { + 0xcb, 0x4a, 0x37, 0x3f, 0xea, 0x40, 0xab, 0x86, + 0xfe, 0xcc, 0x07, 0xd5, 0xdc, 0xb2, 0x25, 0xb6, + 0xfd, 0x2a, 0x72, 0xbc, 0x5e, 0xd4, 0x75, 0xff, + 0x71, 0xfc, 0xce, 0x1e, 0x6f, 0x22, 0xc1 +}; +static const u8 key32[] __initconst = { + 0xfc, 0x6d, 0xc3, 0x80, 0xce, 0xa4, 0x31, 0xa1, + 0xcc, 0xfa, 0x9d, 0x10, 0x0b, 0xc9, 0x11, 0x77, + 0x34, 0xdb, 0xad, 0x1b, 0xc4, 0xfc, 0xeb, 0x79, + 0x91, 0xda, 0x59, 0x3b, 0x0d, 0xb1, 0x19, 0x3b +}; +enum { nonce32 = 0x88551bf050059467ULL }; + +static const u8 input33[] __initconst = { + 0x88, 0x94, 0x71, 0x92, 0xe8, 0xd7, 0xf9, 0xbd, + 0x55, 0xe3, 0x22, 0xdb, 0x99, 0x51, 0xfb, 0x50, + 0xbf, 0x82, 0xb5, 0x70, 0x8b, 0x2b, 0x6a, 0x03, + 0x37, 0xa0, 0xc6, 0x19, 0x5d, 0xc9, 0xbc, 0xcc +}; +static const u8 output33[] __initconst = { + 0xb6, 0x17, 0x51, 0xc8, 0xea, 0x8a, 0x14, 0xdc, + 0x23, 0x1b, 0xd4, 0xed, 0xbf, 0x50, 0xb9, 0x38, + 0x00, 0xc2, 0x3f, 0x78, 0x3d, 0xbf, 0xa0, 0x84, + 0xef, 0x45, 0xb2, 0x7d, 0x48, 0x7b, 0x62, 0xa7 +}; +static const u8 key33[] __initconst = { + 0xb9, 0x8f, 0x6a, 0xad, 0xb4, 0x6f, 0xb5, 0xdc, + 0x48, 0xfa, 0x43, 0x57, 0x62, 0x97, 0xef, 0x89, + 0x4c, 0x5a, 0x7b, 0x67, 0xb8, 0x9d, 0xf0, 0x42, + 0x2b, 0x8f, 0xf3, 0x18, 0x05, 0x2e, 0x48, 0xd0 +}; +enum { nonce33 = 0x31f16488fe8447f5ULL }; + +static const u8 input34[] __initconst = { + 0xda, 0x2b, 0x3d, 0x63, 0x9e, 0x4f, 0xc2, 0xb8, + 0x7f, 0xc2, 0x1a, 0x8b, 0x0d, 0x95, 0x65, 0x55, + 0x52, 0xba, 0x51, 0x51, 0xc0, 0x61, 0x9f, 0x0a, + 0x5d, 0xb0, 0x59, 0x8c, 0x64, 0x6a, 0xab, 0xf5, + 0x57 +}; +static const u8 output34[] __initconst = { + 0x5c, 0xf6, 0x62, 0x24, 0x8c, 0x45, 0xa3, 0x26, + 0xd0, 0xe4, 0x88, 0x1c, 0xed, 0xc4, 0x26, 0x58, + 0xb5, 0x5d, 0x92, 0xc4, 0x17, 0x44, 0x1c, 0xb8, + 0x2c, 0xf3, 0x55, 0x7e, 0xd6, 0xe5, 0xb3, 0x65, + 0xa8 +}; +static const u8 key34[] __initconst = { + 0xde, 0xd1, 0x27, 0xb7, 0x7c, 0xfa, 0xa6, 0x78, + 0x39, 0x80, 0xdf, 0xb7, 0x46, 0xac, 0x71, 0x26, + 0xd0, 0x2a, 0x56, 0x79, 0x12, 0xeb, 0x26, 0x37, + 0x01, 0x0d, 0x30, 0xe0, 0xe3, 0x66, 0xb2, 0xf4 +}; +enum { nonce34 = 0x92d0d9b252c24149ULL }; + +static const u8 input35[] __initconst = { + 0x3a, 0x15, 0x5b, 0x75, 0x6e, 0xd0, 0x52, 0x20, + 0x6c, 0x82, 0xfa, 0xce, 0x5b, 0xea, 0xf5, 0x43, + 0xc1, 0x81, 0x7c, 0xb2, 0xac, 0x16, 0x3f, 0xd3, + 0x5a, 0xaf, 0x55, 0x98, 0xf4, 0xc6, 0xba, 0x71, + 0x25, 0x8b +}; +static const u8 output35[] __initconst = { + 0xb3, 0xaf, 0xac, 0x6d, 0x4d, 0xc7, 0x68, 0x56, + 0x50, 0x5b, 0x69, 0x2a, 0xe5, 0x90, 0xf9, 0x5f, + 0x99, 0x88, 0xff, 0x0c, 0xa6, 0xb1, 0x83, 0xd6, + 0x80, 0xa6, 0x1b, 0xde, 0x94, 0xa4, 0x2c, 0xc3, + 0x74, 0xfa +}; +static const u8 key35[] __initconst = { + 0xd8, 0x24, 0xe2, 0x06, 0xd7, 0x7a, 0xce, 0x81, + 0x52, 0x72, 0x02, 0x69, 0x89, 0xc4, 0xe9, 0x53, + 0x3b, 0x08, 0x5f, 0x98, 0x1e, 0x1b, 0x99, 0x6e, + 0x28, 0x17, 0x6d, 0xba, 0xc0, 0x96, 0xf9, 0x3c +}; +enum { nonce35 = 0x7baf968c4c8e3a37ULL }; + +static const u8 input36[] __initconst = { + 0x31, 0x5d, 0x4f, 0xe3, 0xac, 0xad, 0x17, 0xa6, + 0xb5, 0x01, 0xe2, 0xc6, 0xd4, 0x7e, 0xc4, 0x80, + 0xc0, 0x59, 0x72, 0xbb, 0x4b, 0x74, 0x6a, 0x41, + 0x0f, 0x9c, 0xf6, 0xca, 0x20, 0xb3, 0x73, 0x07, + 0x6b, 0x02, 0x2a +}; +static const u8 output36[] __initconst = { + 0xf9, 0x09, 0x92, 0x94, 0x7e, 0x31, 0xf7, 0x53, + 0xe8, 0x8a, 0x5b, 0x20, 0xef, 0x9b, 0x45, 0x81, + 0xba, 0x5e, 0x45, 0x63, 0xc1, 0xc7, 0x9e, 0x06, + 0x0e, 0xd9, 0x62, 0x8e, 0x96, 0xf9, 0xfa, 0x43, + 0x4d, 0xd4, 0x28 +}; +static const u8 key36[] __initconst = { + 0x13, 0x30, 0x4c, 0x06, 0xae, 0x18, 0xde, 0x03, + 0x1d, 0x02, 0x40, 0xf5, 0xbb, 0x19, 0xe3, 0x88, + 0x41, 0xb1, 0x29, 0x15, 0x97, 0xc2, 0x69, 0x3f, + 0x32, 0x2a, 0x0c, 0x8b, 0xcf, 0x83, 0x8b, 0x6c +}; +enum { nonce36 = 0x226d251d475075a0ULL }; + +static const u8 input37[] __initconst = { + 0x10, 0x18, 0xbe, 0xfd, 0x66, 0xc9, 0x77, 0xcc, + 0x43, 0xe5, 0x46, 0x0b, 0x08, 0x8b, 0xae, 0x11, + 0x86, 0x15, 0xc2, 0xf6, 0x45, 0xd4, 0x5f, 0xd6, + 0xb6, 0x5f, 0x9f, 0x3e, 0x97, 0xb7, 0xd4, 0xad, + 0x0b, 0xe8, 0x31, 0x94 +}; +static const u8 output37[] __initconst = { + 0x03, 0x2c, 0x1c, 0xee, 0xc6, 0xdd, 0xed, 0x38, + 0x80, 0x6d, 0x84, 0x16, 0xc3, 0xc2, 0x04, 0x63, + 0xcd, 0xa7, 0x6e, 0x36, 0x8b, 0xed, 0x78, 0x63, + 0x95, 0xfc, 0x69, 0x7a, 0x3f, 0x8d, 0x75, 0x6b, + 0x6c, 0x26, 0x56, 0x4d +}; +static const u8 key37[] __initconst = { + 0xac, 0x84, 0x4d, 0xa9, 0x29, 0x49, 0x3c, 0x39, + 0x7f, 0xd9, 0xa6, 0x01, 0xf3, 0x7e, 0xfa, 0x4a, + 0x14, 0x80, 0x22, 0x74, 0xf0, 0x29, 0x30, 0x2d, + 0x07, 0x21, 0xda, 0xc0, 0x4d, 0x70, 0x56, 0xa2 +}; +enum { nonce37 = 0x167823ce3b64925aULL }; + +static const u8 input38[] __initconst = { + 0x30, 0x8f, 0xfa, 0x24, 0x29, 0xb1, 0xfb, 0xce, + 0x31, 0x62, 0xdc, 0xd0, 0x46, 0xab, 0xe1, 0x31, + 0xd9, 0xae, 0x60, 0x0d, 0xca, 0x0a, 0x49, 0x12, + 0x3d, 0x92, 0xe9, 0x91, 0x67, 0x12, 0x62, 0x18, + 0x89, 0xe2, 0xf9, 0x1c, 0xcc +}; +static const u8 output38[] __initconst = { + 0x56, 0x9c, 0xc8, 0x7a, 0xc5, 0x98, 0xa3, 0x0f, + 0xba, 0xd5, 0x3e, 0xe1, 0xc9, 0x33, 0x64, 0x33, + 0xf0, 0xd5, 0xf7, 0x43, 0x66, 0x0e, 0x08, 0x9a, + 0x6e, 0x09, 0xe4, 0x01, 0x0d, 0x1e, 0x2f, 0x4b, + 0xed, 0x9c, 0x08, 0x8c, 0x03 +}; +static const u8 key38[] __initconst = { + 0x77, 0x52, 0x2a, 0x23, 0xf1, 0xc5, 0x96, 0x2b, + 0x89, 0x4f, 0x3e, 0xf3, 0xff, 0x0e, 0x94, 0xce, + 0xf1, 0xbd, 0x53, 0xf5, 0x77, 0xd6, 0x9e, 0x47, + 0x49, 0x3d, 0x16, 0x64, 0xff, 0x95, 0x42, 0x42 +}; +enum { nonce38 = 0xff629d7b82cef357ULL }; + +static const u8 input39[] __initconst = { + 0x38, 0x26, 0x27, 0xd0, 0xc2, 0xf5, 0x34, 0xba, + 0xda, 0x0f, 0x1c, 0x1c, 0x9a, 0x70, 0xe5, 0x8a, + 0x78, 0x2d, 0x8f, 0x9a, 0xbf, 0x89, 0x6a, 0xfd, + 0xd4, 0x9c, 0x33, 0xf1, 0xb6, 0x89, 0x16, 0xe3, + 0x6a, 0x00, 0xfa, 0x3a, 0x0f, 0x26 +}; +static const u8 output39[] __initconst = { + 0x0f, 0xaf, 0x91, 0x6d, 0x9c, 0x99, 0xa4, 0xf7, + 0x3b, 0x9d, 0x9a, 0x98, 0xca, 0xbb, 0x50, 0x48, + 0xee, 0xcb, 0x5d, 0xa1, 0x37, 0x2d, 0x36, 0x09, + 0x2a, 0xe2, 0x1c, 0x3d, 0x98, 0x40, 0x1c, 0x16, + 0x56, 0xa7, 0x98, 0xe9, 0x7d, 0x2b +}; +static const u8 key39[] __initconst = { + 0x6e, 0x83, 0x15, 0x4d, 0xf8, 0x78, 0xa8, 0x0e, + 0x71, 0x37, 0xd4, 0x6e, 0x28, 0x5c, 0x06, 0xa1, + 0x2d, 0x6c, 0x72, 0x7a, 0xfd, 0xf8, 0x65, 0x1a, + 0xb8, 0xe6, 0x29, 0x7b, 0xe5, 0xb3, 0x23, 0x79 +}; +enum { nonce39 = 0xa4d8c491cf093e9dULL }; + +static const u8 input40[] __initconst = { + 0x8f, 0x32, 0x7c, 0x40, 0x37, 0x95, 0x08, 0x00, + 0x00, 0xfe, 0x2f, 0x95, 0x20, 0x12, 0x40, 0x18, + 0x5e, 0x7e, 0x5e, 0x99, 0xee, 0x8d, 0x91, 0x7d, + 0x50, 0x7d, 0x21, 0x45, 0x27, 0xe1, 0x7f, 0xd4, + 0x73, 0x10, 0xe1, 0x33, 0xbc, 0xf8, 0xdd +}; +static const u8 output40[] __initconst = { + 0x78, 0x7c, 0xdc, 0x55, 0x2b, 0xd9, 0x2b, 0x3a, + 0xdd, 0x56, 0x11, 0x52, 0xd3, 0x2e, 0xe0, 0x0d, + 0x23, 0x20, 0x8a, 0xf1, 0x4f, 0xee, 0xf1, 0x68, + 0xf6, 0xdc, 0x53, 0xcf, 0x17, 0xd4, 0xf0, 0x6c, + 0xdc, 0x80, 0x5f, 0x1c, 0xa4, 0x91, 0x05 +}; +static const u8 key40[] __initconst = { + 0x0d, 0x86, 0xbf, 0x8a, 0xba, 0x9e, 0x39, 0x91, + 0xa8, 0xe7, 0x22, 0xf0, 0x0c, 0x43, 0x18, 0xe4, + 0x1f, 0xb0, 0xaf, 0x8a, 0x34, 0x31, 0xf4, 0x41, + 0xf0, 0x89, 0x85, 0xca, 0x5d, 0x05, 0x3b, 0x94 +}; +enum { nonce40 = 0xae7acc4f5986439eULL }; + +static const u8 input41[] __initconst = { + 0x20, 0x5f, 0xc1, 0x83, 0x36, 0x02, 0x76, 0x96, + 0xf0, 0xbf, 0x8e, 0x0e, 0x1a, 0xd1, 0xc7, 0x88, + 0x18, 0xc7, 0x09, 0xc4, 0x15, 0xd9, 0x4f, 0x5e, + 0x1f, 0xb3, 0xb4, 0x6d, 0xcb, 0xa0, 0xd6, 0x8a, + 0x3b, 0x40, 0x8e, 0x80, 0xf1, 0xe8, 0x8f, 0x5f +}; +static const u8 output41[] __initconst = { + 0x0b, 0xd1, 0x49, 0x9a, 0x9d, 0xe8, 0x97, 0xb8, + 0xd1, 0xeb, 0x90, 0x62, 0x37, 0xd2, 0x99, 0x15, + 0x67, 0x6d, 0x27, 0x93, 0xce, 0x37, 0x65, 0xa2, + 0x94, 0x88, 0xd6, 0x17, 0xbc, 0x1c, 0x6e, 0xa2, + 0xcc, 0xfb, 0x81, 0x0e, 0x30, 0x60, 0x5a, 0x6f +}; +static const u8 key41[] __initconst = { + 0x36, 0x27, 0x57, 0x01, 0x21, 0x68, 0x97, 0xc7, + 0x00, 0x67, 0x7b, 0xe9, 0x0f, 0x55, 0x49, 0xbb, + 0x92, 0x18, 0x98, 0xf5, 0x5e, 0xbc, 0xe7, 0x5a, + 0x9d, 0x3d, 0xc7, 0xbd, 0x59, 0xec, 0x82, 0x8e +}; +enum { nonce41 = 0x5da05e4c8dfab464ULL }; + +static const u8 input42[] __initconst = { + 0xca, 0x30, 0xcd, 0x63, 0xf0, 0x2d, 0xf1, 0x03, + 0x4d, 0x0d, 0xf2, 0xf7, 0x6f, 0xae, 0xd6, 0x34, + 0xea, 0xf6, 0x13, 0xcf, 0x1c, 0xa0, 0xd0, 0xe8, + 0xa4, 0x78, 0x80, 0x3b, 0x1e, 0xa5, 0x32, 0x4c, + 0x73, 0x12, 0xd4, 0x6a, 0x94, 0xbc, 0xba, 0x80, + 0x5e +}; +static const u8 output42[] __initconst = { + 0xec, 0x3f, 0x18, 0x31, 0xc0, 0x7b, 0xb5, 0xe2, + 0xad, 0xf3, 0xec, 0xa0, 0x16, 0x9d, 0xef, 0xce, + 0x05, 0x65, 0x59, 0x9d, 0x5a, 0xca, 0x3e, 0x13, + 0xb9, 0x5d, 0x5d, 0xb5, 0xeb, 0xae, 0xc0, 0x87, + 0xbb, 0xfd, 0xe7, 0xe4, 0x89, 0x5b, 0xd2, 0x6c, + 0x56 +}; +static const u8 key42[] __initconst = { + 0x7c, 0x6b, 0x7e, 0x77, 0xcc, 0x8c, 0x1b, 0x03, + 0x8b, 0x2a, 0xb3, 0x7c, 0x5a, 0x73, 0xcc, 0xac, + 0xdd, 0x53, 0x54, 0x0c, 0x85, 0xed, 0xcd, 0x47, + 0x24, 0xc1, 0xb8, 0x9b, 0x2e, 0x41, 0x92, 0x36 +}; +enum { nonce42 = 0xe4d7348b09682c9cULL }; + +static const u8 input43[] __initconst = { + 0x52, 0xf2, 0x4b, 0x7c, 0xe5, 0x58, 0xe8, 0xd2, + 0xb7, 0xf3, 0xa1, 0x29, 0x68, 0xa2, 0x50, 0x50, + 0xae, 0x9c, 0x1b, 0xe2, 0x67, 0x77, 0xe2, 0xdb, + 0x85, 0x55, 0x7e, 0x84, 0x8a, 0x12, 0x3c, 0xb6, + 0x2e, 0xed, 0xd3, 0xec, 0x47, 0x68, 0xfa, 0x52, + 0x46, 0x9d +}; +static const u8 output43[] __initconst = { + 0x1b, 0xf0, 0x05, 0xe4, 0x1c, 0xd8, 0x74, 0x9a, + 0xf0, 0xee, 0x00, 0x54, 0xce, 0x02, 0x83, 0x15, + 0xfb, 0x23, 0x35, 0x78, 0xc3, 0xda, 0x98, 0xd8, + 0x9d, 0x1b, 0xb2, 0x51, 0x82, 0xb0, 0xff, 0xbe, + 0x05, 0xa9, 0xa4, 0x04, 0xba, 0xea, 0x4b, 0x73, + 0x47, 0x6e +}; +static const u8 key43[] __initconst = { + 0xeb, 0xec, 0x0e, 0xa1, 0x65, 0xe2, 0x99, 0x46, + 0xd8, 0x54, 0x8c, 0x4a, 0x93, 0xdf, 0x6d, 0xbf, + 0x93, 0x34, 0x94, 0x57, 0xc9, 0x12, 0x9d, 0x68, + 0x05, 0xc5, 0x05, 0xad, 0x5a, 0xc9, 0x2a, 0x3b +}; +enum { nonce43 = 0xe14f6a902b7827fULL }; + +static const u8 input44[] __initconst = { + 0x3e, 0x22, 0x3e, 0x8e, 0xcd, 0x18, 0xe2, 0xa3, + 0x8d, 0x8b, 0x38, 0xc3, 0x02, 0xa3, 0x31, 0x48, + 0xc6, 0x0e, 0xec, 0x99, 0x51, 0x11, 0x6d, 0x8b, + 0x32, 0x35, 0x3b, 0x08, 0x58, 0x76, 0x25, 0x30, + 0xe2, 0xfc, 0xa2, 0x46, 0x7d, 0x6e, 0x34, 0x87, + 0xac, 0x42, 0xbf +}; +static const u8 output44[] __initconst = { + 0x08, 0x92, 0x58, 0x02, 0x1a, 0xf4, 0x1f, 0x3d, + 0x38, 0x7b, 0x6b, 0xf6, 0x84, 0x07, 0xa3, 0x19, + 0x17, 0x2a, 0xed, 0x57, 0x1c, 0xf9, 0x55, 0x37, + 0x4e, 0xf4, 0x68, 0x68, 0x82, 0x02, 0x4f, 0xca, + 0x21, 0x00, 0xc6, 0x66, 0x79, 0x53, 0x19, 0xef, + 0x7f, 0xdd, 0x74 +}; +static const u8 key44[] __initconst = { + 0x73, 0xb6, 0x3e, 0xf4, 0x57, 0x52, 0xa6, 0x43, + 0x51, 0xd8, 0x25, 0x00, 0xdb, 0xb4, 0x52, 0x69, + 0xd6, 0x27, 0x49, 0xeb, 0x9b, 0xf1, 0x7b, 0xa0, + 0xd6, 0x7c, 0x9c, 0xd8, 0x95, 0x03, 0x69, 0x26 +}; +enum { nonce44 = 0xf5e6dc4f35ce24e5ULL }; + +static const u8 input45[] __initconst = { + 0x55, 0x76, 0xc0, 0xf1, 0x74, 0x03, 0x7a, 0x6d, + 0x14, 0xd8, 0x36, 0x2c, 0x9f, 0x9a, 0x59, 0x7a, + 0x2a, 0xf5, 0x77, 0x84, 0x70, 0x7c, 0x1d, 0x04, + 0x90, 0x45, 0xa4, 0xc1, 0x5e, 0xdd, 0x2e, 0x07, + 0x18, 0x34, 0xa6, 0x85, 0x56, 0x4f, 0x09, 0xaf, + 0x2f, 0x83, 0xe1, 0xc6 +}; +static const u8 output45[] __initconst = { + 0x22, 0x46, 0xe4, 0x0b, 0x3a, 0x55, 0xcc, 0x9b, + 0xf0, 0xc0, 0x53, 0xcd, 0x95, 0xc7, 0x57, 0x6c, + 0x77, 0x46, 0x41, 0x72, 0x07, 0xbf, 0xa8, 0xe5, + 0x68, 0x69, 0xd8, 0x1e, 0x45, 0xc1, 0xa2, 0x50, + 0xa5, 0xd1, 0x62, 0xc9, 0x5a, 0x7d, 0x08, 0x14, + 0xae, 0x44, 0x16, 0xb9 +}; +static const u8 key45[] __initconst = { + 0x41, 0xf3, 0x88, 0xb2, 0x51, 0x25, 0x47, 0x02, + 0x39, 0xe8, 0x15, 0x3a, 0x22, 0x78, 0x86, 0x0b, + 0xf9, 0x1e, 0x8d, 0x98, 0xb2, 0x22, 0x82, 0xac, + 0x42, 0x94, 0xde, 0x64, 0xf0, 0xfd, 0xb3, 0x6c +}; +enum { nonce45 = 0xf51a582daf4aa01aULL }; + +static const u8 input46[] __initconst = { + 0xf6, 0xff, 0x20, 0xf9, 0x26, 0x7e, 0x0f, 0xa8, + 0x6a, 0x45, 0x5a, 0x91, 0x73, 0xc4, 0x4c, 0x63, + 0xe5, 0x61, 0x59, 0xca, 0xec, 0xc0, 0x20, 0x35, + 0xbc, 0x9f, 0x58, 0x9c, 0x5e, 0xa1, 0x17, 0x46, + 0xcc, 0xab, 0x6e, 0xd0, 0x4f, 0x24, 0xeb, 0x05, + 0x4d, 0x40, 0x41, 0xe0, 0x9d +}; +static const u8 output46[] __initconst = { + 0x31, 0x6e, 0x63, 0x3f, 0x9c, 0xe6, 0xb1, 0xb7, + 0xef, 0x47, 0x46, 0xd7, 0xb1, 0x53, 0x42, 0x2f, + 0x2c, 0xc8, 0x01, 0xae, 0x8b, 0xec, 0x42, 0x2c, + 0x6b, 0x2c, 0x9c, 0xb2, 0xf0, 0x29, 0x06, 0xa5, + 0xcd, 0x7e, 0xc7, 0x3a, 0x38, 0x98, 0x8a, 0xde, + 0x03, 0x29, 0x14, 0x8f, 0xf9 +}; +static const u8 key46[] __initconst = { + 0xac, 0xa6, 0x44, 0x4a, 0x0d, 0x42, 0x10, 0xbc, + 0xd3, 0xc9, 0x8e, 0x9e, 0x71, 0xa3, 0x1c, 0x14, + 0x9d, 0x65, 0x0d, 0x49, 0x4d, 0x8c, 0xec, 0x46, + 0xe1, 0x41, 0xcd, 0xf5, 0xfc, 0x82, 0x75, 0x34 +}; +enum { nonce46 = 0x25f85182df84dec5ULL }; + +static const u8 input47[] __initconst = { + 0xa1, 0xd2, 0xf2, 0x52, 0x2f, 0x79, 0x50, 0xb2, + 0x42, 0x29, 0x5b, 0x44, 0x20, 0xf9, 0xbd, 0x85, + 0xb7, 0x65, 0x77, 0x86, 0xce, 0x3e, 0x1c, 0xe4, + 0x70, 0x80, 0xdd, 0x72, 0x07, 0x48, 0x0f, 0x84, + 0x0d, 0xfd, 0x97, 0xc0, 0xb7, 0x48, 0x9b, 0xb4, + 0xec, 0xff, 0x73, 0x14, 0x99, 0xe4 +}; +static const u8 output47[] __initconst = { + 0xe5, 0x3c, 0x78, 0x66, 0x31, 0x1e, 0xd6, 0xc4, + 0x9e, 0x71, 0xb3, 0xd7, 0xd5, 0xad, 0x84, 0xf2, + 0x78, 0x61, 0x77, 0xf8, 0x31, 0xf0, 0x13, 0xad, + 0x66, 0xf5, 0x31, 0x7d, 0xeb, 0xdf, 0xaf, 0xcb, + 0xac, 0x28, 0x6c, 0xc2, 0x9e, 0xe7, 0x78, 0xa2, + 0xa2, 0x58, 0xce, 0x84, 0x76, 0x70 +}; +static const u8 key47[] __initconst = { + 0x05, 0x7f, 0xc0, 0x7f, 0x37, 0x20, 0x71, 0x02, + 0x3a, 0xe7, 0x20, 0x5a, 0x0a, 0x8f, 0x79, 0x5a, + 0xfe, 0xbb, 0x43, 0x4d, 0x2f, 0xcb, 0xf6, 0x9e, + 0xa2, 0x97, 0x00, 0xad, 0x0d, 0x51, 0x7e, 0x17 +}; +enum { nonce47 = 0xae707c60f54de32bULL }; + +static const u8 input48[] __initconst = { + 0x80, 0x93, 0x77, 0x2e, 0x8d, 0xe8, 0xe6, 0xc1, + 0x27, 0xe6, 0xf2, 0x89, 0x5b, 0x33, 0x62, 0x18, + 0x80, 0x6e, 0x17, 0x22, 0x8e, 0x83, 0x31, 0x40, + 0x8f, 0xc9, 0x5c, 0x52, 0x6c, 0x0e, 0xa5, 0xe9, + 0x6c, 0x7f, 0xd4, 0x6a, 0x27, 0x56, 0x99, 0xce, + 0x8d, 0x37, 0x59, 0xaf, 0xc0, 0x0e, 0xe1 +}; +static const u8 output48[] __initconst = { + 0x02, 0xa4, 0x2e, 0x33, 0xb7, 0x7c, 0x2b, 0x9a, + 0x18, 0x5a, 0xba, 0x53, 0x38, 0xaf, 0x00, 0xeb, + 0xd8, 0x3d, 0x02, 0x77, 0x43, 0x45, 0x03, 0x91, + 0xe2, 0x5e, 0x4e, 0xeb, 0x50, 0xd5, 0x5b, 0xe0, + 0xf3, 0x33, 0xa7, 0xa2, 0xac, 0x07, 0x6f, 0xeb, + 0x3f, 0x6c, 0xcd, 0xf2, 0x6c, 0x61, 0x64 +}; +static const u8 key48[] __initconst = { + 0xf3, 0x79, 0xe7, 0xf8, 0x0e, 0x02, 0x05, 0x6b, + 0x83, 0x1a, 0xe7, 0x86, 0x6b, 0xe6, 0x8f, 0x3f, + 0xd3, 0xa3, 0xe4, 0x6e, 0x29, 0x06, 0xad, 0xbc, + 0xe8, 0x33, 0x56, 0x39, 0xdf, 0xb0, 0xe2, 0xfe +}; +enum { nonce48 = 0xd849b938c6569da0ULL }; + +static const u8 input49[] __initconst = { + 0x89, 0x3b, 0x88, 0x9e, 0x7b, 0x38, 0x16, 0x9f, + 0xa1, 0x28, 0xf6, 0xf5, 0x23, 0x74, 0x28, 0xb0, + 0xdf, 0x6c, 0x9e, 0x8a, 0x71, 0xaf, 0xed, 0x7a, + 0x39, 0x21, 0x57, 0x7d, 0x31, 0x6c, 0xee, 0x0d, + 0x11, 0x8d, 0x41, 0x9a, 0x5f, 0xb7, 0x27, 0x40, + 0x08, 0xad, 0xc6, 0xe0, 0x00, 0x43, 0x9e, 0xae +}; +static const u8 output49[] __initconst = { + 0x4d, 0xfd, 0xdb, 0x4c, 0x77, 0xc1, 0x05, 0x07, + 0x4d, 0x6d, 0x32, 0xcb, 0x2e, 0x0e, 0xff, 0x65, + 0xc9, 0x27, 0xeb, 0xa9, 0x46, 0x5b, 0xab, 0x06, + 0xe6, 0xb6, 0x5a, 0x1e, 0x00, 0xfb, 0xcf, 0xe4, + 0xb9, 0x71, 0x40, 0x10, 0xef, 0x12, 0x39, 0xf0, + 0xea, 0x40, 0xb8, 0x9a, 0xa2, 0x85, 0x38, 0x48 +}; +static const u8 key49[] __initconst = { + 0xe7, 0x10, 0x40, 0xd9, 0x66, 0xc0, 0xa8, 0x6d, + 0xa3, 0xcc, 0x8b, 0xdd, 0x93, 0xf2, 0x6e, 0xe0, + 0x90, 0x7f, 0xd0, 0xf4, 0x37, 0x0c, 0x8b, 0x9b, + 0x4c, 0x4d, 0xe6, 0xf2, 0x1f, 0xe9, 0x95, 0x24 +}; +enum { nonce49 = 0xf269817bdae01bc0ULL }; + +static const u8 input50[] __initconst = { + 0xda, 0x5b, 0x60, 0xcd, 0xed, 0x58, 0x8e, 0x7f, + 0xae, 0xdd, 0xc8, 0x2e, 0x16, 0x90, 0xea, 0x4b, + 0x0c, 0x74, 0x14, 0x35, 0xeb, 0xee, 0x2c, 0xff, + 0x46, 0x99, 0x97, 0x6e, 0xae, 0xa7, 0x8e, 0x6e, + 0x38, 0xfe, 0x63, 0xe7, 0x51, 0xd9, 0xaa, 0xce, + 0x7b, 0x1e, 0x7e, 0x5d, 0xc0, 0xe8, 0x10, 0x06, + 0x14 +}; +static const u8 output50[] __initconst = { + 0xe4, 0xe5, 0x86, 0x1b, 0x66, 0x19, 0xac, 0x49, + 0x1c, 0xbd, 0xee, 0x03, 0xaf, 0x11, 0xfc, 0x1f, + 0x6a, 0xd2, 0x50, 0x5c, 0xea, 0x2c, 0xa5, 0x75, + 0xfd, 0xb7, 0x0e, 0x80, 0x8f, 0xed, 0x3f, 0x31, + 0x47, 0xac, 0x67, 0x43, 0xb8, 0x2e, 0xb4, 0x81, + 0x6d, 0xe4, 0x1e, 0xb7, 0x8b, 0x0c, 0x53, 0xa9, + 0x26 +}; +static const u8 key50[] __initconst = { + 0xd7, 0xb2, 0x04, 0x76, 0x30, 0xcc, 0x38, 0x45, + 0xef, 0xdb, 0xc5, 0x86, 0x08, 0x61, 0xf0, 0xee, + 0x6d, 0xd8, 0x22, 0x04, 0x8c, 0xfb, 0xcb, 0x37, + 0xa6, 0xfb, 0x95, 0x22, 0xe1, 0x87, 0xb7, 0x6f +}; +enum { nonce50 = 0x3b44d09c45607d38ULL }; + +static const u8 input51[] __initconst = { + 0xa9, 0x41, 0x02, 0x4b, 0xd7, 0xd5, 0xd1, 0xf1, + 0x21, 0x55, 0xb2, 0x75, 0x6d, 0x77, 0x1b, 0x86, + 0xa9, 0xc8, 0x90, 0xfd, 0xed, 0x4a, 0x7b, 0x6c, + 0xb2, 0x5f, 0x9b, 0x5f, 0x16, 0xa1, 0x54, 0xdb, + 0xd6, 0x3f, 0x6a, 0x7f, 0x2e, 0x51, 0x9d, 0x49, + 0x5b, 0xa5, 0x0e, 0xf9, 0xfb, 0x2a, 0x38, 0xff, + 0x20, 0x8c +}; +static const u8 output51[] __initconst = { + 0x18, 0xf7, 0x88, 0xc1, 0x72, 0xfd, 0x90, 0x4b, + 0xa9, 0x2d, 0xdb, 0x47, 0xb0, 0xa5, 0xc4, 0x37, + 0x01, 0x95, 0xc4, 0xb1, 0xab, 0xc5, 0x5b, 0xcd, + 0xe1, 0x97, 0x78, 0x13, 0xde, 0x6a, 0xff, 0x36, + 0xce, 0xa4, 0x67, 0xc5, 0x4a, 0x45, 0x2b, 0xd9, + 0xff, 0x8f, 0x06, 0x7c, 0x63, 0xbb, 0x83, 0x17, + 0xb4, 0x6b +}; +static const u8 key51[] __initconst = { + 0x82, 0x1a, 0x79, 0xab, 0x9a, 0xb5, 0x49, 0x6a, + 0x30, 0x6b, 0x99, 0x19, 0x11, 0xc7, 0xa2, 0xf4, + 0xca, 0x55, 0xb9, 0xdd, 0xe7, 0x2f, 0xe7, 0xc1, + 0xdd, 0x27, 0xad, 0x80, 0xf2, 0x56, 0xad, 0xf3 +}; +enum { nonce51 = 0xe93aff94ca71a4a6ULL }; + +static const u8 input52[] __initconst = { + 0x89, 0xdd, 0xf3, 0xfa, 0xb6, 0xc1, 0xaa, 0x9a, + 0xc8, 0xad, 0x6b, 0x00, 0xa1, 0x65, 0xea, 0x14, + 0x55, 0x54, 0x31, 0x8f, 0xf0, 0x03, 0x84, 0x51, + 0x17, 0x1e, 0x0a, 0x93, 0x6e, 0x79, 0x96, 0xa3, + 0x2a, 0x85, 0x9c, 0x89, 0xf8, 0xd1, 0xe2, 0x15, + 0x95, 0x05, 0xf4, 0x43, 0x4d, 0x6b, 0xf0, 0x71, + 0x3b, 0x3e, 0xba +}; +static const u8 output52[] __initconst = { + 0x0c, 0x42, 0x6a, 0xb3, 0x66, 0x63, 0x5d, 0x2c, + 0x9f, 0x3d, 0xa6, 0x6e, 0xc7, 0x5f, 0x79, 0x2f, + 0x50, 0xe3, 0xd6, 0x07, 0x56, 0xa4, 0x2b, 0x2d, + 0x8d, 0x10, 0xc0, 0x6c, 0xa2, 0xfc, 0x97, 0xec, + 0x3f, 0x5c, 0x8d, 0x59, 0xbe, 0x84, 0xf1, 0x3e, + 0x38, 0x47, 0x4f, 0x75, 0x25, 0x66, 0x88, 0x14, + 0x03, 0xdd, 0xde +}; +static const u8 key52[] __initconst = { + 0x4f, 0xb0, 0x27, 0xb6, 0xdd, 0x24, 0x0c, 0xdb, + 0x6b, 0x71, 0x2e, 0xac, 0xfc, 0x3f, 0xa6, 0x48, + 0x5d, 0xd5, 0xff, 0x53, 0xb5, 0x62, 0xf1, 0xe0, + 0x93, 0xfe, 0x39, 0x4c, 0x9f, 0x03, 0x11, 0xa7 +}; +enum { nonce52 = 0xed8becec3bdf6f25ULL }; + +static const u8 input53[] __initconst = { + 0x68, 0xd1, 0xc7, 0x74, 0x44, 0x1c, 0x84, 0xde, + 0x27, 0x27, 0x35, 0xf0, 0x18, 0x0b, 0x57, 0xaa, + 0xd0, 0x1a, 0xd3, 0x3b, 0x5e, 0x5c, 0x62, 0x93, + 0xd7, 0x6b, 0x84, 0x3b, 0x71, 0x83, 0x77, 0x01, + 0x3e, 0x59, 0x45, 0xf4, 0x77, 0x6c, 0x6b, 0xcb, + 0x88, 0x45, 0x09, 0x1d, 0xc6, 0x45, 0x6e, 0xdc, + 0x6e, 0x51, 0xb8, 0x28 +}; +static const u8 output53[] __initconst = { + 0xc5, 0x90, 0x96, 0x78, 0x02, 0xf5, 0xc4, 0x3c, + 0xde, 0xd4, 0xd4, 0xc6, 0xa7, 0xad, 0x12, 0x47, + 0x45, 0xce, 0xcd, 0x8c, 0x35, 0xcc, 0xa6, 0x9e, + 0x5a, 0xc6, 0x60, 0xbb, 0xe3, 0xed, 0xec, 0x68, + 0x3f, 0x64, 0xf7, 0x06, 0x63, 0x9c, 0x8c, 0xc8, + 0x05, 0x3a, 0xad, 0x32, 0x79, 0x8b, 0x45, 0x96, + 0x93, 0x73, 0x4c, 0xe0 +}; +static const u8 key53[] __initconst = { + 0x42, 0x4b, 0x20, 0x81, 0x49, 0x50, 0xe9, 0xc2, + 0x43, 0x69, 0x36, 0xe7, 0x68, 0xae, 0xd5, 0x7e, + 0x42, 0x1a, 0x1b, 0xb4, 0x06, 0x4d, 0xa7, 0x17, + 0xb5, 0x31, 0xd6, 0x0c, 0xb0, 0x5c, 0x41, 0x0b +}; +enum { nonce53 = 0xf44ce1931fbda3d7ULL }; + +static const u8 input54[] __initconst = { + 0x7b, 0xf6, 0x8b, 0xae, 0xc0, 0xcb, 0x10, 0x8e, + 0xe8, 0xd8, 0x2e, 0x3b, 0x14, 0xba, 0xb4, 0xd2, + 0x58, 0x6b, 0x2c, 0xec, 0xc1, 0x81, 0x71, 0xb4, + 0xc6, 0xea, 0x08, 0xc5, 0xc9, 0x78, 0xdb, 0xa2, + 0xfa, 0x44, 0x50, 0x9b, 0xc8, 0x53, 0x8d, 0x45, + 0x42, 0xe7, 0x09, 0xc4, 0x29, 0xd8, 0x75, 0x02, + 0xbb, 0xb2, 0x78, 0xcf, 0xe7 +}; +static const u8 output54[] __initconst = { + 0xaf, 0x2c, 0x83, 0x26, 0x6e, 0x7f, 0xa6, 0xe9, + 0x03, 0x75, 0xfe, 0xfe, 0x87, 0x58, 0xcf, 0xb5, + 0xbc, 0x3c, 0x9d, 0xa1, 0x6e, 0x13, 0xf1, 0x0f, + 0x9e, 0xbc, 0xe0, 0x54, 0x24, 0x32, 0xce, 0x95, + 0xe6, 0xa5, 0x59, 0x3d, 0x24, 0x1d, 0x8f, 0xb1, + 0x74, 0x6c, 0x56, 0xe7, 0x96, 0xc1, 0x91, 0xc8, + 0x2d, 0x0e, 0xb7, 0x51, 0x10 +}; +static const u8 key54[] __initconst = { + 0x00, 0x68, 0x74, 0xdc, 0x30, 0x9e, 0xe3, 0x52, + 0xa9, 0xae, 0xb6, 0x7c, 0xa1, 0xdc, 0x12, 0x2d, + 0x98, 0x32, 0x7a, 0x77, 0xe1, 0xdd, 0xa3, 0x76, + 0x72, 0x34, 0x83, 0xd8, 0xb7, 0x69, 0xba, 0x77 +}; +enum { nonce54 = 0xbea57d79b798b63aULL }; + +static const u8 input55[] __initconst = { + 0xb5, 0xf4, 0x2f, 0xc1, 0x5e, 0x10, 0xa7, 0x4e, + 0x74, 0x3d, 0xa3, 0x96, 0xc0, 0x4d, 0x7b, 0x92, + 0x8f, 0xdb, 0x2d, 0x15, 0x52, 0x6a, 0x95, 0x5e, + 0x40, 0x81, 0x4f, 0x70, 0x73, 0xea, 0x84, 0x65, + 0x3d, 0x9a, 0x4e, 0x03, 0x95, 0xf8, 0x5d, 0x2f, + 0x07, 0x02, 0x13, 0x13, 0xdd, 0x82, 0xe6, 0x3b, + 0xe1, 0x5f, 0xb3, 0x37, 0x9b, 0x88 +}; +static const u8 output55[] __initconst = { + 0xc1, 0x88, 0xbd, 0x92, 0x77, 0xad, 0x7c, 0x5f, + 0xaf, 0xa8, 0x57, 0x0e, 0x40, 0x0a, 0xdc, 0x70, + 0xfb, 0xc6, 0x71, 0xfd, 0xc4, 0x74, 0x60, 0xcc, + 0xa0, 0x89, 0x8e, 0x99, 0xf0, 0x06, 0xa6, 0x7c, + 0x97, 0x42, 0x21, 0x81, 0x6a, 0x07, 0xe7, 0xb3, + 0xf7, 0xa5, 0x03, 0x71, 0x50, 0x05, 0x63, 0x17, + 0xa9, 0x46, 0x0b, 0xff, 0x30, 0x78 +}; +static const u8 key55[] __initconst = { + 0x19, 0x8f, 0xe7, 0xd7, 0x6b, 0x7f, 0x6f, 0x69, + 0x86, 0x91, 0x0f, 0xa7, 0x4a, 0x69, 0x8e, 0x34, + 0xf3, 0xdb, 0xde, 0xaf, 0xf2, 0x66, 0x1d, 0x64, + 0x97, 0x0c, 0xcf, 0xfa, 0x33, 0x84, 0xfd, 0x0c +}; +enum { nonce55 = 0x80aa3d3e2c51ef06ULL }; + +static const u8 input56[] __initconst = { + 0x6b, 0xe9, 0x73, 0x42, 0x27, 0x5e, 0x12, 0xcd, + 0xaa, 0x45, 0x12, 0x8b, 0xb3, 0xe6, 0x54, 0x33, + 0x31, 0x7d, 0xe2, 0x25, 0xc6, 0x86, 0x47, 0x67, + 0x86, 0x83, 0xe4, 0x46, 0xb5, 0x8f, 0x2c, 0xbb, + 0xe4, 0xb8, 0x9f, 0xa2, 0xa4, 0xe8, 0x75, 0x96, + 0x92, 0x51, 0x51, 0xac, 0x8e, 0x2e, 0x6f, 0xfc, + 0xbd, 0x0d, 0xa3, 0x9f, 0x16, 0x55, 0x3e +}; +static const u8 output56[] __initconst = { + 0x42, 0x99, 0x73, 0x6c, 0xd9, 0x4b, 0x16, 0xe5, + 0x18, 0x63, 0x1a, 0xd9, 0x0e, 0xf1, 0x15, 0x2e, + 0x0f, 0x4b, 0xe4, 0x5f, 0xa0, 0x4d, 0xde, 0x9f, + 0xa7, 0x18, 0xc1, 0x0c, 0x0b, 0xae, 0x55, 0xe4, + 0x89, 0x18, 0xa4, 0x78, 0x9d, 0x25, 0x0d, 0xd5, + 0x94, 0x0f, 0xf9, 0x78, 0xa3, 0xa6, 0xe9, 0x9e, + 0x2c, 0x73, 0xf0, 0xf7, 0x35, 0xf3, 0x2b +}; +static const u8 key56[] __initconst = { + 0x7d, 0x12, 0xad, 0x51, 0xd5, 0x6f, 0x8f, 0x96, + 0xc0, 0x5d, 0x9a, 0xd1, 0x7e, 0x20, 0x98, 0x0e, + 0x3c, 0x0a, 0x67, 0x6b, 0x1b, 0x88, 0x69, 0xd4, + 0x07, 0x8c, 0xaf, 0x0f, 0x3a, 0x28, 0xe4, 0x5d +}; +enum { nonce56 = 0x70f4c372fb8b5984ULL }; + +static const u8 input57[] __initconst = { + 0x28, 0xa3, 0x06, 0xe8, 0xe7, 0x08, 0xb9, 0xef, + 0x0d, 0x63, 0x15, 0x99, 0xb2, 0x78, 0x7e, 0xaf, + 0x30, 0x50, 0xcf, 0xea, 0xc9, 0x91, 0x41, 0x2f, + 0x3b, 0x38, 0x70, 0xc4, 0x87, 0xb0, 0x3a, 0xee, + 0x4a, 0xea, 0xe3, 0x83, 0x68, 0x8b, 0xcf, 0xda, + 0x04, 0xa5, 0xbd, 0xb2, 0xde, 0x3c, 0x55, 0x13, + 0xfe, 0x96, 0xad, 0xc1, 0x61, 0x1b, 0x98, 0xde +}; +static const u8 output57[] __initconst = { + 0xf4, 0x44, 0xe9, 0xd2, 0x6d, 0xc2, 0x5a, 0xe9, + 0xfd, 0x7e, 0x41, 0x54, 0x3f, 0xf4, 0x12, 0xd8, + 0x55, 0x0d, 0x12, 0x9b, 0xd5, 0x2e, 0x95, 0xe5, + 0x77, 0x42, 0x3f, 0x2c, 0xfb, 0x28, 0x9d, 0x72, + 0x6d, 0x89, 0x82, 0x27, 0x64, 0x6f, 0x0d, 0x57, + 0xa1, 0x25, 0xa3, 0x6b, 0x88, 0x9a, 0xac, 0x0c, + 0x76, 0x19, 0x90, 0xe2, 0x50, 0x5a, 0xf8, 0x12 +}; +static const u8 key57[] __initconst = { + 0x08, 0x26, 0xb8, 0xac, 0xf3, 0xa5, 0xc6, 0xa3, + 0x7f, 0x09, 0x87, 0xf5, 0x6c, 0x5a, 0x85, 0x6c, + 0x3d, 0xbd, 0xde, 0xd5, 0x87, 0xa3, 0x98, 0x7a, + 0xaa, 0x40, 0x3e, 0xf7, 0xff, 0x44, 0x5d, 0xee +}; +enum { nonce57 = 0xc03a6130bf06b089ULL }; + +static const u8 input58[] __initconst = { + 0x82, 0xa5, 0x38, 0x6f, 0xaa, 0xb4, 0xaf, 0xb2, + 0x42, 0x01, 0xa8, 0x39, 0x3f, 0x15, 0x51, 0xa8, + 0x11, 0x1b, 0x93, 0xca, 0x9c, 0xa0, 0x57, 0x68, + 0x8f, 0xdb, 0x68, 0x53, 0x51, 0x6d, 0x13, 0x22, + 0x12, 0x9b, 0xbd, 0x33, 0xa8, 0x52, 0x40, 0x57, + 0x80, 0x9b, 0x98, 0xef, 0x56, 0x70, 0x11, 0xfa, + 0x36, 0x69, 0x7d, 0x15, 0x48, 0xf9, 0x3b, 0xeb, + 0x42 +}; +static const u8 output58[] __initconst = { + 0xff, 0x3a, 0x74, 0xc3, 0x3e, 0x44, 0x64, 0x4d, + 0x0e, 0x5f, 0x9d, 0xa8, 0xdb, 0xbe, 0x12, 0xef, + 0xba, 0x56, 0x65, 0x50, 0x76, 0xaf, 0xa4, 0x4e, + 0x01, 0xc1, 0xd3, 0x31, 0x14, 0xe2, 0xbe, 0x7b, + 0xa5, 0x67, 0xb4, 0xe3, 0x68, 0x40, 0x9c, 0xb0, + 0xb1, 0x78, 0xef, 0x49, 0x03, 0x0f, 0x2d, 0x56, + 0xb4, 0x37, 0xdb, 0xbc, 0x2d, 0x68, 0x1c, 0x3c, + 0xf1 +}; +static const u8 key58[] __initconst = { + 0x7e, 0xf1, 0x7c, 0x20, 0x65, 0xed, 0xcd, 0xd7, + 0x57, 0xe8, 0xdb, 0x90, 0x87, 0xdb, 0x5f, 0x63, + 0x3d, 0xdd, 0xb8, 0x2b, 0x75, 0x8e, 0x04, 0xb5, + 0xf4, 0x12, 0x79, 0xa9, 0x4d, 0x42, 0x16, 0x7f +}; +enum { nonce58 = 0x92838183f80d2f7fULL }; + +static const u8 input59[] __initconst = { + 0x37, 0xf1, 0x9d, 0xdd, 0xd7, 0x08, 0x9f, 0x13, + 0xc5, 0x21, 0x82, 0x75, 0x08, 0x9e, 0x25, 0x16, + 0xb1, 0xd1, 0x71, 0x42, 0x28, 0x63, 0xac, 0x47, + 0x71, 0x54, 0xb1, 0xfc, 0x39, 0xf0, 0x61, 0x4f, + 0x7c, 0x6d, 0x4f, 0xc8, 0x33, 0xef, 0x7e, 0xc8, + 0xc0, 0x97, 0xfc, 0x1a, 0x61, 0xb4, 0x87, 0x6f, + 0xdd, 0x5a, 0x15, 0x7b, 0x1b, 0x95, 0x50, 0x94, + 0x1d, 0xba +}; +static const u8 output59[] __initconst = { + 0x73, 0x67, 0xc5, 0x07, 0xbb, 0x57, 0x79, 0xd5, + 0xc9, 0x04, 0xdd, 0x88, 0xf3, 0x86, 0xe5, 0x70, + 0x49, 0x31, 0xe0, 0xcc, 0x3b, 0x1d, 0xdf, 0xb0, + 0xaf, 0xf4, 0x2d, 0xe0, 0x06, 0x10, 0x91, 0x8d, + 0x1c, 0xcf, 0x31, 0x0b, 0xf6, 0x73, 0xda, 0x1c, + 0xf0, 0x17, 0x52, 0x9e, 0x20, 0x2e, 0x9f, 0x8c, + 0xb3, 0x59, 0xce, 0xd4, 0xd3, 0xc1, 0x81, 0xe9, + 0x11, 0x36 +}; +static const u8 key59[] __initconst = { + 0xbd, 0x07, 0xd0, 0x53, 0x2c, 0xb3, 0xcc, 0x3f, + 0xc4, 0x95, 0xfd, 0xe7, 0x81, 0xb3, 0x29, 0x99, + 0x05, 0x45, 0xd6, 0x95, 0x25, 0x0b, 0x72, 0xd3, + 0xcd, 0xbb, 0x73, 0xf8, 0xfa, 0xc0, 0x9b, 0x7a +}; +enum { nonce59 = 0x4a0db819b0d519e2ULL }; + +static const u8 input60[] __initconst = { + 0x58, 0x4e, 0xdf, 0x94, 0x3c, 0x76, 0x0a, 0x79, + 0x47, 0xf1, 0xbe, 0x88, 0xd3, 0xba, 0x94, 0xd8, + 0xe2, 0x8f, 0xe3, 0x2f, 0x2f, 0x74, 0x82, 0x55, + 0xc3, 0xda, 0xe2, 0x4e, 0x2c, 0x8c, 0x45, 0x1d, + 0x72, 0x8f, 0x54, 0x41, 0xb5, 0xb7, 0x69, 0xe4, + 0xdc, 0xd2, 0x36, 0x21, 0x5c, 0x28, 0x52, 0xf7, + 0x98, 0x8e, 0x72, 0xa7, 0x6d, 0x57, 0xed, 0xdc, + 0x3c, 0xe6, 0x6a +}; +static const u8 output60[] __initconst = { + 0xda, 0xaf, 0xb5, 0xe3, 0x30, 0x65, 0x5c, 0xb1, + 0x48, 0x08, 0x43, 0x7b, 0x9e, 0xd2, 0x6a, 0x62, + 0x56, 0x7c, 0xad, 0xd9, 0xe5, 0xf6, 0x09, 0x71, + 0xcd, 0xe6, 0x05, 0x6b, 0x3f, 0x44, 0x3a, 0x5c, + 0xf6, 0xf8, 0xd7, 0xce, 0x7d, 0xd1, 0xe0, 0x4f, + 0x88, 0x15, 0x04, 0xd8, 0x20, 0xf0, 0x3e, 0xef, + 0xae, 0xa6, 0x27, 0xa3, 0x0e, 0xfc, 0x18, 0x90, + 0x33, 0xcd, 0xd3 +}; +static const u8 key60[] __initconst = { + 0xbf, 0xfd, 0x25, 0xb5, 0xb2, 0xfc, 0x78, 0x0c, + 0x8e, 0xb9, 0x57, 0x2f, 0x26, 0x4a, 0x7e, 0x71, + 0xcc, 0xf2, 0xe0, 0xfd, 0x24, 0x11, 0x20, 0x23, + 0x57, 0x00, 0xff, 0x80, 0x11, 0x0c, 0x1e, 0xff +}; +enum { nonce60 = 0xf18df56fdb7954adULL }; + +static const u8 input61[] __initconst = { + 0xb0, 0xf3, 0x06, 0xbc, 0x22, 0xae, 0x49, 0x40, + 0xae, 0xff, 0x1b, 0x31, 0xa7, 0x98, 0xab, 0x1d, + 0xe7, 0x40, 0x23, 0x18, 0x4f, 0xab, 0x8e, 0x93, + 0x82, 0xf4, 0x56, 0x61, 0xfd, 0x2b, 0xcf, 0xa7, + 0xc4, 0xb4, 0x0a, 0xf4, 0xcb, 0xc7, 0x8c, 0x40, + 0x57, 0xac, 0x0b, 0x3e, 0x2a, 0x0a, 0x67, 0x83, + 0x50, 0xbf, 0xec, 0xb0, 0xc7, 0xf1, 0x32, 0x26, + 0x98, 0x80, 0x33, 0xb4 +}; +static const u8 output61[] __initconst = { + 0x9d, 0x23, 0x0e, 0xff, 0xcc, 0x7c, 0xd5, 0xcf, + 0x1a, 0xb8, 0x59, 0x1e, 0x92, 0xfd, 0x7f, 0xca, + 0xca, 0x3c, 0x18, 0x81, 0xde, 0xfa, 0x59, 0xc8, + 0x6f, 0x9c, 0x24, 0x3f, 0x3a, 0xe6, 0x0b, 0xb4, + 0x34, 0x48, 0x69, 0xfc, 0xb6, 0xea, 0xb2, 0xde, + 0x9f, 0xfd, 0x92, 0x36, 0x18, 0x98, 0x99, 0xaa, + 0x65, 0xe2, 0xea, 0xf4, 0xb1, 0x47, 0x8e, 0xb0, + 0xe7, 0xd4, 0x7a, 0x2c +}; +static const u8 key61[] __initconst = { + 0xd7, 0xfd, 0x9b, 0xbd, 0x8f, 0x65, 0x0d, 0x00, + 0xca, 0xa1, 0x6c, 0x85, 0x85, 0xa4, 0x6d, 0xf1, + 0xb1, 0x68, 0x0c, 0x8b, 0x5d, 0x37, 0x72, 0xd0, + 0xd8, 0xd2, 0x25, 0xab, 0x9f, 0x7b, 0x7d, 0x95 +}; +enum { nonce61 = 0xd82caf72a9c4864fULL }; + +static const u8 input62[] __initconst = { + 0x10, 0x77, 0xf3, 0x2f, 0xc2, 0x50, 0xd6, 0x0c, + 0xba, 0xa8, 0x8d, 0xce, 0x0d, 0x58, 0x9e, 0x87, + 0xb1, 0x59, 0x66, 0x0a, 0x4a, 0xb3, 0xd8, 0xca, + 0x0a, 0x6b, 0xf8, 0xc6, 0x2b, 0x3f, 0x8e, 0x09, + 0xe0, 0x0a, 0x15, 0x85, 0xfe, 0xaa, 0xc6, 0xbd, + 0x30, 0xef, 0xe4, 0x10, 0x78, 0x03, 0xc1, 0xc7, + 0x8a, 0xd9, 0xde, 0x0b, 0x51, 0x07, 0xc4, 0x7b, + 0xe2, 0x2e, 0x36, 0x3a, 0xc2 +}; +static const u8 output62[] __initconst = { + 0xa0, 0x0c, 0xfc, 0xc1, 0xf6, 0xaf, 0xc2, 0xb8, + 0x5c, 0xef, 0x6e, 0xf3, 0xce, 0x15, 0x48, 0x05, + 0xb5, 0x78, 0x49, 0x51, 0x1f, 0x9d, 0xf4, 0xbf, + 0x2f, 0x53, 0xa2, 0xd1, 0x15, 0x20, 0x82, 0x6b, + 0xd2, 0x22, 0x6c, 0x4e, 0x14, 0x87, 0xe3, 0xd7, + 0x49, 0x45, 0x84, 0xdb, 0x5f, 0x68, 0x60, 0xc4, + 0xb3, 0xe6, 0x3f, 0xd1, 0xfc, 0xa5, 0x73, 0xf3, + 0xfc, 0xbb, 0xbe, 0xc8, 0x9d +}; +static const u8 key62[] __initconst = { + 0x6e, 0xc9, 0xaf, 0xce, 0x35, 0xb9, 0x86, 0xd1, + 0xce, 0x5f, 0xd9, 0xbb, 0xd5, 0x1f, 0x7c, 0xcd, + 0xfe, 0x19, 0xaa, 0x3d, 0xea, 0x64, 0xc1, 0x28, + 0x40, 0xba, 0xa1, 0x28, 0xcd, 0x40, 0xb6, 0xf2 +}; +enum { nonce62 = 0xa1c0c265f900cde8ULL }; + +static const u8 input63[] __initconst = { + 0x7a, 0x70, 0x21, 0x2c, 0xef, 0xa6, 0x36, 0xd4, + 0xe0, 0xab, 0x8c, 0x25, 0x73, 0x34, 0xc8, 0x94, + 0x6c, 0x81, 0xcb, 0x19, 0x8d, 0x5a, 0x49, 0xaa, + 0x6f, 0xba, 0x83, 0x72, 0x02, 0x5e, 0xf5, 0x89, + 0xce, 0x79, 0x7e, 0x13, 0x3d, 0x5b, 0x98, 0x60, + 0x5d, 0xd9, 0xfb, 0x15, 0x93, 0x4c, 0xf3, 0x51, + 0x49, 0x55, 0xd1, 0x58, 0xdd, 0x7e, 0x6d, 0xfe, + 0xdd, 0x84, 0x23, 0x05, 0xba, 0xe9 +}; +static const u8 output63[] __initconst = { + 0x20, 0xb3, 0x5c, 0x03, 0x03, 0x78, 0x17, 0xfc, + 0x3b, 0x35, 0x30, 0x9a, 0x00, 0x18, 0xf5, 0xc5, + 0x06, 0x53, 0xf5, 0x04, 0x24, 0x9d, 0xd1, 0xb2, + 0xac, 0x5a, 0xb6, 0x2a, 0xa5, 0xda, 0x50, 0x00, + 0xec, 0xff, 0xa0, 0x7a, 0x14, 0x7b, 0xe4, 0x6b, + 0x63, 0xe8, 0x66, 0x86, 0x34, 0xfd, 0x74, 0x44, + 0xa2, 0x50, 0x97, 0x0d, 0xdc, 0xc3, 0x84, 0xf8, + 0x71, 0x02, 0x31, 0x95, 0xed, 0x54 +}; +static const u8 key63[] __initconst = { + 0x7d, 0x64, 0xb4, 0x12, 0x81, 0xe4, 0xe6, 0x8f, + 0xcc, 0xe7, 0xd1, 0x1f, 0x70, 0x20, 0xfd, 0xb8, + 0x3a, 0x7d, 0xa6, 0x53, 0x65, 0x30, 0x5d, 0xe3, + 0x1a, 0x44, 0xbe, 0x62, 0xed, 0x90, 0xc4, 0xd1 +}; +enum { nonce63 = 0xe8e849596c942276ULL }; + +static const u8 input64[] __initconst = { + 0x84, 0xf8, 0xda, 0x87, 0x23, 0x39, 0x60, 0xcf, + 0xc5, 0x50, 0x7e, 0xc5, 0x47, 0x29, 0x7c, 0x05, + 0xc2, 0xb4, 0xf4, 0xb2, 0xec, 0x5d, 0x48, 0x36, + 0xbf, 0xfc, 0x06, 0x8c, 0xf2, 0x0e, 0x88, 0xe7, + 0xc9, 0xc5, 0xa4, 0xa2, 0x83, 0x20, 0xa1, 0x6f, + 0x37, 0xe5, 0x2d, 0xa1, 0x72, 0xa1, 0x19, 0xef, + 0x05, 0x42, 0x08, 0xf2, 0x57, 0x47, 0x31, 0x1e, + 0x17, 0x76, 0x13, 0xd3, 0xcc, 0x75, 0x2c +}; +static const u8 output64[] __initconst = { + 0xcb, 0xec, 0x90, 0x88, 0xeb, 0x31, 0x69, 0x20, + 0xa6, 0xdc, 0xff, 0x76, 0x98, 0xb0, 0x24, 0x49, + 0x7b, 0x20, 0xd9, 0xd1, 0x1b, 0xe3, 0x61, 0xdc, + 0xcf, 0x51, 0xf6, 0x70, 0x72, 0x33, 0x28, 0x94, + 0xac, 0x73, 0x18, 0xcf, 0x93, 0xfd, 0xca, 0x08, + 0x0d, 0xa2, 0xb9, 0x57, 0x1e, 0x51, 0xb6, 0x07, + 0x5c, 0xc1, 0x13, 0x64, 0x1d, 0x18, 0x6f, 0xe6, + 0x0b, 0xb7, 0x14, 0x03, 0x43, 0xb6, 0xaf +}; +static const u8 key64[] __initconst = { + 0xbf, 0x82, 0x65, 0xe4, 0x50, 0xf9, 0x5e, 0xea, + 0x28, 0x91, 0xd1, 0xd2, 0x17, 0x7c, 0x13, 0x7e, + 0xf5, 0xd5, 0x6b, 0x06, 0x1c, 0x20, 0xc2, 0x82, + 0xa1, 0x7a, 0xa2, 0x14, 0xa1, 0xb0, 0x54, 0x58 +}; +enum { nonce64 = 0xe57c5095aa5723c9ULL }; + +static const u8 input65[] __initconst = { + 0x1c, 0xfb, 0xd3, 0x3f, 0x85, 0xd7, 0xba, 0x7b, + 0xae, 0xb1, 0xa5, 0xd2, 0xe5, 0x40, 0xce, 0x4d, + 0x3e, 0xab, 0x17, 0x9d, 0x7d, 0x9f, 0x03, 0x98, + 0x3f, 0x9f, 0xc8, 0xdd, 0x36, 0x17, 0x43, 0x5c, + 0x34, 0xd1, 0x23, 0xe0, 0x77, 0xbf, 0x35, 0x5d, + 0x8f, 0xb1, 0xcb, 0x82, 0xbb, 0x39, 0x69, 0xd8, + 0x90, 0x45, 0x37, 0xfd, 0x98, 0x25, 0xf7, 0x5b, + 0xce, 0x06, 0x43, 0xba, 0x61, 0xa8, 0x47, 0xb9 +}; +static const u8 output65[] __initconst = { + 0x73, 0xa5, 0x68, 0xab, 0x8b, 0xa5, 0xc3, 0x7e, + 0x74, 0xf8, 0x9d, 0xf5, 0x93, 0x6e, 0xf2, 0x71, + 0x6d, 0xde, 0x82, 0xc5, 0x40, 0xa0, 0x46, 0xb3, + 0x9a, 0x78, 0xa8, 0xf7, 0xdf, 0xb1, 0xc3, 0xdd, + 0x8d, 0x90, 0x00, 0x68, 0x21, 0x48, 0xe8, 0xba, + 0x56, 0x9f, 0x8f, 0xe7, 0xa4, 0x4d, 0x36, 0x55, + 0xd0, 0x34, 0x99, 0xa6, 0x1c, 0x4c, 0xc1, 0xe2, + 0x65, 0x98, 0x14, 0x8e, 0x6a, 0x05, 0xb1, 0x2b +}; +static const u8 key65[] __initconst = { + 0xbd, 0x5c, 0x8a, 0xb0, 0x11, 0x29, 0xf3, 0x00, + 0x7a, 0x78, 0x32, 0x63, 0x34, 0x00, 0xe6, 0x7d, + 0x30, 0x54, 0xde, 0x37, 0xda, 0xc2, 0xc4, 0x3d, + 0x92, 0x6b, 0x4c, 0xc2, 0x92, 0xe9, 0x9e, 0x2a +}; +enum { nonce65 = 0xf654a3031de746f2ULL }; + +static const u8 input66[] __initconst = { + 0x4b, 0x27, 0x30, 0x8f, 0x28, 0xd8, 0x60, 0x46, + 0x39, 0x06, 0x49, 0xea, 0x1b, 0x71, 0x26, 0xe0, + 0x99, 0x2b, 0xd4, 0x8f, 0x64, 0x64, 0xcd, 0xac, + 0x1d, 0x78, 0x88, 0x90, 0xe1, 0x5c, 0x24, 0x4b, + 0xdc, 0x2d, 0xb7, 0xee, 0x3a, 0xe6, 0x86, 0x2c, + 0x21, 0xe4, 0x2b, 0xfc, 0xe8, 0x19, 0xca, 0x65, + 0xe7, 0xdd, 0x6f, 0x52, 0xb3, 0x11, 0xe1, 0xe2, + 0xbf, 0xe8, 0x70, 0xe3, 0x0d, 0x45, 0xb8, 0xa5, + 0x20, 0xb7, 0xb5, 0xaf, 0xff, 0x08, 0xcf, 0x23, + 0x65, 0xdf, 0x8d, 0xc3, 0x31, 0xf3, 0x1e, 0x6a, + 0x58, 0x8d, 0xcc, 0x45, 0x16, 0x86, 0x1f, 0x31, + 0x5c, 0x27, 0xcd, 0xc8, 0x6b, 0x19, 0x1e, 0xec, + 0x44, 0x75, 0x63, 0x97, 0xfd, 0x79, 0xf6, 0x62, + 0xc5, 0xba, 0x17, 0xc7, 0xab, 0x8f, 0xbb, 0xed, + 0x85, 0x2a, 0x98, 0x79, 0x21, 0xec, 0x6e, 0x4d, + 0xdc, 0xfa, 0x72, 0x52, 0xba, 0xc8, 0x4c +}; +static const u8 output66[] __initconst = { + 0x76, 0x5b, 0x2c, 0xa7, 0x62, 0xb9, 0x08, 0x4a, + 0xc6, 0x4a, 0x92, 0xc3, 0xbb, 0x10, 0xb3, 0xee, + 0xff, 0xb9, 0x07, 0xc7, 0x27, 0xcb, 0x1e, 0xcf, + 0x58, 0x6f, 0xa1, 0x64, 0xe8, 0xf1, 0x4e, 0xe1, + 0xef, 0x18, 0x96, 0xab, 0x97, 0x28, 0xd1, 0x7c, + 0x71, 0x6c, 0xd1, 0xe2, 0xfa, 0xd9, 0x75, 0xcb, + 0xeb, 0xea, 0x0c, 0x86, 0x82, 0xd8, 0xf4, 0xcc, + 0xea, 0xa3, 0x00, 0xfa, 0x82, 0xd2, 0xcd, 0xcb, + 0xdb, 0x63, 0x28, 0xe2, 0x82, 0xe9, 0x01, 0xed, + 0x31, 0xe6, 0x71, 0x45, 0x08, 0x89, 0x8a, 0x23, + 0xa8, 0xb5, 0xc2, 0xe2, 0x9f, 0xe9, 0xb8, 0x9a, + 0xc4, 0x79, 0x6d, 0x71, 0x52, 0x61, 0x74, 0x6c, + 0x1b, 0xd7, 0x65, 0x6d, 0x03, 0xc4, 0x1a, 0xc0, + 0x50, 0xba, 0xd6, 0xc9, 0x43, 0x50, 0xbe, 0x09, + 0x09, 0x8a, 0xdb, 0xaa, 0x76, 0x4e, 0x3b, 0x61, + 0x3c, 0x7c, 0x44, 0xe7, 0xdb, 0x10, 0xa7 +}; +static const u8 key66[] __initconst = { + 0x88, 0xdf, 0xca, 0x68, 0xaf, 0x4f, 0xb3, 0xfd, + 0x6e, 0xa7, 0x95, 0x35, 0x8a, 0xe8, 0x37, 0xe8, + 0xc8, 0x55, 0xa2, 0x2a, 0x6d, 0x77, 0xf8, 0x93, + 0x7a, 0x41, 0xf3, 0x7b, 0x95, 0xdf, 0x89, 0xf5 +}; +enum { nonce66 = 0x1024b4fdd415cf82ULL }; + +static const u8 input67[] __initconst = { + 0xd4, 0x2e, 0xfa, 0x92, 0xe9, 0x29, 0x68, 0xb7, + 0x54, 0x2c, 0xf7, 0xa4, 0x2d, 0xb7, 0x50, 0xb5, + 0xc5, 0xb2, 0x9d, 0x17, 0x5e, 0x0a, 0xca, 0x37, + 0xbf, 0x60, 0xae, 0xd2, 0x98, 0xe9, 0xfa, 0x59, + 0x67, 0x62, 0xe6, 0x43, 0x0c, 0x77, 0x80, 0x82, + 0x33, 0x61, 0xa3, 0xff, 0xc1, 0xa0, 0x8f, 0x56, + 0xbc, 0xec, 0x65, 0x43, 0x88, 0xa5, 0xff, 0x51, + 0x64, 0x30, 0xee, 0x34, 0xb7, 0x5c, 0x28, 0x68, + 0xc3, 0x52, 0xd2, 0xac, 0x78, 0x2a, 0xa6, 0x10, + 0xb8, 0xb2, 0x4c, 0x80, 0x4f, 0x99, 0xb2, 0x36, + 0x94, 0x8f, 0x66, 0xcb, 0xa1, 0x91, 0xed, 0x06, + 0x42, 0x6d, 0xc1, 0xae, 0x55, 0x93, 0xdd, 0x93, + 0x9e, 0x88, 0x34, 0x7f, 0x98, 0xeb, 0xbe, 0x61, + 0xf9, 0xa9, 0x0f, 0xd9, 0xc4, 0x87, 0xd5, 0xef, + 0xcc, 0x71, 0x8c, 0x0e, 0xce, 0xad, 0x02, 0xcf, + 0xa2, 0x61, 0xdf, 0xb1, 0xfe, 0x3b, 0xdc, 0xc0, + 0x58, 0xb5, 0x71, 0xa1, 0x83, 0xc9, 0xb4, 0xaf, + 0x9d, 0x54, 0x12, 0xcd, 0xea, 0x06, 0xd6, 0x4e, + 0xe5, 0x27, 0x0c, 0xc3, 0xbb, 0xa8, 0x0a, 0x81, + 0x75, 0xc3, 0xc9, 0xd4, 0x35, 0x3e, 0x53, 0x9f, + 0xaa, 0x20, 0xc0, 0x68, 0x39, 0x2c, 0x96, 0x39, + 0x53, 0x81, 0xda, 0x07, 0x0f, 0x44, 0xa5, 0x47, + 0x0e, 0xb3, 0x87, 0x0d, 0x1b, 0xc1, 0xe5, 0x41, + 0x35, 0x12, 0x58, 0x96, 0x69, 0x8a, 0x1a, 0xa3, + 0x9d, 0x3d, 0xd4, 0xb1, 0x8e, 0x1f, 0x96, 0x87, + 0xda, 0xd3, 0x19, 0xe2, 0xb1, 0x3a, 0x19, 0x74, + 0xa0, 0x00, 0x9f, 0x4d, 0xbc, 0xcb, 0x0c, 0xe9, + 0xec, 0x10, 0xdf, 0x2a, 0x88, 0xdc, 0x30, 0x51, + 0x46, 0x56, 0x53, 0x98, 0x6a, 0x26, 0x14, 0x05, + 0x54, 0x81, 0x55, 0x0b, 0x3c, 0x85, 0xdd, 0x33, + 0x81, 0x11, 0x29, 0x82, 0x46, 0x35, 0xe1, 0xdb, + 0x59, 0x7b +}; +static const u8 output67[] __initconst = { + 0x64, 0x6c, 0xda, 0x7f, 0xd4, 0xa9, 0x2a, 0x5e, + 0x22, 0xae, 0x8d, 0x67, 0xdb, 0xee, 0xfd, 0xd0, + 0x44, 0x80, 0x17, 0xb2, 0xe3, 0x87, 0xad, 0x57, + 0x15, 0xcb, 0x88, 0x64, 0xc0, 0xf1, 0x49, 0x3d, + 0xfa, 0xbe, 0xa8, 0x9f, 0x12, 0xc3, 0x57, 0x56, + 0x70, 0xa5, 0xc5, 0x6b, 0xf1, 0xab, 0xd5, 0xde, + 0x77, 0x92, 0x6a, 0x56, 0x03, 0xf5, 0x21, 0x0d, + 0xb6, 0xc4, 0xcc, 0x62, 0x44, 0x3f, 0xb1, 0xc1, + 0x61, 0x41, 0x90, 0xb2, 0xd5, 0xb8, 0xf3, 0x57, + 0xfb, 0xc2, 0x6b, 0x25, 0x58, 0xc8, 0x45, 0x20, + 0x72, 0x29, 0x6f, 0x9d, 0xb5, 0x81, 0x4d, 0x2b, + 0xb2, 0x89, 0x9e, 0x91, 0x53, 0x97, 0x1c, 0xd9, + 0x3d, 0x79, 0xdc, 0x14, 0xae, 0x01, 0x73, 0x75, + 0xf0, 0xca, 0xd5, 0xab, 0x62, 0x5c, 0x7a, 0x7d, + 0x3f, 0xfe, 0x22, 0x7d, 0xee, 0xe2, 0xcb, 0x76, + 0x55, 0xec, 0x06, 0xdd, 0x41, 0x47, 0x18, 0x62, + 0x1d, 0x57, 0xd0, 0xd6, 0xb6, 0x0f, 0x4b, 0xfc, + 0x79, 0x19, 0xf4, 0xd6, 0x37, 0x86, 0x18, 0x1f, + 0x98, 0x0d, 0x9e, 0x15, 0x2d, 0xb6, 0x9a, 0x8a, + 0x8c, 0x80, 0x22, 0x2f, 0x82, 0xc4, 0xc7, 0x36, + 0xfa, 0xfa, 0x07, 0xbd, 0xc2, 0x2a, 0xe2, 0xea, + 0x93, 0xc8, 0xb2, 0x90, 0x33, 0xf2, 0xee, 0x4b, + 0x1b, 0xf4, 0x37, 0x92, 0x13, 0xbb, 0xe2, 0xce, + 0xe3, 0x03, 0xcf, 0x07, 0x94, 0xab, 0x9a, 0xc9, + 0xff, 0x83, 0x69, 0x3a, 0xda, 0x2c, 0xd0, 0x47, + 0x3d, 0x6c, 0x1a, 0x60, 0x68, 0x47, 0xb9, 0x36, + 0x52, 0xdd, 0x16, 0xef, 0x6c, 0xbf, 0x54, 0x11, + 0x72, 0x62, 0xce, 0x8c, 0x9d, 0x90, 0xa0, 0x25, + 0x06, 0x92, 0x3e, 0x12, 0x7e, 0x1a, 0x1d, 0xe5, + 0xa2, 0x71, 0xce, 0x1c, 0x4c, 0x6a, 0x7c, 0xdc, + 0x3d, 0xe3, 0x6e, 0x48, 0x9d, 0xb3, 0x64, 0x7d, + 0x78, 0x40 +}; +static const u8 key67[] __initconst = { + 0xa9, 0x20, 0x75, 0x89, 0x7e, 0x37, 0x85, 0x48, + 0xa3, 0xfb, 0x7b, 0xe8, 0x30, 0xa7, 0xe3, 0x6e, + 0xa6, 0xc1, 0x71, 0x17, 0xc1, 0x6c, 0x9b, 0xc2, + 0xde, 0xf0, 0xa7, 0x19, 0xec, 0xce, 0xc6, 0x53 +}; +enum { nonce67 = 0x4adc4d1f968c8a10ULL }; + +static const u8 input68[] __initconst = { + 0x99, 0xae, 0x72, 0xfb, 0x16, 0xe1, 0xf1, 0x59, + 0x43, 0x15, 0x4e, 0x33, 0xa0, 0x95, 0xe7, 0x6c, + 0x74, 0x24, 0x31, 0xca, 0x3b, 0x2e, 0xeb, 0xd7, + 0x11, 0xd8, 0xe0, 0x56, 0x92, 0x91, 0x61, 0x57, + 0xe2, 0x82, 0x9f, 0x8f, 0x37, 0xf5, 0x3d, 0x24, + 0x92, 0x9d, 0x87, 0x00, 0x8d, 0x89, 0xe0, 0x25, + 0x8b, 0xe4, 0x20, 0x5b, 0x8a, 0x26, 0x2c, 0x61, + 0x78, 0xb0, 0xa6, 0x3e, 0x82, 0x18, 0xcf, 0xdc, + 0x2d, 0x24, 0xdd, 0x81, 0x42, 0xc4, 0x95, 0xf0, + 0x48, 0x60, 0x71, 0xe3, 0xe3, 0xac, 0xec, 0xbe, + 0x98, 0x6b, 0x0c, 0xb5, 0x6a, 0xa9, 0xc8, 0x79, + 0x23, 0x2e, 0x38, 0x0b, 0x72, 0x88, 0x8c, 0xe7, + 0x71, 0x8b, 0x36, 0xe3, 0x58, 0x3d, 0x9c, 0xa0, + 0xa2, 0xea, 0xcf, 0x0c, 0x6a, 0x6c, 0x64, 0xdf, + 0x97, 0x21, 0x8f, 0x93, 0xfb, 0xba, 0xf3, 0x5a, + 0xd7, 0x8f, 0xa6, 0x37, 0x15, 0x50, 0x43, 0x02, + 0x46, 0x7f, 0x93, 0x46, 0x86, 0x31, 0xe2, 0xaa, + 0x24, 0xa8, 0x26, 0xae, 0xe6, 0xc0, 0x05, 0x73, + 0x0b, 0x4f, 0x7e, 0xed, 0x65, 0xeb, 0x56, 0x1e, + 0xb6, 0xb3, 0x0b, 0xc3, 0x0e, 0x31, 0x95, 0xa9, + 0x18, 0x4d, 0xaf, 0x38, 0xd7, 0xec, 0xc6, 0x44, + 0x72, 0x77, 0x4e, 0x25, 0x4b, 0x25, 0xdd, 0x1e, + 0x8c, 0xa2, 0xdf, 0xf6, 0x2a, 0x97, 0x1a, 0x88, + 0x2c, 0x8a, 0x5d, 0xfe, 0xe8, 0xfb, 0x35, 0xe8, + 0x0f, 0x2b, 0x7a, 0x18, 0x69, 0x43, 0x31, 0x1d, + 0x38, 0x6a, 0x62, 0x95, 0x0f, 0x20, 0x4b, 0xbb, + 0x97, 0x3c, 0xe0, 0x64, 0x2f, 0x52, 0xc9, 0x2d, + 0x4d, 0x9d, 0x54, 0x04, 0x3d, 0xc9, 0xea, 0xeb, + 0xd0, 0x86, 0x52, 0xff, 0x42, 0xe1, 0x0d, 0x7a, + 0xad, 0x88, 0xf9, 0x9b, 0x1e, 0x5e, 0x12, 0x27, + 0x95, 0x3e, 0x0c, 0x2c, 0x13, 0x00, 0x6f, 0x8e, + 0x93, 0x69, 0x0e, 0x01, 0x8c, 0xc1, 0xfd, 0xb3 +}; +static const u8 output68[] __initconst = { + 0x26, 0x3e, 0xf2, 0xb1, 0xf5, 0xef, 0x81, 0xa4, + 0xb7, 0x42, 0xd4, 0x26, 0x18, 0x4b, 0xdd, 0x6a, + 0x47, 0x15, 0xcb, 0x0e, 0x57, 0xdb, 0xa7, 0x29, + 0x7e, 0x7b, 0x3f, 0x47, 0x89, 0x57, 0xab, 0xea, + 0x14, 0x7b, 0xcf, 0x37, 0xdb, 0x1c, 0xe1, 0x11, + 0x77, 0xae, 0x2e, 0x4c, 0xd2, 0x08, 0x3f, 0xa6, + 0x62, 0x86, 0xa6, 0xb2, 0x07, 0xd5, 0x3f, 0x9b, + 0xdc, 0xc8, 0x50, 0x4b, 0x7b, 0xb9, 0x06, 0xe6, + 0xeb, 0xac, 0x98, 0x8c, 0x36, 0x0c, 0x1e, 0xb2, + 0xc8, 0xfb, 0x24, 0x60, 0x2c, 0x08, 0x17, 0x26, + 0x5b, 0xc8, 0xc2, 0xdf, 0x9c, 0x73, 0x67, 0x4a, + 0xdb, 0xcf, 0xd5, 0x2c, 0x2b, 0xca, 0x24, 0xcc, + 0xdb, 0xc9, 0xa8, 0xf2, 0x5d, 0x67, 0xdf, 0x5c, + 0x62, 0x0b, 0x58, 0xc0, 0x83, 0xde, 0x8b, 0xf6, + 0x15, 0x0a, 0xd6, 0x32, 0xd8, 0xf5, 0xf2, 0x5f, + 0x33, 0xce, 0x7e, 0xab, 0x76, 0xcd, 0x14, 0x91, + 0xd8, 0x41, 0x90, 0x93, 0xa1, 0xaf, 0xf3, 0x45, + 0x6c, 0x1b, 0x25, 0xbd, 0x48, 0x51, 0x6d, 0x15, + 0x47, 0xe6, 0x23, 0x50, 0x32, 0x69, 0x1e, 0xb5, + 0x94, 0xd3, 0x97, 0xba, 0xd7, 0x37, 0x4a, 0xba, + 0xb9, 0xcd, 0xfb, 0x96, 0x9a, 0x90, 0xe0, 0x37, + 0xf8, 0xdf, 0x91, 0x6c, 0x62, 0x13, 0x19, 0x21, + 0x4b, 0xa9, 0xf1, 0x12, 0x66, 0xe2, 0x74, 0xd7, + 0x81, 0xa0, 0x74, 0x8d, 0x7e, 0x7e, 0xc9, 0xb1, + 0x69, 0x8f, 0xed, 0xb3, 0xf6, 0x97, 0xcd, 0x72, + 0x78, 0x93, 0xd3, 0x54, 0x6b, 0x43, 0xac, 0x29, + 0xb4, 0xbc, 0x7d, 0xa4, 0x26, 0x4b, 0x7b, 0xab, + 0xd6, 0x67, 0x22, 0xff, 0x03, 0x92, 0xb6, 0xd4, + 0x96, 0x94, 0x5a, 0xe5, 0x02, 0x35, 0x77, 0xfa, + 0x3f, 0x54, 0x1d, 0xdd, 0x35, 0x39, 0xfe, 0x03, + 0xdd, 0x8e, 0x3c, 0x8c, 0xc2, 0x69, 0x2a, 0xb1, + 0xb7, 0xb3, 0xa1, 0x89, 0x84, 0xea, 0x16, 0xe2 +}; +static const u8 key68[] __initconst = { + 0xd2, 0x49, 0x7f, 0xd7, 0x49, 0x66, 0x0d, 0xb3, + 0x5a, 0x7e, 0x3c, 0xfc, 0x37, 0x83, 0x0e, 0xf7, + 0x96, 0xd8, 0xd6, 0x33, 0x79, 0x2b, 0x84, 0x53, + 0x06, 0xbc, 0x6c, 0x0a, 0x55, 0x84, 0xfe, 0xab +}; +enum { nonce68 = 0x6a6df7ff0a20de06ULL }; + +static const u8 input69[] __initconst = { + 0xf9, 0x18, 0x4c, 0xd2, 0x3f, 0xf7, 0x22, 0xd9, + 0x58, 0xb6, 0x3b, 0x38, 0x69, 0x79, 0xf4, 0x71, + 0x5f, 0x38, 0x52, 0x1f, 0x17, 0x6f, 0x6f, 0xd9, + 0x09, 0x2b, 0xfb, 0x67, 0xdc, 0xc9, 0xe8, 0x4a, + 0x70, 0x9f, 0x2e, 0x3c, 0x06, 0xe5, 0x12, 0x20, + 0x25, 0x29, 0xd0, 0xdc, 0x81, 0xc5, 0xc6, 0x0f, + 0xd2, 0xa8, 0x81, 0x15, 0x98, 0xb2, 0x71, 0x5a, + 0x9a, 0xe9, 0xfb, 0xaf, 0x0e, 0x5f, 0x8a, 0xf3, + 0x16, 0x4a, 0x47, 0xf2, 0x5c, 0xbf, 0xda, 0x52, + 0x9a, 0xa6, 0x36, 0xfd, 0xc6, 0xf7, 0x66, 0x00, + 0xcc, 0x6c, 0xd4, 0xb3, 0x07, 0x6d, 0xeb, 0xfe, + 0x92, 0x71, 0x25, 0xd0, 0xcf, 0x9c, 0xe8, 0x65, + 0x45, 0x10, 0xcf, 0x62, 0x74, 0x7d, 0xf2, 0x1b, + 0x57, 0xa0, 0xf1, 0x6b, 0xa4, 0xd5, 0xfa, 0x12, + 0x27, 0x5a, 0xf7, 0x99, 0xfc, 0xca, 0xf3, 0xb8, + 0x2c, 0x8b, 0xba, 0x28, 0x74, 0xde, 0x8f, 0x78, + 0xa2, 0x8c, 0xaf, 0x89, 0x4b, 0x05, 0xe2, 0xf3, + 0xf8, 0xd2, 0xef, 0xac, 0xa4, 0xc4, 0xe2, 0xe2, + 0x36, 0xbb, 0x5e, 0xae, 0xe6, 0x87, 0x3d, 0x88, + 0x9f, 0xb8, 0x11, 0xbb, 0xcf, 0x57, 0xce, 0xd0, + 0xba, 0x62, 0xf4, 0xf8, 0x9b, 0x95, 0x04, 0xc9, + 0xcf, 0x01, 0xe9, 0xf1, 0xc8, 0xc6, 0x22, 0xa4, + 0xf2, 0x8b, 0x2f, 0x24, 0x0a, 0xf5, 0x6e, 0xb7, + 0xd4, 0x2c, 0xb6, 0xf7, 0x5c, 0x97, 0x61, 0x0b, + 0xd9, 0xb5, 0x06, 0xcd, 0xed, 0x3e, 0x1f, 0xc5, + 0xb2, 0x6c, 0xa3, 0xea, 0xb8, 0xad, 0xa6, 0x42, + 0x88, 0x7a, 0x52, 0xd5, 0x64, 0xba, 0xb5, 0x20, + 0x10, 0xa0, 0x0f, 0x0d, 0xea, 0xef, 0x5a, 0x9b, + 0x27, 0xb8, 0xca, 0x20, 0x19, 0x6d, 0xa8, 0xc4, + 0x46, 0x04, 0xb3, 0xe8, 0xf8, 0x66, 0x1b, 0x0a, + 0xce, 0x76, 0x5d, 0x59, 0x58, 0x05, 0xee, 0x3e, + 0x3c, 0x86, 0x5b, 0x49, 0x1c, 0x72, 0x18, 0x01, + 0x62, 0x92, 0x0f, 0x3e, 0xd1, 0x57, 0x5e, 0x20, + 0x7b, 0xfb, 0x4d, 0x3c, 0xc5, 0x35, 0x43, 0x2f, + 0xb0, 0xc5, 0x7c, 0xe4, 0xa2, 0x84, 0x13, 0x77 +}; +static const u8 output69[] __initconst = { + 0xbb, 0x4a, 0x7f, 0x7c, 0xd5, 0x2f, 0x89, 0x06, + 0xec, 0x20, 0xf1, 0x9a, 0x11, 0x09, 0x14, 0x2e, + 0x17, 0x50, 0xf9, 0xd5, 0xf5, 0x48, 0x7c, 0x7a, + 0x55, 0xc0, 0x57, 0x03, 0xe3, 0xc4, 0xb2, 0xb7, + 0x18, 0x47, 0x95, 0xde, 0xaf, 0x80, 0x06, 0x3c, + 0x5a, 0xf2, 0xc3, 0x53, 0xe3, 0x29, 0x92, 0xf8, + 0xff, 0x64, 0x85, 0xb9, 0xf7, 0xd3, 0x80, 0xd2, + 0x0c, 0x5d, 0x7b, 0x57, 0x0c, 0x51, 0x79, 0x86, + 0xf3, 0x20, 0xd2, 0xb8, 0x6e, 0x0c, 0x5a, 0xce, + 0xeb, 0x88, 0x02, 0x8b, 0x82, 0x1b, 0x7f, 0xf5, + 0xde, 0x7f, 0x48, 0x48, 0xdf, 0xa0, 0x55, 0xc6, + 0x0c, 0x22, 0xa1, 0x80, 0x8d, 0x3b, 0xcb, 0x40, + 0x2d, 0x3d, 0x0b, 0xf2, 0xe0, 0x22, 0x13, 0x99, + 0xe1, 0xa7, 0x27, 0x68, 0x31, 0xe1, 0x24, 0x5d, + 0xd2, 0xee, 0x16, 0xc1, 0xd7, 0xa8, 0x14, 0x19, + 0x23, 0x72, 0x67, 0x27, 0xdc, 0x5e, 0xb9, 0xc7, + 0xd8, 0xe3, 0x55, 0x50, 0x40, 0x98, 0x7b, 0xe7, + 0x34, 0x1c, 0x3b, 0x18, 0x14, 0xd8, 0x62, 0xc1, + 0x93, 0x84, 0xf3, 0x5b, 0xdd, 0x9e, 0x1f, 0x3b, + 0x0b, 0xbc, 0x4e, 0x5b, 0x79, 0xa3, 0xca, 0x74, + 0x2a, 0x98, 0xe8, 0x04, 0x39, 0xef, 0xc6, 0x76, + 0x6d, 0xee, 0x9f, 0x67, 0x5b, 0x59, 0x3a, 0xe5, + 0xf2, 0x3b, 0xca, 0x89, 0xe8, 0x9b, 0x03, 0x3d, + 0x11, 0xd2, 0x4a, 0x70, 0xaf, 0x88, 0xb0, 0x94, + 0x96, 0x26, 0xab, 0x3c, 0xc1, 0xb8, 0xe4, 0xe7, + 0x14, 0x61, 0x64, 0x3a, 0x61, 0x08, 0x0f, 0xa9, + 0xce, 0x64, 0xb2, 0x40, 0xf8, 0x20, 0x3a, 0xa9, + 0x31, 0xbd, 0x7e, 0x16, 0xca, 0xf5, 0x62, 0x0f, + 0x91, 0x9f, 0x8e, 0x1d, 0xa4, 0x77, 0xf3, 0x87, + 0x61, 0xe8, 0x14, 0xde, 0x18, 0x68, 0x4e, 0x9d, + 0x73, 0xcd, 0x8a, 0xe4, 0x80, 0x84, 0x23, 0xaa, + 0x9d, 0x64, 0x1c, 0x80, 0x41, 0xca, 0x82, 0x40, + 0x94, 0x55, 0xe3, 0x28, 0xa1, 0x97, 0x71, 0xba, + 0xf2, 0x2c, 0x39, 0x62, 0x29, 0x56, 0xd0, 0xff, + 0xb2, 0x82, 0x20, 0x59, 0x1f, 0xc3, 0x64, 0x57 +}; +static const u8 key69[] __initconst = { + 0x19, 0x09, 0xe9, 0x7c, 0xd9, 0x02, 0x4a, 0x0c, + 0x52, 0x25, 0xad, 0x5c, 0x2e, 0x8d, 0x86, 0x10, + 0x85, 0x2b, 0xba, 0xa4, 0x44, 0x5b, 0x39, 0x3e, + 0x18, 0xaa, 0xce, 0x0e, 0xe2, 0x69, 0x3c, 0xcf +}; +enum { nonce69 = 0xdb925a1948f0f060ULL }; + +static const u8 input70[] __initconst = { + 0x10, 0xe7, 0x83, 0xcf, 0x42, 0x9f, 0xf2, 0x41, + 0xc7, 0xe4, 0xdb, 0xf9, 0xa3, 0x02, 0x1d, 0x8d, + 0x50, 0x81, 0x2c, 0x6b, 0x92, 0xe0, 0x4e, 0xea, + 0x26, 0x83, 0x2a, 0xd0, 0x31, 0xf1, 0x23, 0xf3, + 0x0e, 0x88, 0x14, 0x31, 0xf9, 0x01, 0x63, 0x59, + 0x21, 0xd1, 0x8b, 0xdd, 0x06, 0xd0, 0xc6, 0xab, + 0x91, 0x71, 0x82, 0x4d, 0xd4, 0x62, 0x37, 0x17, + 0xf9, 0x50, 0xf9, 0xb5, 0x74, 0xce, 0x39, 0x80, + 0x80, 0x78, 0xf8, 0xdc, 0x1c, 0xdb, 0x7c, 0x3d, + 0xd4, 0x86, 0x31, 0x00, 0x75, 0x7b, 0xd1, 0x42, + 0x9f, 0x1b, 0x97, 0x88, 0x0e, 0x14, 0x0e, 0x1e, + 0x7d, 0x7b, 0xc4, 0xd2, 0xf3, 0xc1, 0x6d, 0x17, + 0x5d, 0xc4, 0x75, 0x54, 0x0f, 0x38, 0x65, 0x89, + 0xd8, 0x7d, 0xab, 0xc9, 0xa7, 0x0a, 0x21, 0x0b, + 0x37, 0x12, 0x05, 0x07, 0xb5, 0x68, 0x32, 0x32, + 0xb9, 0xf8, 0x97, 0x17, 0x03, 0xed, 0x51, 0x8f, + 0x3d, 0x5a, 0xd0, 0x12, 0x01, 0x6e, 0x2e, 0x91, + 0x1c, 0xbe, 0x6b, 0xa3, 0xcc, 0x75, 0x62, 0x06, + 0x8e, 0x65, 0xbb, 0xe2, 0x29, 0x71, 0x4b, 0x89, + 0x6a, 0x9d, 0x85, 0x8c, 0x8c, 0xdf, 0x94, 0x95, + 0x23, 0x66, 0xf8, 0x92, 0xee, 0x56, 0xeb, 0xb3, + 0xeb, 0xd2, 0x4a, 0x3b, 0x77, 0x8a, 0x6e, 0xf6, + 0xca, 0xd2, 0x34, 0x00, 0xde, 0xbe, 0x1d, 0x7a, + 0x73, 0xef, 0x2b, 0x80, 0x56, 0x16, 0x29, 0xbf, + 0x6e, 0x33, 0xed, 0x0d, 0xe2, 0x02, 0x60, 0x74, + 0xe9, 0x0a, 0xbc, 0xd1, 0xc5, 0xe8, 0x53, 0x02, + 0x79, 0x0f, 0x25, 0x0c, 0xef, 0xab, 0xd3, 0xbc, + 0xb7, 0xfc, 0xf3, 0xb0, 0x34, 0xd1, 0x07, 0xd2, + 0x5a, 0x31, 0x1f, 0xec, 0x1f, 0x87, 0xed, 0xdd, + 0x6a, 0xc1, 0xe8, 0xb3, 0x25, 0x4c, 0xc6, 0x9b, + 0x91, 0x73, 0xec, 0x06, 0x73, 0x9e, 0x57, 0x65, + 0x32, 0x75, 0x11, 0x74, 0x6e, 0xa4, 0x7d, 0x0d, + 0x74, 0x9f, 0x51, 0x10, 0x10, 0x47, 0xc9, 0x71, + 0x6e, 0x97, 0xae, 0x44, 0x41, 0xef, 0x98, 0x78, + 0xf4, 0xc5, 0xbd, 0x5e, 0x00, 0xe5, 0xfd, 0xe2, + 0xbe, 0x8c, 0xc2, 0xae, 0xc2, 0xee, 0x59, 0xf6, + 0xcb, 0x20, 0x54, 0x84, 0xc3, 0x31, 0x7e, 0x67, + 0x71, 0xb6, 0x76, 0xbe, 0x81, 0x8f, 0x82, 0xad, + 0x01, 0x8f, 0xc4, 0x00, 0x04, 0x3d, 0x8d, 0x34, + 0xaa, 0xea, 0xc0, 0xea, 0x91, 0x42, 0xb6, 0xb8, + 0x43, 0xf3, 0x17, 0xb2, 0x73, 0x64, 0x82, 0x97, + 0xd5, 0xc9, 0x07, 0x77, 0xb1, 0x26, 0xe2, 0x00, + 0x6a, 0xae, 0x70, 0x0b, 0xbe, 0xe6, 0xb8, 0x42, + 0x81, 0x55, 0xf7, 0xb8, 0x96, 0x41, 0x9d, 0xd4, + 0x2c, 0x27, 0x00, 0xcc, 0x91, 0x28, 0x22, 0xa4, + 0x7b, 0x42, 0x51, 0x9e, 0xd6, 0xec, 0xf3, 0x6b, + 0x00, 0xff, 0x5c, 0xa2, 0xac, 0x47, 0x33, 0x2d, + 0xf8, 0x11, 0x65, 0x5f, 0x4d, 0x79, 0x8b, 0x4f, + 0xad, 0xf0, 0x9d, 0xcd, 0xb9, 0x7b, 0x08, 0xf7, + 0x32, 0x51, 0xfa, 0x39, 0xaa, 0x78, 0x05, 0xb1, + 0xf3, 0x5d, 0xe8, 0x7c, 0x8e, 0x4f, 0xa2, 0xe0, + 0x98, 0x0c, 0xb2, 0xa7, 0xf0, 0x35, 0x8e, 0x70, + 0x7c, 0x82, 0xf3, 0x1b, 0x26, 0x28, 0x12, 0xe5, + 0x23, 0x57, 0xe4, 0xb4, 0x9b, 0x00, 0x39, 0x97, + 0xef, 0x7c, 0x46, 0x9b, 0x34, 0x6b, 0xe7, 0x0e, + 0xa3, 0x2a, 0x18, 0x11, 0x64, 0xc6, 0x7c, 0x8b, + 0x06, 0x02, 0xf5, 0x69, 0x76, 0xf9, 0xaa, 0x09, + 0x5f, 0x68, 0xf8, 0x4a, 0x79, 0x58, 0xec, 0x37, + 0xcf, 0x3a, 0xcc, 0x97, 0x70, 0x1d, 0x3e, 0x52, + 0x18, 0x0a, 0xad, 0x28, 0x5b, 0x3b, 0xe9, 0x03, + 0x84, 0xe9, 0x68, 0x50, 0xce, 0xc4, 0xbc, 0x3e, + 0x21, 0xad, 0x63, 0xfe, 0xc6, 0xfd, 0x6e, 0x69, + 0x84, 0xa9, 0x30, 0xb1, 0x7a, 0xc4, 0x31, 0x10, + 0xc1, 0x1f, 0x6e, 0xeb, 0xa5, 0xa6, 0x01 +}; +static const u8 output70[] __initconst = { + 0x0f, 0x93, 0x2a, 0x20, 0xb3, 0x87, 0x2d, 0xce, + 0xd1, 0x3b, 0x30, 0xfd, 0x06, 0x6d, 0x0a, 0xaa, + 0x3e, 0xc4, 0x29, 0x02, 0x8a, 0xde, 0xa6, 0x4b, + 0x45, 0x1b, 0x4f, 0x25, 0x59, 0xd5, 0x56, 0x6a, + 0x3b, 0x37, 0xbd, 0x3e, 0x47, 0x12, 0x2c, 0x4e, + 0x60, 0x5f, 0x05, 0x75, 0x61, 0x23, 0x05, 0x74, + 0xcb, 0xfc, 0x5a, 0xb3, 0xac, 0x5c, 0x3d, 0xab, + 0x52, 0x5f, 0x05, 0xbc, 0x57, 0xc0, 0x7e, 0xcf, + 0x34, 0x5d, 0x7f, 0x41, 0xa3, 0x17, 0x78, 0xd5, + 0x9f, 0xec, 0x0f, 0x1e, 0xf9, 0xfe, 0xa3, 0xbd, + 0x28, 0xb0, 0xba, 0x4d, 0x84, 0xdb, 0xae, 0x8f, + 0x1d, 0x98, 0xb7, 0xdc, 0xf9, 0xad, 0x55, 0x9c, + 0x89, 0xfe, 0x9b, 0x9c, 0xa9, 0x89, 0xf6, 0x97, + 0x9c, 0x3f, 0x09, 0x3e, 0xc6, 0x02, 0xc2, 0x55, + 0x58, 0x09, 0x54, 0x66, 0xe4, 0x36, 0x81, 0x35, + 0xca, 0x88, 0x17, 0x89, 0x80, 0x24, 0x2b, 0x21, + 0x89, 0xee, 0x45, 0x5a, 0xe7, 0x1f, 0xd5, 0xa5, + 0x16, 0xa4, 0xda, 0x70, 0x7e, 0xe9, 0x4f, 0x24, + 0x61, 0x97, 0xab, 0xa0, 0xe0, 0xe7, 0xb8, 0x5c, + 0x0f, 0x25, 0x17, 0x37, 0x75, 0x12, 0xb5, 0x40, + 0xde, 0x1c, 0x0d, 0x8a, 0x77, 0x62, 0x3c, 0x86, + 0xd9, 0x70, 0x2e, 0x96, 0x30, 0xd2, 0x55, 0xb3, + 0x6b, 0xc3, 0xf2, 0x9c, 0x47, 0xf3, 0x3a, 0x24, + 0x52, 0xc6, 0x38, 0xd8, 0x22, 0xb3, 0x0c, 0xfd, + 0x2f, 0xa3, 0x3c, 0xb5, 0xe8, 0x26, 0xe1, 0xa3, + 0xad, 0xb0, 0x82, 0x17, 0xc1, 0x53, 0xb8, 0x34, + 0x48, 0xee, 0x39, 0xae, 0x51, 0x43, 0xec, 0x82, + 0xce, 0x87, 0xc6, 0x76, 0xb9, 0x76, 0xd3, 0x53, + 0xfe, 0x49, 0x24, 0x7d, 0x02, 0x42, 0x2b, 0x72, + 0xfb, 0xcb, 0xd8, 0x96, 0x02, 0xc6, 0x9a, 0x20, + 0xf3, 0x5a, 0x67, 0xe8, 0x13, 0xf8, 0xb2, 0xcb, + 0xa2, 0xec, 0x18, 0x20, 0x4a, 0xb0, 0x73, 0x53, + 0x21, 0xb0, 0x77, 0x53, 0xd8, 0x76, 0xa1, 0x30, + 0x17, 0x72, 0x2e, 0x33, 0x5f, 0x33, 0x6b, 0x28, + 0xfb, 0xb0, 0xf4, 0xec, 0x8e, 0xed, 0x20, 0x7d, + 0x57, 0x8c, 0x74, 0x28, 0x64, 0x8b, 0xeb, 0x59, + 0x38, 0x3f, 0xe7, 0x83, 0x2e, 0xe5, 0x64, 0x4d, + 0x5c, 0x1f, 0xe1, 0x3b, 0xd9, 0x84, 0xdb, 0xc9, + 0xec, 0xd8, 0xc1, 0x7c, 0x1f, 0x1b, 0x68, 0x35, + 0xc6, 0x34, 0x10, 0xef, 0x19, 0xc9, 0x0a, 0xd6, + 0x43, 0x7f, 0xa6, 0xcb, 0x9d, 0xf4, 0xf0, 0x16, + 0xb1, 0xb1, 0x96, 0x64, 0xec, 0x8d, 0x22, 0x4c, + 0x4b, 0xe8, 0x1a, 0xba, 0x6f, 0xb7, 0xfc, 0xa5, + 0x69, 0x3e, 0xad, 0x78, 0x79, 0x19, 0xb5, 0x04, + 0x69, 0xe5, 0x3f, 0xff, 0x60, 0x8c, 0xda, 0x0b, + 0x7b, 0xf7, 0xe7, 0xe6, 0x29, 0x3a, 0x85, 0xba, + 0xb5, 0xb0, 0x35, 0xbd, 0x38, 0xce, 0x34, 0x5e, + 0xf2, 0xdc, 0xd1, 0x8f, 0xc3, 0x03, 0x24, 0xa2, + 0x03, 0xf7, 0x4e, 0x49, 0x5b, 0xcf, 0x6d, 0xb0, + 0xeb, 0xe3, 0x30, 0x28, 0xd5, 0x5b, 0x82, 0x5f, + 0xe4, 0x7c, 0x1e, 0xec, 0xd2, 0x39, 0xf9, 0x6f, + 0x2e, 0xb3, 0xcd, 0x01, 0xb1, 0x67, 0xaa, 0xea, + 0xaa, 0xb3, 0x63, 0xaf, 0xd9, 0xb2, 0x1f, 0xba, + 0x05, 0x20, 0xeb, 0x19, 0x32, 0xf0, 0x6c, 0x3f, + 0x40, 0xcc, 0x93, 0xb3, 0xd8, 0x25, 0xa6, 0xe4, + 0xce, 0xd7, 0x7e, 0x48, 0x99, 0x65, 0x7f, 0x86, + 0xc5, 0xd4, 0x79, 0x6b, 0xab, 0x43, 0xb8, 0x6b, + 0xf1, 0x2f, 0xea, 0x4c, 0x5e, 0xf0, 0x3b, 0xb4, + 0xb8, 0xb0, 0x94, 0x0c, 0x6b, 0xe7, 0x22, 0x93, + 0xaa, 0x01, 0xcb, 0xf1, 0x11, 0x60, 0xf6, 0x69, + 0xcf, 0x14, 0xde, 0xfb, 0x90, 0x05, 0x27, 0x0c, + 0x1a, 0x9e, 0xf0, 0xb4, 0xc6, 0xa1, 0xe8, 0xdd, + 0xd0, 0x4c, 0x25, 0x4f, 0x9c, 0xb7, 0xb1, 0xb0, + 0x21, 0xdb, 0x87, 0x09, 0x03, 0xf2, 0xb3 +}; +static const u8 key70[] __initconst = { + 0x3b, 0x5b, 0x59, 0x36, 0x44, 0xd1, 0xba, 0x71, + 0x55, 0x87, 0x4d, 0x62, 0x3d, 0xc2, 0xfc, 0xaa, + 0x3f, 0x4e, 0x1a, 0xe4, 0xca, 0x09, 0xfc, 0x6a, + 0xb2, 0xd6, 0x5d, 0x79, 0xf9, 0x1a, 0x91, 0xa7 +}; +enum { nonce70 = 0x3fd6786dd147a85ULL }; + +static const u8 input71[] __initconst = { + 0x18, 0x78, 0xd6, 0x79, 0xe4, 0x9a, 0x6c, 0x73, + 0x17, 0xd4, 0x05, 0x0f, 0x1e, 0x9f, 0xd9, 0x2b, + 0x86, 0x48, 0x7d, 0xf4, 0xd9, 0x1c, 0x76, 0xfc, + 0x8e, 0x22, 0x34, 0xe1, 0x48, 0x4a, 0x8d, 0x79, + 0xb7, 0xbb, 0x88, 0xab, 0x90, 0xde, 0xc5, 0xb4, + 0xb4, 0xe7, 0x85, 0x49, 0xda, 0x57, 0xeb, 0xc9, + 0xcd, 0x21, 0xfc, 0x45, 0x6e, 0x32, 0x67, 0xf2, + 0x4f, 0xa6, 0x54, 0xe5, 0x20, 0xed, 0xcf, 0xc6, + 0x62, 0x25, 0x8e, 0x00, 0xf8, 0x6b, 0xa2, 0x80, + 0xac, 0x88, 0xa6, 0x59, 0x27, 0x83, 0x95, 0x11, + 0x3f, 0x70, 0x5e, 0x3f, 0x11, 0xfb, 0x26, 0xbf, + 0xe1, 0x48, 0x75, 0xf9, 0x86, 0xbf, 0xa6, 0x5d, + 0x15, 0x61, 0x66, 0xbf, 0x78, 0x8f, 0x6b, 0x9b, + 0xda, 0x98, 0xb7, 0x19, 0xe2, 0xf2, 0xa3, 0x9c, + 0x7c, 0x6a, 0x9a, 0xd8, 0x3d, 0x4c, 0x2c, 0xe1, + 0x09, 0xb4, 0x28, 0x82, 0x4e, 0xab, 0x0c, 0x75, + 0x63, 0xeb, 0xbc, 0xd0, 0x71, 0xa2, 0x73, 0x85, + 0xed, 0x53, 0x7a, 0x3f, 0x68, 0x9f, 0xd0, 0xa9, + 0x00, 0x5a, 0x9e, 0x80, 0x55, 0x00, 0xe6, 0xae, + 0x0c, 0x03, 0x40, 0xed, 0xfc, 0x68, 0x4a, 0xb7, + 0x1e, 0x09, 0x65, 0x30, 0x5a, 0x3d, 0x97, 0x4d, + 0x5e, 0x51, 0x8e, 0xda, 0xc3, 0x55, 0x8c, 0xfb, + 0xcf, 0x83, 0x05, 0x35, 0x0d, 0x08, 0x1b, 0xf3, + 0x3a, 0x57, 0x96, 0xac, 0x58, 0x8b, 0xfa, 0x00, + 0x49, 0x15, 0x78, 0xd2, 0x4b, 0xed, 0xb8, 0x59, + 0x78, 0x9b, 0x7f, 0xaa, 0xfc, 0xe7, 0x46, 0xdc, + 0x7b, 0x34, 0xd0, 0x34, 0xe5, 0x10, 0xff, 0x4d, + 0x5a, 0x4d, 0x60, 0xa7, 0x16, 0x54, 0xc4, 0xfd, + 0xca, 0x5d, 0x68, 0xc7, 0x4a, 0x01, 0x8d, 0x7f, + 0x74, 0x5d, 0xff, 0xb8, 0x37, 0x15, 0x62, 0xfa, + 0x44, 0x45, 0xcf, 0x77, 0x3b, 0x1d, 0xb2, 0xd2, + 0x0d, 0x42, 0x00, 0x39, 0x68, 0x1f, 0xcc, 0x89, + 0x73, 0x5d, 0xa9, 0x2e, 0xfd, 0x58, 0x62, 0xca, + 0x35, 0x8e, 0x70, 0x70, 0xaa, 0x6e, 0x14, 0xe9, + 0xa4, 0xe2, 0x10, 0x66, 0x71, 0xdc, 0x4c, 0xfc, + 0xa9, 0xdc, 0x8f, 0x57, 0x4d, 0xc5, 0xac, 0xd7, + 0xa9, 0xf3, 0xf3, 0xa1, 0xff, 0x62, 0xa0, 0x8f, + 0xe4, 0x96, 0x3e, 0xcb, 0x9f, 0x76, 0x42, 0x39, + 0x1f, 0x24, 0xfd, 0xfd, 0x79, 0xe8, 0x27, 0xdf, + 0xa8, 0xf6, 0x33, 0x8b, 0x31, 0x59, 0x69, 0xcf, + 0x6a, 0xef, 0x89, 0x4d, 0xa7, 0xf6, 0x7e, 0x97, + 0x14, 0xbd, 0xda, 0xdd, 0xb4, 0x84, 0x04, 0x24, + 0xe0, 0x17, 0xe1, 0x0f, 0x1f, 0x8a, 0x6a, 0x71, + 0x74, 0x41, 0xdc, 0x59, 0x5c, 0x8f, 0x01, 0x25, + 0x92, 0xf0, 0x2e, 0x15, 0x62, 0x71, 0x9a, 0x9f, + 0x87, 0xdf, 0x62, 0x49, 0x7f, 0x86, 0x62, 0xfc, + 0x20, 0x84, 0xd7, 0xe3, 0x3a, 0xd9, 0x37, 0x85, + 0xb7, 0x84, 0x5a, 0xf9, 0xed, 0x21, 0x32, 0x94, + 0x3e, 0x04, 0xe7, 0x8c, 0x46, 0x76, 0x21, 0x67, + 0xf6, 0x95, 0x64, 0x92, 0xb7, 0x15, 0xf6, 0xe3, + 0x41, 0x27, 0x9d, 0xd7, 0xe3, 0x79, 0x75, 0x92, + 0xd0, 0xc1, 0xf3, 0x40, 0x92, 0x08, 0xde, 0x90, + 0x22, 0x82, 0xb2, 0x69, 0xae, 0x1a, 0x35, 0x11, + 0x89, 0xc8, 0x06, 0x82, 0x95, 0x23, 0x44, 0x08, + 0x22, 0xf2, 0x71, 0x73, 0x1b, 0x88, 0x11, 0xcf, + 0x1c, 0x7e, 0x8a, 0x2e, 0xdc, 0x79, 0x57, 0xce, + 0x1f, 0xe7, 0x6c, 0x07, 0xd8, 0x06, 0xbe, 0xec, + 0xa3, 0xcf, 0xf9, 0x68, 0xa5, 0xb8, 0xf0, 0xe3, + 0x3f, 0x01, 0x92, 0xda, 0xf1, 0xa0, 0x2d, 0x7b, + 0xab, 0x57, 0x58, 0x2a, 0xaf, 0xab, 0xbd, 0xf2, + 0xe5, 0xaf, 0x7e, 0x1f, 0x46, 0x24, 0x9e, 0x20, + 0x22, 0x0f, 0x84, 0x4c, 0xb7, 0xd8, 0x03, 0xe8, + 0x09, 0x73, 0x6c, 0xc6, 0x9b, 0x90, 0xe0, 0xdb, + 0xf2, 0x71, 0xba, 0xad, 0xb3, 0xec, 0xda, 0x7a +}; +static const u8 output71[] __initconst = { + 0x28, 0xc5, 0x9b, 0x92, 0xf9, 0x21, 0x4f, 0xbb, + 0xef, 0x3b, 0xf0, 0xf5, 0x3a, 0x6d, 0x7f, 0xd6, + 0x6a, 0x8d, 0xa1, 0x01, 0x5c, 0x62, 0x20, 0x8b, + 0x5b, 0x39, 0xd5, 0xd3, 0xc2, 0xf6, 0x9d, 0x5e, + 0xcc, 0xe1, 0xa2, 0x61, 0x16, 0xe2, 0xce, 0xe9, + 0x86, 0xd0, 0xfc, 0xce, 0x9a, 0x28, 0x27, 0xc4, + 0x0c, 0xb9, 0xaa, 0x8d, 0x48, 0xdb, 0xbf, 0x82, + 0x7d, 0xd0, 0x35, 0xc4, 0x06, 0x34, 0xb4, 0x19, + 0x51, 0x73, 0xf4, 0x7a, 0xf4, 0xfd, 0xe9, 0x1d, + 0xdc, 0x0f, 0x7e, 0xf7, 0x96, 0x03, 0xe3, 0xb1, + 0x2e, 0x22, 0x59, 0xb7, 0x6d, 0x1c, 0x97, 0x8c, + 0xd7, 0x31, 0x08, 0x26, 0x4c, 0x6d, 0xc6, 0x14, + 0xa5, 0xeb, 0x45, 0x6a, 0x88, 0xa3, 0xa2, 0x36, + 0xc4, 0x35, 0xb1, 0x5a, 0xa0, 0xad, 0xf7, 0x06, + 0x9b, 0x5d, 0xc1, 0x15, 0xc1, 0xce, 0x0a, 0xb0, + 0x57, 0x2e, 0x3f, 0x6f, 0x0d, 0x10, 0xd9, 0x11, + 0x2c, 0x9c, 0xad, 0x2d, 0xa5, 0x81, 0xfb, 0x4e, + 0x8f, 0xd5, 0x32, 0x4e, 0xaf, 0x5c, 0xc1, 0x86, + 0xde, 0x56, 0x5a, 0x33, 0x29, 0xf7, 0x67, 0xc6, + 0x37, 0x6f, 0xb2, 0x37, 0x4e, 0xd4, 0x69, 0x79, + 0xaf, 0xd5, 0x17, 0x79, 0xe0, 0xba, 0x62, 0xa3, + 0x68, 0xa4, 0x87, 0x93, 0x8d, 0x7e, 0x8f, 0xa3, + 0x9c, 0xef, 0xda, 0xe3, 0xa5, 0x1f, 0xcd, 0x30, + 0xa6, 0x55, 0xac, 0x4c, 0x69, 0x74, 0x02, 0xc7, + 0x5d, 0x95, 0x81, 0x4a, 0x68, 0x11, 0xd3, 0xa9, + 0x98, 0xb1, 0x0b, 0x0d, 0xae, 0x40, 0x86, 0x65, + 0xbf, 0xcc, 0x2d, 0xef, 0x57, 0xca, 0x1f, 0xe4, + 0x34, 0x4e, 0xa6, 0x5e, 0x82, 0x6e, 0x61, 0xad, + 0x0b, 0x3c, 0xf8, 0xeb, 0x01, 0x43, 0x7f, 0x87, + 0xa2, 0xa7, 0x6a, 0xe9, 0x62, 0x23, 0x24, 0x61, + 0xf1, 0xf7, 0x36, 0xdb, 0x10, 0xe5, 0x57, 0x72, + 0x3a, 0xc2, 0xae, 0xcc, 0x75, 0xc7, 0x80, 0x05, + 0x0a, 0x5c, 0x4c, 0x95, 0xda, 0x02, 0x01, 0x14, + 0x06, 0x6b, 0x5c, 0x65, 0xc2, 0xb8, 0x4a, 0xd6, + 0xd3, 0xb4, 0xd8, 0x12, 0x52, 0xb5, 0x60, 0xd3, + 0x8e, 0x5f, 0x5c, 0x76, 0x33, 0x7a, 0x05, 0xe5, + 0xcb, 0xef, 0x4f, 0x89, 0xf1, 0xba, 0x32, 0x6f, + 0x33, 0xcd, 0x15, 0x8d, 0xa3, 0x0c, 0x3f, 0x63, + 0x11, 0xe7, 0x0e, 0xe0, 0x00, 0x01, 0xe9, 0xe8, + 0x8e, 0x36, 0x34, 0x8d, 0x96, 0xb5, 0x03, 0xcf, + 0x55, 0x62, 0x49, 0x7a, 0x34, 0x44, 0xa5, 0xee, + 0x8c, 0x46, 0x06, 0x22, 0xab, 0x1d, 0x53, 0x9c, + 0xa1, 0xf9, 0x67, 0x18, 0x57, 0x89, 0xf9, 0xc2, + 0xd1, 0x7e, 0xbe, 0x36, 0x40, 0xcb, 0xe9, 0x04, + 0xde, 0xb1, 0x3b, 0x29, 0x52, 0xc5, 0x9a, 0xb5, + 0xa2, 0x7c, 0x7b, 0xfe, 0xe5, 0x92, 0x73, 0xea, + 0xea, 0x7b, 0xba, 0x0a, 0x8c, 0x88, 0x15, 0xe6, + 0x53, 0xbf, 0x1c, 0x33, 0xf4, 0x9b, 0x9a, 0x5e, + 0x8d, 0xae, 0x60, 0xdc, 0xcb, 0x5d, 0xfa, 0xbe, + 0x06, 0xc3, 0x3f, 0x06, 0xe7, 0x00, 0x40, 0x7b, + 0xaa, 0x94, 0xfa, 0x6d, 0x1f, 0xe4, 0xc5, 0xa9, + 0x1b, 0x5f, 0x36, 0xea, 0x5a, 0xdd, 0xa5, 0x48, + 0x6a, 0x55, 0xd2, 0x47, 0x28, 0xbf, 0x96, 0xf1, + 0x9f, 0xb6, 0x11, 0x4b, 0xd3, 0x44, 0x7d, 0x48, + 0x41, 0x61, 0xdb, 0x12, 0xd4, 0xc2, 0x59, 0x82, + 0x4c, 0x47, 0x5c, 0x04, 0xf6, 0x7b, 0xd3, 0x92, + 0x2e, 0xe8, 0x40, 0xef, 0x15, 0x32, 0x97, 0xdc, + 0x35, 0x4c, 0x6e, 0xa4, 0x97, 0xe9, 0x24, 0xde, + 0x63, 0x8b, 0xb1, 0x6b, 0x48, 0xbb, 0x46, 0x1f, + 0x84, 0xd6, 0x17, 0xb0, 0x5a, 0x4a, 0x4e, 0xd5, + 0x31, 0xd7, 0xcf, 0xa0, 0x39, 0xc6, 0x2e, 0xfc, + 0xa6, 0xa3, 0xd3, 0x0f, 0xa4, 0x28, 0xac, 0xb2, + 0xf4, 0x48, 0x8d, 0x50, 0xa5, 0x1c, 0x44, 0x5d, + 0x6e, 0x38, 0xb7, 0x2b, 0x8a, 0x45, 0xa7, 0x3d +}; +static const u8 key71[] __initconst = { + 0x8b, 0x68, 0xc4, 0xb7, 0x0d, 0x81, 0xef, 0x52, + 0x1e, 0x05, 0x96, 0x72, 0x62, 0x89, 0x27, 0x83, + 0xd0, 0xc7, 0x33, 0x6d, 0xf2, 0xcc, 0x69, 0xf9, + 0x23, 0xae, 0x99, 0xb1, 0xd1, 0x05, 0x4e, 0x54 +}; +enum { nonce71 = 0x983f03656d64b5f6ULL }; + +static const u8 input72[] __initconst = { + 0x6b, 0x09, 0xc9, 0x57, 0x3d, 0x79, 0x04, 0x8c, + 0x65, 0xad, 0x4a, 0x0f, 0xa1, 0x31, 0x3a, 0xdd, + 0x14, 0x8e, 0xe8, 0xfe, 0xbf, 0x42, 0x87, 0x98, + 0x2e, 0x8d, 0x83, 0xa3, 0xf8, 0x55, 0x3d, 0x84, + 0x1e, 0x0e, 0x05, 0x4a, 0x38, 0x9e, 0xe7, 0xfe, + 0xd0, 0x4d, 0x79, 0x74, 0x3a, 0x0b, 0x9b, 0xe1, + 0xfd, 0x51, 0x84, 0x4e, 0xb2, 0x25, 0xe4, 0x64, + 0x4c, 0xda, 0xcf, 0x46, 0xec, 0xba, 0x12, 0xeb, + 0x5a, 0x33, 0x09, 0x6e, 0x78, 0x77, 0x8f, 0x30, + 0xb1, 0x7d, 0x3f, 0x60, 0x8c, 0xf2, 0x1d, 0x8e, + 0xb4, 0x70, 0xa2, 0x90, 0x7c, 0x79, 0x1a, 0x2c, + 0xf6, 0x28, 0x79, 0x7c, 0x53, 0xc5, 0xfa, 0xcc, + 0x65, 0x9b, 0xe1, 0x51, 0xd1, 0x7f, 0x1d, 0xc4, + 0xdb, 0xd4, 0xd9, 0x04, 0x61, 0x7d, 0xbe, 0x12, + 0xfc, 0xcd, 0xaf, 0xe4, 0x0f, 0x9c, 0x20, 0xb5, + 0x22, 0x40, 0x18, 0xda, 0xe4, 0xda, 0x8c, 0x2d, + 0x84, 0xe3, 0x5f, 0x53, 0x17, 0xed, 0x78, 0xdc, + 0x2f, 0xe8, 0x31, 0xc7, 0xe6, 0x39, 0x71, 0x40, + 0xb4, 0x0f, 0xc9, 0xa9, 0x7e, 0x78, 0x87, 0xc1, + 0x05, 0x78, 0xbb, 0x01, 0xf2, 0x8f, 0x33, 0xb0, + 0x6e, 0x84, 0xcd, 0x36, 0x33, 0x5c, 0x5b, 0x8e, + 0xf1, 0xac, 0x30, 0xfe, 0x33, 0xec, 0x08, 0xf3, + 0x7e, 0xf2, 0xf0, 0x4c, 0xf2, 0xad, 0xd8, 0xc1, + 0xd4, 0x4e, 0x87, 0x06, 0xd4, 0x75, 0xe7, 0xe3, + 0x09, 0xd3, 0x4d, 0xe3, 0x21, 0x32, 0xba, 0xb4, + 0x68, 0x68, 0xcb, 0x4c, 0xa3, 0x1e, 0xb3, 0x87, + 0x7b, 0xd3, 0x0c, 0x63, 0x37, 0x71, 0x79, 0xfb, + 0x58, 0x36, 0x57, 0x0f, 0x34, 0x1d, 0xc1, 0x42, + 0x02, 0x17, 0xe7, 0xed, 0xe8, 0xe7, 0x76, 0xcb, + 0x42, 0xc4, 0x4b, 0xe2, 0xb2, 0x5e, 0x42, 0xd5, + 0xec, 0x9d, 0xc1, 0x32, 0x71, 0xe4, 0xeb, 0x10, + 0x68, 0x1a, 0x6e, 0x99, 0x8e, 0x73, 0x12, 0x1f, + 0x97, 0x0c, 0x9e, 0xcd, 0x02, 0x3e, 0x4c, 0xa0, + 0xf2, 0x8d, 0xe5, 0x44, 0xca, 0x6d, 0xfe, 0x07, + 0xe3, 0xe8, 0x9b, 0x76, 0xc1, 0x6d, 0xb7, 0x6e, + 0x0d, 0x14, 0x00, 0x6f, 0x8a, 0xfd, 0x43, 0xc6, + 0x43, 0xa5, 0x9c, 0x02, 0x47, 0x10, 0xd4, 0xb4, + 0x9b, 0x55, 0x67, 0xc8, 0x7f, 0xc1, 0x8a, 0x1f, + 0x1e, 0xd1, 0xbc, 0x99, 0x5d, 0x50, 0x4f, 0x89, + 0xf1, 0xe6, 0x5d, 0x91, 0x40, 0xdc, 0x20, 0x67, + 0x56, 0xc2, 0xef, 0xbd, 0x2c, 0xa2, 0x99, 0x38, + 0xe0, 0x45, 0xec, 0x44, 0x05, 0x52, 0x65, 0x11, + 0xfc, 0x3b, 0x19, 0xcb, 0x71, 0xc2, 0x8e, 0x0e, + 0x03, 0x2a, 0x03, 0x3b, 0x63, 0x06, 0x31, 0x9a, + 0xac, 0x53, 0x04, 0x14, 0xd4, 0x80, 0x9d, 0x6b, + 0x42, 0x7e, 0x7e, 0x4e, 0xdc, 0xc7, 0x01, 0x49, + 0x9f, 0xf5, 0x19, 0x86, 0x13, 0x28, 0x2b, 0xa6, + 0xa6, 0xbe, 0xa1, 0x7e, 0x71, 0x05, 0x00, 0xff, + 0x59, 0x2d, 0xb6, 0x63, 0xf0, 0x1e, 0x2e, 0x69, + 0x9b, 0x85, 0xf1, 0x1e, 0x8a, 0x64, 0x39, 0xab, + 0x00, 0x12, 0xe4, 0x33, 0x4b, 0xb5, 0xd8, 0xb3, + 0x6b, 0x5b, 0x8b, 0x5c, 0xd7, 0x6f, 0x23, 0xcf, + 0x3f, 0x2e, 0x5e, 0x47, 0xb9, 0xb8, 0x1f, 0xf0, + 0x1d, 0xda, 0xe7, 0x4f, 0x6e, 0xab, 0xc3, 0x36, + 0xb4, 0x74, 0x6b, 0xeb, 0xc7, 0x5d, 0x91, 0xe5, + 0xda, 0xf2, 0xc2, 0x11, 0x17, 0x48, 0xf8, 0x9c, + 0xc9, 0x8b, 0xc1, 0xa2, 0xf4, 0xcd, 0x16, 0xf8, + 0x27, 0xd9, 0x6c, 0x6f, 0xb5, 0x8f, 0x77, 0xca, + 0x1b, 0xd8, 0xef, 0x84, 0x68, 0x71, 0x53, 0xc1, + 0x43, 0x0f, 0x9f, 0x98, 0xae, 0x7e, 0x31, 0xd2, + 0x98, 0xfb, 0x20, 0xa2, 0xad, 0x00, 0x10, 0x83, + 0x00, 0x8b, 0xeb, 0x56, 0xd2, 0xc4, 0xcc, 0x7f, + 0x2f, 0x4e, 0xfa, 0x88, 0x13, 0xa4, 0x2c, 0xde, + 0x6b, 0x77, 0x86, 0x10, 0x6a, 0xab, 0x43, 0x0a, + 0x02 +}; +static const u8 output72[] __initconst = { + 0x42, 0x89, 0xa4, 0x80, 0xd2, 0xcb, 0x5f, 0x7f, + 0x2a, 0x1a, 0x23, 0x00, 0xa5, 0x6a, 0x95, 0xa3, + 0x9a, 0x41, 0xa1, 0xd0, 0x2d, 0x1e, 0xd6, 0x13, + 0x34, 0x40, 0x4e, 0x7f, 0x1a, 0xbe, 0xa0, 0x3d, + 0x33, 0x9c, 0x56, 0x2e, 0x89, 0x25, 0x45, 0xf9, + 0xf0, 0xba, 0x9c, 0x6d, 0xd1, 0xd1, 0xde, 0x51, + 0x47, 0x63, 0xc9, 0xbd, 0xfa, 0xa2, 0x9e, 0xad, + 0x6a, 0x7b, 0x21, 0x1a, 0x6c, 0x3e, 0xff, 0x46, + 0xbe, 0xf3, 0x35, 0x7a, 0x6e, 0xb3, 0xb9, 0xf7, + 0xda, 0x5e, 0xf0, 0x14, 0xb5, 0x70, 0xa4, 0x2b, + 0xdb, 0xbb, 0xc7, 0x31, 0x4b, 0x69, 0x5a, 0x83, + 0x70, 0xd9, 0x58, 0xd4, 0x33, 0x84, 0x23, 0xf0, + 0xae, 0xbb, 0x6d, 0x26, 0x7c, 0xc8, 0x30, 0xf7, + 0x24, 0xad, 0xbd, 0xe4, 0x2c, 0x38, 0x38, 0xac, + 0xe1, 0x4a, 0x9b, 0xac, 0x33, 0x0e, 0x4a, 0xf4, + 0x93, 0xed, 0x07, 0x82, 0x81, 0x4f, 0x8f, 0xb1, + 0xdd, 0x73, 0xd5, 0x50, 0x6d, 0x44, 0x1e, 0xbe, + 0xa7, 0xcd, 0x17, 0x57, 0xd5, 0x3b, 0x62, 0x36, + 0xcf, 0x7d, 0xc8, 0xd8, 0xd1, 0x78, 0xd7, 0x85, + 0x46, 0x76, 0x5d, 0xcc, 0xfe, 0xe8, 0x94, 0xc5, + 0xad, 0xbc, 0x5e, 0xbc, 0x8d, 0x1d, 0xdf, 0x03, + 0xc9, 0x6b, 0x1b, 0x81, 0xd1, 0xb6, 0x5a, 0x24, + 0xe3, 0xdc, 0x3f, 0x20, 0xc9, 0x07, 0x73, 0x4c, + 0x43, 0x13, 0x87, 0x58, 0x34, 0x0d, 0x14, 0x63, + 0x0f, 0x6f, 0xad, 0x8d, 0xac, 0x7c, 0x67, 0x68, + 0xa3, 0x9d, 0x7f, 0x00, 0xdf, 0x28, 0xee, 0x67, + 0xf4, 0x5c, 0x26, 0xcb, 0xef, 0x56, 0x71, 0xc8, + 0xc6, 0x67, 0x5f, 0x38, 0xbb, 0xa0, 0xb1, 0x5c, + 0x1f, 0xb3, 0x08, 0xd9, 0x38, 0xcf, 0x74, 0x54, + 0xc6, 0xa4, 0xc4, 0xc0, 0x9f, 0xb3, 0xd0, 0xda, + 0x62, 0x67, 0x8b, 0x81, 0x33, 0xf0, 0xa9, 0x73, + 0xa4, 0xd1, 0x46, 0x88, 0x8d, 0x85, 0x12, 0x40, + 0xba, 0x1a, 0xcd, 0x82, 0xd8, 0x8d, 0xc4, 0x52, + 0xe7, 0x01, 0x94, 0x2e, 0x0e, 0xd0, 0xaf, 0xe7, + 0x2d, 0x3f, 0x3c, 0xaa, 0xf4, 0xf5, 0xa7, 0x01, + 0x4c, 0x14, 0xe2, 0xc2, 0x96, 0x76, 0xbe, 0x05, + 0xaa, 0x19, 0xb1, 0xbd, 0x95, 0xbb, 0x5a, 0xf9, + 0xa5, 0xa7, 0xe6, 0x16, 0x38, 0x34, 0xf7, 0x9d, + 0x19, 0x66, 0x16, 0x8e, 0x7f, 0x2b, 0x5a, 0xfb, + 0xb5, 0x29, 0x79, 0xbf, 0x52, 0xae, 0x30, 0x95, + 0x3f, 0x31, 0x33, 0x28, 0xde, 0xc5, 0x0d, 0x55, + 0x89, 0xec, 0x21, 0x11, 0x0f, 0x8b, 0xfe, 0x63, + 0x3a, 0xf1, 0x95, 0x5c, 0xcd, 0x50, 0xe4, 0x5d, + 0x8f, 0xa7, 0xc8, 0xca, 0x93, 0xa0, 0x67, 0x82, + 0x63, 0x5c, 0xd0, 0xed, 0xe7, 0x08, 0xc5, 0x60, + 0xf8, 0xb4, 0x47, 0xf0, 0x1a, 0x65, 0x4e, 0xa3, + 0x51, 0x68, 0xc7, 0x14, 0xa1, 0xd9, 0x39, 0x72, + 0xa8, 0x6f, 0x7c, 0x7e, 0xf6, 0x03, 0x0b, 0x25, + 0x9b, 0xf2, 0xca, 0x49, 0xae, 0x5b, 0xf8, 0x0f, + 0x71, 0x51, 0x01, 0xa6, 0x23, 0xa9, 0xdf, 0xd0, + 0x7a, 0x39, 0x19, 0xf5, 0xc5, 0x26, 0x44, 0x7b, + 0x0a, 0x4a, 0x41, 0xbf, 0xf2, 0x8e, 0x83, 0x50, + 0x91, 0x96, 0x72, 0x02, 0xf6, 0x80, 0xbf, 0x95, + 0x41, 0xac, 0xda, 0xb0, 0xba, 0xe3, 0x76, 0xb1, + 0x9d, 0xff, 0x1f, 0x33, 0x02, 0x85, 0xfc, 0x2a, + 0x29, 0xe6, 0xe3, 0x9d, 0xd0, 0xef, 0xc2, 0xd6, + 0x9c, 0x4a, 0x62, 0xac, 0xcb, 0xea, 0x8b, 0xc3, + 0x08, 0x6e, 0x49, 0x09, 0x26, 0x19, 0xc1, 0x30, + 0xcc, 0x27, 0xaa, 0xc6, 0x45, 0x88, 0xbd, 0xae, + 0xd6, 0x79, 0xff, 0x4e, 0xfc, 0x66, 0x4d, 0x02, + 0xa5, 0xee, 0x8e, 0xa5, 0xb6, 0x15, 0x72, 0x24, + 0xb1, 0xbf, 0xbf, 0x64, 0xcf, 0xcc, 0x93, 0xe9, + 0xb6, 0xfd, 0xb4, 0xb6, 0x21, 0xb5, 0x48, 0x08, + 0x0f, 0x11, 0x65, 0xe1, 0x47, 0xee, 0x93, 0x29, + 0xad +}; +static const u8 key72[] __initconst = { + 0xb9, 0xa2, 0xfc, 0x59, 0x06, 0x3f, 0x77, 0xa5, + 0x66, 0xd0, 0x2b, 0x22, 0x74, 0x22, 0x4c, 0x1e, + 0x6a, 0x39, 0xdf, 0xe1, 0x0d, 0x4c, 0x64, 0x99, + 0x54, 0x8a, 0xba, 0x1d, 0x2c, 0x21, 0x5f, 0xc3 +}; +enum { nonce72 = 0x3d069308fa3db04bULL }; + +static const u8 input73[] __initconst = { + 0xe4, 0xdd, 0x36, 0xd4, 0xf5, 0x70, 0x51, 0x73, + 0x97, 0x1d, 0x45, 0x05, 0x92, 0xe7, 0xeb, 0xb7, + 0x09, 0x82, 0x6e, 0x25, 0x6c, 0x50, 0xf5, 0x40, + 0x19, 0xba, 0xbc, 0xf4, 0x39, 0x14, 0xc5, 0x15, + 0x83, 0x40, 0xbd, 0x26, 0xe0, 0xff, 0x3b, 0x22, + 0x7c, 0x7c, 0xd7, 0x0b, 0xe9, 0x25, 0x0c, 0x3d, + 0x92, 0x38, 0xbe, 0xe4, 0x22, 0x75, 0x65, 0xf1, + 0x03, 0x85, 0x34, 0x09, 0xb8, 0x77, 0xfb, 0x48, + 0xb1, 0x2e, 0x21, 0x67, 0x9b, 0x9d, 0xad, 0x18, + 0x82, 0x0d, 0x6b, 0xc3, 0xcf, 0x00, 0x61, 0x6e, + 0xda, 0xdc, 0xa7, 0x0b, 0x5c, 0x02, 0x1d, 0xa6, + 0x4e, 0x0d, 0x7f, 0x37, 0x01, 0x5a, 0x37, 0xf3, + 0x2b, 0xbf, 0xba, 0xe2, 0x1c, 0xb3, 0xa3, 0xbc, + 0x1c, 0x93, 0x1a, 0xb1, 0x71, 0xaf, 0xe2, 0xdd, + 0x17, 0xee, 0x53, 0xfa, 0xfb, 0x02, 0x40, 0x3e, + 0x03, 0xca, 0xe7, 0xc3, 0x51, 0x81, 0xcc, 0x8c, + 0xca, 0xcf, 0x4e, 0xc5, 0x78, 0x99, 0xfd, 0xbf, + 0xea, 0xab, 0x38, 0x81, 0xfc, 0xd1, 0x9e, 0x41, + 0x0b, 0x84, 0x25, 0xf1, 0x6b, 0x3c, 0xf5, 0x40, + 0x0d, 0xc4, 0x3e, 0xb3, 0x6a, 0xec, 0x6e, 0x75, + 0xdc, 0x9b, 0xdf, 0x08, 0x21, 0x16, 0xfb, 0x7a, + 0x8e, 0x19, 0x13, 0x02, 0xa7, 0xfc, 0x58, 0x21, + 0xc3, 0xb3, 0x59, 0x5a, 0x9c, 0xef, 0x38, 0xbd, + 0x87, 0x55, 0xd7, 0x0d, 0x1f, 0x84, 0xdc, 0x98, + 0x22, 0xca, 0x87, 0x96, 0x71, 0x6d, 0x68, 0x00, + 0xcb, 0x4f, 0x2f, 0xc4, 0x64, 0x0c, 0xc1, 0x53, + 0x0c, 0x90, 0xe7, 0x3c, 0x88, 0xca, 0xc5, 0x85, + 0xa3, 0x2a, 0x96, 0x7c, 0x82, 0x6d, 0x45, 0xf5, + 0xb7, 0x8d, 0x17, 0x69, 0xd6, 0xcd, 0x3c, 0xd3, + 0xe7, 0x1c, 0xce, 0x93, 0x50, 0xd4, 0x59, 0xa2, + 0xd8, 0x8b, 0x72, 0x60, 0x5b, 0x25, 0x14, 0xcd, + 0x5a, 0xe8, 0x8c, 0xdb, 0x23, 0x8d, 0x2b, 0x59, + 0x12, 0x13, 0x10, 0x47, 0xa4, 0xc8, 0x3c, 0xc1, + 0x81, 0x89, 0x6c, 0x98, 0xec, 0x8f, 0x7b, 0x32, + 0xf2, 0x87, 0xd9, 0xa2, 0x0d, 0xc2, 0x08, 0xf9, + 0xd5, 0xf3, 0x91, 0xe7, 0xb3, 0x87, 0xa7, 0x0b, + 0x64, 0x8f, 0xb9, 0x55, 0x1c, 0x81, 0x96, 0x6c, + 0xa1, 0xc9, 0x6e, 0x3b, 0xcd, 0x17, 0x1b, 0xfc, + 0xa6, 0x05, 0xba, 0x4a, 0x7d, 0x03, 0x3c, 0x59, + 0xc8, 0xee, 0x50, 0xb2, 0x5b, 0xe1, 0x4d, 0x6a, + 0x1f, 0x09, 0xdc, 0xa2, 0x51, 0xd1, 0x93, 0x3a, + 0x5f, 0x72, 0x1d, 0x26, 0x14, 0x62, 0xa2, 0x41, + 0x3d, 0x08, 0x70, 0x7b, 0x27, 0x3d, 0xbc, 0xdf, + 0x15, 0xfa, 0xb9, 0x5f, 0xb5, 0x38, 0x84, 0x0b, + 0x58, 0x3d, 0xee, 0x3f, 0x32, 0x65, 0x6d, 0xd7, + 0xce, 0x97, 0x3c, 0x8d, 0xfb, 0x63, 0xb9, 0xb0, + 0xa8, 0x4a, 0x72, 0x99, 0x97, 0x58, 0xc8, 0xa7, + 0xf9, 0x4c, 0xae, 0xc1, 0x63, 0xb9, 0x57, 0x18, + 0x8a, 0xfa, 0xab, 0xe9, 0xf3, 0x67, 0xe6, 0xfd, + 0xd2, 0x9d, 0x5c, 0xa9, 0x8e, 0x11, 0x0a, 0xf4, + 0x4b, 0xf1, 0xec, 0x1a, 0xaf, 0x50, 0x5d, 0x16, + 0x13, 0x69, 0x2e, 0xbd, 0x0d, 0xe6, 0xf0, 0xb2, + 0xed, 0xb4, 0x4c, 0x59, 0x77, 0x37, 0x00, 0x0b, + 0xc7, 0xa7, 0x9e, 0x37, 0xf3, 0x60, 0x70, 0xef, + 0xf3, 0xc1, 0x74, 0x52, 0x87, 0xc6, 0xa1, 0x81, + 0xbd, 0x0a, 0x2c, 0x5d, 0x2c, 0x0c, 0x6a, 0x81, + 0xa1, 0xfe, 0x26, 0x78, 0x6c, 0x03, 0x06, 0x07, + 0x34, 0xaa, 0xd1, 0x1b, 0x40, 0x03, 0x39, 0x56, + 0xcf, 0x2a, 0x92, 0xc1, 0x4e, 0xdf, 0x29, 0x24, + 0x83, 0x22, 0x7a, 0xea, 0x67, 0x1e, 0xe7, 0x54, + 0x64, 0xd3, 0xbd, 0x3a, 0x5d, 0xae, 0xca, 0xf0, + 0x9c, 0xd6, 0x5a, 0x9a, 0x62, 0xc8, 0xc7, 0x83, + 0xf9, 0x89, 0xde, 0x2d, 0x53, 0x64, 0x61, 0xf7, + 0xa3, 0xa7, 0x31, 0x38, 0xc6, 0x22, 0x9c, 0xb4, + 0x87, 0xe0 +}; +static const u8 output73[] __initconst = { + 0x34, 0xed, 0x05, 0xb0, 0x14, 0xbc, 0x8c, 0xcc, + 0x95, 0xbd, 0x99, 0x0f, 0xb1, 0x98, 0x17, 0x10, + 0xae, 0xe0, 0x08, 0x53, 0xa3, 0x69, 0xd2, 0xed, + 0x66, 0xdb, 0x2a, 0x34, 0x8d, 0x0c, 0x6e, 0xce, + 0x63, 0x69, 0xc9, 0xe4, 0x57, 0xc3, 0x0c, 0x8b, + 0xa6, 0x2c, 0xa7, 0xd2, 0x08, 0xff, 0x4f, 0xec, + 0x61, 0x8c, 0xee, 0x0d, 0xfa, 0x6b, 0xe0, 0xe8, + 0x71, 0xbc, 0x41, 0x46, 0xd7, 0x33, 0x1d, 0xc0, + 0xfd, 0xad, 0xca, 0x8b, 0x34, 0x56, 0xa4, 0x86, + 0x71, 0x62, 0xae, 0x5e, 0x3d, 0x2b, 0x66, 0x3e, + 0xae, 0xd8, 0xc0, 0xe1, 0x21, 0x3b, 0xca, 0xd2, + 0x6b, 0xa2, 0xb8, 0xc7, 0x98, 0x4a, 0xf3, 0xcf, + 0xb8, 0x62, 0xd8, 0x33, 0xe6, 0x80, 0xdb, 0x2f, + 0x0a, 0xaf, 0x90, 0x3c, 0xe1, 0xec, 0xe9, 0x21, + 0x29, 0x42, 0x9e, 0xa5, 0x50, 0xe9, 0x93, 0xd3, + 0x53, 0x1f, 0xac, 0x2a, 0x24, 0x07, 0xb8, 0xed, + 0xed, 0x38, 0x2c, 0xc4, 0xa1, 0x2b, 0x31, 0x5d, + 0x9c, 0x24, 0x7b, 0xbf, 0xd9, 0xbb, 0x4e, 0x87, + 0x8f, 0x32, 0x30, 0xf1, 0x11, 0x29, 0x54, 0x94, + 0x00, 0x95, 0x1d, 0x1d, 0x24, 0xc0, 0xd4, 0x34, + 0x49, 0x1d, 0xd5, 0xe3, 0xa6, 0xde, 0x8b, 0xbf, + 0x5a, 0x9f, 0x58, 0x5a, 0x9b, 0x70, 0xe5, 0x9b, + 0xb3, 0xdb, 0xe8, 0xb8, 0xca, 0x1b, 0x43, 0xe3, + 0xc6, 0x6f, 0x0a, 0xd6, 0x32, 0x11, 0xd4, 0x04, + 0xef, 0xa3, 0xe4, 0x3f, 0x12, 0xd8, 0xc1, 0x73, + 0x51, 0x87, 0x03, 0xbd, 0xba, 0x60, 0x79, 0xee, + 0x08, 0xcc, 0xf7, 0xc0, 0xaa, 0x4c, 0x33, 0xc4, + 0xc7, 0x09, 0xf5, 0x91, 0xcb, 0x74, 0x57, 0x08, + 0x1b, 0x90, 0xa9, 0x1b, 0x60, 0x02, 0xd2, 0x3f, + 0x7a, 0xbb, 0xfd, 0x78, 0xf0, 0x15, 0xf9, 0x29, + 0x82, 0x8f, 0xc4, 0xb2, 0x88, 0x1f, 0xbc, 0xcc, + 0x53, 0x27, 0x8b, 0x07, 0x5f, 0xfc, 0x91, 0x29, + 0x82, 0x80, 0x59, 0x0a, 0x3c, 0xea, 0xc4, 0x7e, + 0xad, 0xd2, 0x70, 0x46, 0xbd, 0x9e, 0x3b, 0x1c, + 0x8a, 0x62, 0xea, 0x69, 0xbd, 0xf6, 0x96, 0x15, + 0xb5, 0x57, 0xe8, 0x63, 0x5f, 0x65, 0x46, 0x84, + 0x58, 0x50, 0x87, 0x4b, 0x0e, 0x5b, 0x52, 0x90, + 0xb0, 0xae, 0x37, 0x0f, 0xdd, 0x7e, 0xa2, 0xa0, + 0x8b, 0x78, 0xc8, 0x5a, 0x1f, 0x53, 0xdb, 0xc5, + 0xbf, 0x73, 0x20, 0xa9, 0x44, 0xfb, 0x1e, 0xc7, + 0x97, 0xb2, 0x3a, 0x5a, 0x17, 0xe6, 0x8b, 0x9b, + 0xe8, 0xf8, 0x2a, 0x01, 0x27, 0xa3, 0x71, 0x28, + 0xe3, 0x19, 0xc6, 0xaf, 0xf5, 0x3a, 0x26, 0xc0, + 0x5c, 0x69, 0x30, 0x78, 0x75, 0x27, 0xf2, 0x0c, + 0x22, 0x71, 0x65, 0xc6, 0x8e, 0x7b, 0x47, 0xe3, + 0x31, 0xaf, 0x7b, 0xc6, 0xc2, 0x55, 0x68, 0x81, + 0xaa, 0x1b, 0x21, 0x65, 0xfb, 0x18, 0x35, 0x45, + 0x36, 0x9a, 0x44, 0xba, 0x5c, 0xff, 0x06, 0xde, + 0x3a, 0xc8, 0x44, 0x0b, 0xaa, 0x8e, 0x34, 0xe2, + 0x84, 0xac, 0x18, 0xfe, 0x9b, 0xe1, 0x4f, 0xaa, + 0xb6, 0x90, 0x0b, 0x1c, 0x2c, 0xd9, 0x9a, 0x10, + 0x18, 0xf9, 0x49, 0x41, 0x42, 0x1b, 0xb5, 0xe1, + 0x26, 0xac, 0x2d, 0x38, 0x00, 0x00, 0xe4, 0xb4, + 0x50, 0x6f, 0x14, 0x18, 0xd6, 0x3d, 0x00, 0x59, + 0x3c, 0x45, 0xf3, 0x42, 0x13, 0x44, 0xb8, 0x57, + 0xd4, 0x43, 0x5c, 0x8a, 0x2a, 0xb4, 0xfc, 0x0a, + 0x25, 0x5a, 0xdc, 0x8f, 0x11, 0x0b, 0x11, 0x44, + 0xc7, 0x0e, 0x54, 0x8b, 0x22, 0x01, 0x7e, 0x67, + 0x2e, 0x15, 0x3a, 0xb9, 0xee, 0x84, 0x10, 0xd4, + 0x80, 0x57, 0xd7, 0x75, 0xcf, 0x8b, 0xcb, 0x03, + 0xc9, 0x92, 0x2b, 0x69, 0xd8, 0x5a, 0x9b, 0x06, + 0x85, 0x47, 0xaa, 0x4c, 0x28, 0xde, 0x49, 0x58, + 0xe6, 0x11, 0x1e, 0x5e, 0x64, 0x8e, 0x3b, 0xe0, + 0x40, 0x2e, 0xac, 0x96, 0x97, 0x15, 0x37, 0x1e, + 0x30, 0xdd +}; +static const u8 key73[] __initconst = { + 0x96, 0x06, 0x1e, 0xc1, 0x6d, 0xba, 0x49, 0x5b, + 0x65, 0x80, 0x79, 0xdd, 0xf3, 0x67, 0xa8, 0x6e, + 0x2d, 0x9c, 0x54, 0x46, 0xd8, 0x4a, 0xeb, 0x7e, + 0x23, 0x86, 0x51, 0xd8, 0x49, 0x49, 0x56, 0xe0 +}; +enum { nonce73 = 0xbefb83cb67e11ffdULL }; + +static const u8 input74[] __initconst = { + 0x47, 0x22, 0x70, 0xe5, 0x2f, 0x41, 0x18, 0x45, + 0x07, 0xd3, 0x6d, 0x32, 0x0d, 0x43, 0x92, 0x2b, + 0x9b, 0x65, 0x73, 0x13, 0x1a, 0x4f, 0x49, 0x8f, + 0xff, 0xf8, 0xcc, 0xae, 0x15, 0xab, 0x9d, 0x7d, + 0xee, 0x22, 0x5d, 0x8b, 0xde, 0x81, 0x5b, 0x81, + 0x83, 0x49, 0x35, 0x9b, 0xb4, 0xbc, 0x4e, 0x01, + 0xc2, 0x29, 0xa7, 0xf1, 0xca, 0x3a, 0xce, 0x3f, + 0xf5, 0x31, 0x93, 0xa8, 0xe2, 0xc9, 0x7d, 0x03, + 0x26, 0xa4, 0xbc, 0xa8, 0x9c, 0xb9, 0x68, 0xf3, + 0xb3, 0x91, 0xe8, 0xe6, 0xc7, 0x2b, 0x1a, 0xce, + 0xd2, 0x41, 0x53, 0xbd, 0xa3, 0x2c, 0x54, 0x94, + 0x21, 0xa1, 0x40, 0xae, 0xc9, 0x0c, 0x11, 0x92, + 0xfd, 0x91, 0xa9, 0x40, 0xca, 0xde, 0x21, 0x4e, + 0x1e, 0x3d, 0xcc, 0x2c, 0x87, 0x11, 0xef, 0x46, + 0xed, 0x52, 0x03, 0x11, 0x19, 0x43, 0x25, 0xc7, + 0x0d, 0xc3, 0x37, 0x5f, 0xd3, 0x6f, 0x0c, 0x6a, + 0x45, 0x30, 0x88, 0xec, 0xf0, 0x21, 0xef, 0x1d, + 0x7b, 0x38, 0x63, 0x4b, 0x49, 0x0c, 0x72, 0xf6, + 0x4c, 0x40, 0xc3, 0xcc, 0x03, 0xa7, 0xae, 0xa8, + 0x8c, 0x37, 0x03, 0x1c, 0x11, 0xae, 0x0d, 0x1b, + 0x62, 0x97, 0x27, 0xfc, 0x56, 0x4b, 0xb7, 0xfd, + 0xbc, 0xfb, 0x0e, 0xfc, 0x61, 0xad, 0xc6, 0xb5, + 0x9c, 0x8c, 0xc6, 0x38, 0x27, 0x91, 0x29, 0x3d, + 0x29, 0xc8, 0x37, 0xc9, 0x96, 0x69, 0xe3, 0xdc, + 0x3e, 0x61, 0x35, 0x9b, 0x99, 0x4f, 0xb9, 0x4e, + 0x5a, 0x29, 0x1c, 0x2e, 0xcf, 0x16, 0xcb, 0x69, + 0x87, 0xe4, 0x1a, 0xc4, 0x6e, 0x78, 0x43, 0x00, + 0x03, 0xb2, 0x8b, 0x03, 0xd0, 0xb4, 0xf1, 0xd2, + 0x7d, 0x2d, 0x7e, 0xfc, 0x19, 0x66, 0x5b, 0xa3, + 0x60, 0x3f, 0x9d, 0xbd, 0xfa, 0x3e, 0xca, 0x7b, + 0x26, 0x08, 0x19, 0x16, 0x93, 0x5d, 0x83, 0xfd, + 0xf9, 0x21, 0xc6, 0x31, 0x34, 0x6f, 0x0c, 0xaa, + 0x28, 0xf9, 0x18, 0xa2, 0xc4, 0x78, 0x3b, 0x56, + 0xc0, 0x88, 0x16, 0xba, 0x22, 0x2c, 0x07, 0x2f, + 0x70, 0xd0, 0xb0, 0x46, 0x35, 0xc7, 0x14, 0xdc, + 0xbb, 0x56, 0x23, 0x1e, 0x36, 0x36, 0x2d, 0x73, + 0x78, 0xc7, 0xce, 0xf3, 0x58, 0xf7, 0x58, 0xb5, + 0x51, 0xff, 0x33, 0x86, 0x0e, 0x3b, 0x39, 0xfb, + 0x1a, 0xfd, 0xf8, 0x8b, 0x09, 0x33, 0x1b, 0x83, + 0xf2, 0xe6, 0x38, 0x37, 0xef, 0x47, 0x84, 0xd9, + 0x82, 0x77, 0x2b, 0x82, 0xcc, 0xf9, 0xee, 0x94, + 0x71, 0x78, 0x81, 0xc8, 0x4d, 0x91, 0xd7, 0x35, + 0x29, 0x31, 0x30, 0x5c, 0x4a, 0x23, 0x23, 0xb1, + 0x38, 0x6b, 0xac, 0x22, 0x3f, 0x80, 0xc7, 0xe0, + 0x7d, 0xfa, 0x76, 0x47, 0xd4, 0x6f, 0x93, 0xa0, + 0xa0, 0x93, 0x5d, 0x68, 0xf7, 0x43, 0x25, 0x8f, + 0x1b, 0xc7, 0x87, 0xea, 0x59, 0x0c, 0xa2, 0xfa, + 0xdb, 0x2f, 0x72, 0x43, 0xcf, 0x90, 0xf1, 0xd6, + 0x58, 0xf3, 0x17, 0x6a, 0xdf, 0xb3, 0x4e, 0x0e, + 0x38, 0x24, 0x48, 0x1f, 0xb7, 0x01, 0xec, 0x81, + 0xb1, 0x87, 0x5b, 0xec, 0x9c, 0x11, 0x1a, 0xff, + 0xa5, 0xca, 0x5a, 0x63, 0x31, 0xb2, 0xe4, 0xc6, + 0x3c, 0x1d, 0xaf, 0x27, 0xb2, 0xd4, 0x19, 0xa2, + 0xcc, 0x04, 0x92, 0x42, 0xd2, 0xc1, 0x8c, 0x3b, + 0xce, 0xf5, 0x74, 0xc1, 0x81, 0xf8, 0x20, 0x23, + 0x6f, 0x20, 0x6d, 0x78, 0x36, 0x72, 0x2c, 0x52, + 0xdf, 0x5e, 0xe8, 0x75, 0xce, 0x1c, 0x49, 0x9d, + 0x93, 0x6f, 0x65, 0xeb, 0xb1, 0xbd, 0x8e, 0x5e, + 0xe5, 0x89, 0xc4, 0x8a, 0x81, 0x3d, 0x9a, 0xa7, + 0x11, 0x82, 0x8e, 0x38, 0x5b, 0x5b, 0xca, 0x7d, + 0x4b, 0x72, 0xc2, 0x9c, 0x30, 0x5e, 0x7f, 0xc0, + 0x6f, 0x91, 0xd5, 0x67, 0x8c, 0x3e, 0xae, 0xda, + 0x2b, 0x3c, 0x53, 0xcc, 0x50, 0x97, 0x36, 0x0b, + 0x79, 0xd6, 0x73, 0x6e, 0x7d, 0x42, 0x56, 0xe1, + 0xaa, 0xfc, 0xb3, 0xa7, 0xc8, 0x01, 0xaa, 0xc1, + 0xfc, 0x5c, 0x72, 0x8e, 0x63, 0xa8, 0x46, 0x18, + 0xee, 0x11, 0xe7, 0x30, 0x09, 0x83, 0x6c, 0xd9, + 0xf4, 0x7a, 0x7b, 0xb5, 0x1f, 0x6d, 0xc7, 0xbc, + 0xcb, 0x55, 0xea, 0x40, 0x58, 0x7a, 0x00, 0x00, + 0x90, 0x60, 0xc5, 0x64, 0x69, 0x05, 0x99, 0xd2, + 0x49, 0x62, 0x4f, 0xcb, 0x97, 0xdf, 0xdd, 0x6b, + 0x60, 0x75, 0xe2, 0xe0, 0x6f, 0x76, 0xd0, 0x37, + 0x67, 0x0a, 0xcf, 0xff, 0xc8, 0x61, 0x84, 0x14, + 0x80, 0x7c, 0x1d, 0x31, 0x8d, 0x90, 0xde, 0x0b, + 0x1c, 0x74, 0x9f, 0x82, 0x96, 0x80, 0xda, 0xaf, + 0x8d, 0x99, 0x86, 0x9f, 0x24, 0x99, 0x28, 0x3e, + 0xe0, 0xa3, 0xc3, 0x90, 0x2d, 0x14, 0x65, 0x1e, + 0x3b, 0xb9, 0xba, 0x13, 0xa5, 0x77, 0x73, 0x63, + 0x9a, 0x06, 0x3d, 0xa9, 0x28, 0x9b, 0xba, 0x25, + 0x61, 0xc9, 0xcd, 0xcf, 0x7a, 0x4d, 0x96, 0x09, + 0xcb, 0xca, 0x03, 0x9c, 0x54, 0x34, 0x31, 0x85, + 0xa0, 0x3d, 0xe5, 0xbc, 0xa5, 0x5f, 0x1b, 0xd3, + 0x10, 0x63, 0x74, 0x9d, 0x01, 0x92, 0x88, 0xf0, + 0x27, 0x9c, 0x28, 0xd9, 0xfd, 0xe2, 0x4e, 0x01, + 0x8d, 0x61, 0x79, 0x60, 0x61, 0x5b, 0x76, 0xab, + 0x06, 0xd3, 0x44, 0x87, 0x43, 0x52, 0xcd, 0x06, + 0x68, 0x1e, 0x2d, 0xc5, 0xb0, 0x07, 0x25, 0xdf, + 0x0a, 0x50, 0xd7, 0xd9, 0x08, 0x53, 0x65, 0xf1, + 0x0c, 0x2c, 0xde, 0x3f, 0x9d, 0x03, 0x1f, 0xe1, + 0x49, 0x43, 0x3c, 0x83, 0x81, 0x37, 0xf8, 0xa2, + 0x0b, 0xf9, 0x61, 0x1c, 0xc1, 0xdb, 0x79, 0xbc, + 0x64, 0xce, 0x06, 0x4e, 0x87, 0x89, 0x62, 0x73, + 0x51, 0xbc, 0xa4, 0x32, 0xd4, 0x18, 0x62, 0xab, + 0x65, 0x7e, 0xad, 0x1e, 0x91, 0xa3, 0xfa, 0x2d, + 0x58, 0x9e, 0x2a, 0xe9, 0x74, 0x44, 0x64, 0x11, + 0xe6, 0xb6, 0xb3, 0x00, 0x7e, 0xa3, 0x16, 0xef, + 0x72 +}; +static const u8 output74[] __initconst = { + 0xf5, 0xca, 0x45, 0x65, 0x50, 0x35, 0x47, 0x67, + 0x6f, 0x4f, 0x67, 0xff, 0x34, 0xd9, 0xc3, 0x37, + 0x2a, 0x26, 0xb0, 0x4f, 0x08, 0x1e, 0x45, 0x13, + 0xc7, 0x2c, 0x14, 0x75, 0x33, 0xd8, 0x8e, 0x1e, + 0x1b, 0x11, 0x0d, 0x97, 0x04, 0x33, 0x8a, 0xe4, + 0xd8, 0x8d, 0x0e, 0x12, 0x8d, 0xdb, 0x6e, 0x02, + 0xfa, 0xe5, 0xbd, 0x3a, 0xb5, 0x28, 0x07, 0x7d, + 0x20, 0xf0, 0x12, 0x64, 0x83, 0x2f, 0x59, 0x79, + 0x17, 0x88, 0x3c, 0x2d, 0x08, 0x2f, 0x55, 0xda, + 0xcc, 0x02, 0x3a, 0x82, 0xcd, 0x03, 0x94, 0xdf, + 0xdf, 0xab, 0x8a, 0x13, 0xf5, 0xe6, 0x74, 0xdf, + 0x7b, 0xe2, 0xab, 0x34, 0xbc, 0x00, 0x85, 0xbf, + 0x5a, 0x48, 0xc8, 0xff, 0x8d, 0x6c, 0x27, 0x48, + 0x19, 0x2d, 0x08, 0xfa, 0x82, 0x62, 0x39, 0x55, + 0x32, 0x11, 0xa8, 0xd7, 0xb9, 0x08, 0x2c, 0xd6, + 0x7a, 0xd9, 0x83, 0x9f, 0x9b, 0xfb, 0xec, 0x3a, + 0xd1, 0x08, 0xc7, 0xad, 0xdc, 0x98, 0x4c, 0xbc, + 0x98, 0xeb, 0x36, 0xb0, 0x39, 0xf4, 0x3a, 0xd6, + 0x53, 0x02, 0xa0, 0xa9, 0x73, 0xa1, 0xca, 0xef, + 0xd8, 0xd2, 0xec, 0x0e, 0xf8, 0xf5, 0xac, 0x8d, + 0x34, 0x41, 0x06, 0xa8, 0xc6, 0xc3, 0x31, 0xbc, + 0xe5, 0xcc, 0x7e, 0x72, 0x63, 0x59, 0x3e, 0x63, + 0xc2, 0x8d, 0x2b, 0xd5, 0xb9, 0xfd, 0x1e, 0x31, + 0x69, 0x32, 0x05, 0xd6, 0xde, 0xc9, 0xe6, 0x4c, + 0xac, 0x68, 0xf7, 0x1f, 0x9d, 0xcd, 0x0e, 0xa2, + 0x15, 0x3d, 0xd6, 0x47, 0x99, 0xab, 0x08, 0x5f, + 0x28, 0xc3, 0x4c, 0xc2, 0xd5, 0xdd, 0x10, 0xb7, + 0xbd, 0xdb, 0x9b, 0xcf, 0x85, 0x27, 0x29, 0x76, + 0x98, 0xeb, 0xad, 0x31, 0x64, 0xe7, 0xfb, 0x61, + 0xe0, 0xd8, 0x1a, 0xa6, 0xe2, 0xe7, 0x43, 0x42, + 0x77, 0xc9, 0x82, 0x00, 0xac, 0x85, 0xe0, 0xa2, + 0xd4, 0x62, 0xe3, 0xb7, 0x17, 0x6e, 0xb2, 0x9e, + 0x21, 0x58, 0x73, 0xa9, 0x53, 0x2d, 0x3c, 0xe1, + 0xdd, 0xd6, 0x6e, 0x92, 0xf2, 0x1d, 0xc2, 0x22, + 0x5f, 0x9a, 0x7e, 0xd0, 0x52, 0xbf, 0x54, 0x19, + 0xd7, 0x80, 0x63, 0x3e, 0xd0, 0x08, 0x2d, 0x37, + 0x0c, 0x15, 0xf7, 0xde, 0xab, 0x2b, 0xe3, 0x16, + 0x21, 0x3a, 0xee, 0xa5, 0xdc, 0xdf, 0xde, 0xa3, + 0x69, 0xcb, 0xfd, 0x92, 0x89, 0x75, 0xcf, 0xc9, + 0x8a, 0xa4, 0xc8, 0xdd, 0xcc, 0x21, 0xe6, 0xfe, + 0x9e, 0x43, 0x76, 0xb2, 0x45, 0x22, 0xb9, 0xb5, + 0xac, 0x7e, 0x3d, 0x26, 0xb0, 0x53, 0xc8, 0xab, + 0xfd, 0xea, 0x2c, 0xd1, 0x44, 0xc5, 0x60, 0x1b, + 0x8a, 0x99, 0x0d, 0xa5, 0x0e, 0x67, 0x6e, 0x3a, + 0x96, 0x55, 0xec, 0xe8, 0xcc, 0xbe, 0x49, 0xd9, + 0xf2, 0x72, 0x9f, 0x30, 0x21, 0x97, 0x57, 0x19, + 0xbe, 0x5e, 0x33, 0x0c, 0xee, 0xc0, 0x72, 0x0d, + 0x2e, 0xd1, 0xe1, 0x52, 0xc2, 0xea, 0x41, 0xbb, + 0xe1, 0x6d, 0xd4, 0x17, 0xa9, 0x8d, 0x89, 0xa9, + 0xd6, 0x4b, 0xc6, 0x4c, 0xf2, 0x88, 0x97, 0x54, + 0x3f, 0x4f, 0x57, 0xb7, 0x37, 0xf0, 0x2c, 0x11, + 0x15, 0x56, 0xdb, 0x28, 0xb5, 0x16, 0x84, 0x66, + 0xce, 0x45, 0x3f, 0x61, 0x75, 0xb6, 0xbe, 0x00, + 0xd1, 0xe4, 0xf5, 0x27, 0x54, 0x7f, 0xc2, 0xf1, + 0xb3, 0x32, 0x9a, 0xe8, 0x07, 0x02, 0xf3, 0xdb, + 0xa9, 0xd1, 0xc2, 0xdf, 0xee, 0xad, 0xe5, 0x8a, + 0x3c, 0xfa, 0x67, 0xec, 0x6b, 0xa4, 0x08, 0xfe, + 0xba, 0x5a, 0x58, 0x0b, 0x78, 0x11, 0x91, 0x76, + 0xe3, 0x1a, 0x28, 0x54, 0x5e, 0xbd, 0x71, 0x1b, + 0x8b, 0xdc, 0x6c, 0xf4, 0x6f, 0xd7, 0xf4, 0xf3, + 0xe1, 0x03, 0xa4, 0x3c, 0x8d, 0x91, 0x2e, 0xba, + 0x5f, 0x7f, 0x8c, 0xaf, 0x69, 0x89, 0x29, 0x0a, + 0x5b, 0x25, 0x13, 0xc4, 0x2e, 0x16, 0xc2, 0x15, + 0x07, 0x5d, 0x58, 0x33, 0x7c, 0xe0, 0xf0, 0x55, + 0x5f, 0xbf, 0x5e, 0xf0, 0x71, 0x48, 0x8f, 0xf7, + 0x48, 0xb3, 0xf7, 0x0d, 0xa1, 0xd0, 0x63, 0xb1, + 0xad, 0xae, 0xb5, 0xb0, 0x5f, 0x71, 0xaf, 0x24, + 0x8b, 0xb9, 0x1c, 0x44, 0xd2, 0x1a, 0x53, 0xd1, + 0xd5, 0xb4, 0xa9, 0xff, 0x88, 0x73, 0xb5, 0xaa, + 0x15, 0x32, 0x5f, 0x59, 0x9d, 0x2e, 0xb5, 0xcb, + 0xde, 0x21, 0x2e, 0xe9, 0x35, 0xed, 0xfd, 0x0f, + 0xb6, 0xbb, 0xe6, 0x4b, 0x16, 0xf1, 0x45, 0x1e, + 0xb4, 0x84, 0xe9, 0x58, 0x1c, 0x0c, 0x95, 0xc0, + 0xcf, 0x49, 0x8b, 0x59, 0xa1, 0x78, 0xe6, 0x80, + 0x12, 0x49, 0x7a, 0xd4, 0x66, 0x62, 0xdf, 0x9c, + 0x18, 0xc8, 0x8c, 0xda, 0xc1, 0xa6, 0xbc, 0x65, + 0x28, 0xd2, 0xa4, 0xe8, 0xf1, 0x35, 0xdb, 0x5a, + 0x75, 0x1f, 0x73, 0x60, 0xec, 0xa8, 0xda, 0x5a, + 0x43, 0x15, 0x83, 0x9b, 0xe7, 0xb1, 0xa6, 0x81, + 0xbb, 0xef, 0xf3, 0x8f, 0x0f, 0xd3, 0x79, 0xa2, + 0xe5, 0xaa, 0x42, 0xef, 0xa0, 0x13, 0x4e, 0x91, + 0x2d, 0xcb, 0x61, 0x7a, 0x9a, 0x33, 0x14, 0x50, + 0x77, 0x4a, 0xd0, 0x91, 0x48, 0xe0, 0x0c, 0xe0, + 0x11, 0xcb, 0xdf, 0xb0, 0xce, 0x06, 0xd2, 0x79, + 0x4d, 0x69, 0xb9, 0xc9, 0x36, 0x74, 0x8f, 0x81, + 0x72, 0x73, 0xf3, 0x17, 0xb7, 0x13, 0xcb, 0x5b, + 0xd2, 0x5c, 0x33, 0x61, 0xb7, 0x61, 0x79, 0xb0, + 0xc0, 0x4d, 0xa1, 0xc7, 0x5d, 0x98, 0xc9, 0xe1, + 0x98, 0xbd, 0x78, 0x5a, 0x2c, 0x64, 0x53, 0xaf, + 0xaf, 0x66, 0x51, 0x47, 0xe4, 0x48, 0x66, 0x8b, + 0x07, 0x52, 0xa3, 0x03, 0x93, 0x28, 0xad, 0xcc, + 0xa3, 0x86, 0xad, 0x63, 0x04, 0x35, 0x6c, 0x49, + 0xd5, 0x28, 0x0e, 0x00, 0x47, 0xf4, 0xd4, 0x32, + 0x27, 0x19, 0xb3, 0x29, 0xe7, 0xbc, 0xbb, 0xce, + 0x3e, 0x3e, 0xd5, 0x67, 0x20, 0xe4, 0x0b, 0x75, + 0x95, 0x24, 0xe0, 0x6c, 0xb6, 0x29, 0x0c, 0x14, + 0xfd +}; +static const u8 key74[] __initconst = { + 0xf0, 0x41, 0x5b, 0x00, 0x56, 0xc4, 0xac, 0xf6, + 0xa2, 0x4c, 0x33, 0x41, 0x16, 0x09, 0x1b, 0x8e, + 0x4d, 0xe8, 0x8c, 0xd9, 0x48, 0xab, 0x3e, 0x60, + 0xcb, 0x49, 0x3e, 0xaf, 0x2b, 0x8b, 0xc8, 0xf0 +}; +enum { nonce74 = 0xcbdb0ffd0e923384ULL }; + +static const struct chacha20_testvec chacha20_testvecs[] __initconst = { + { input01, output01, key01, nonce01, sizeof(input01) }, + { input02, output02, key02, nonce02, sizeof(input02) }, + { input03, output03, key03, nonce03, sizeof(input03) }, + { input04, output04, key04, nonce04, sizeof(input04) }, + { input05, output05, key05, nonce05, sizeof(input05) }, + { input06, output06, key06, nonce06, sizeof(input06) }, + { input07, output07, key07, nonce07, sizeof(input07) }, + { input08, output08, key08, nonce08, sizeof(input08) }, + { input09, output09, key09, nonce09, sizeof(input09) }, + { input10, output10, key10, nonce10, sizeof(input10) }, + { input11, output11, key11, nonce11, sizeof(input11) }, + { input12, output12, key12, nonce12, sizeof(input12) }, + { input13, output13, key13, nonce13, sizeof(input13) }, + { input14, output14, key14, nonce14, sizeof(input14) }, + { input15, output15, key15, nonce15, sizeof(input15) }, + { input16, output16, key16, nonce16, sizeof(input16) }, + { input17, output17, key17, nonce17, sizeof(input17) }, + { input18, output18, key18, nonce18, sizeof(input18) }, + { input19, output19, key19, nonce19, sizeof(input19) }, + { input20, output20, key20, nonce20, sizeof(input20) }, + { input21, output21, key21, nonce21, sizeof(input21) }, + { input22, output22, key22, nonce22, sizeof(input22) }, + { input23, output23, key23, nonce23, sizeof(input23) }, + { input24, output24, key24, nonce24, sizeof(input24) }, + { input25, output25, key25, nonce25, sizeof(input25) }, + { input26, output26, key26, nonce26, sizeof(input26) }, + { input27, output27, key27, nonce27, sizeof(input27) }, + { input28, output28, key28, nonce28, sizeof(input28) }, + { input29, output29, key29, nonce29, sizeof(input29) }, + { input30, output30, key30, nonce30, sizeof(input30) }, + { input31, output31, key31, nonce31, sizeof(input31) }, + { input32, output32, key32, nonce32, sizeof(input32) }, + { input33, output33, key33, nonce33, sizeof(input33) }, + { input34, output34, key34, nonce34, sizeof(input34) }, + { input35, output35, key35, nonce35, sizeof(input35) }, + { input36, output36, key36, nonce36, sizeof(input36) }, + { input37, output37, key37, nonce37, sizeof(input37) }, + { input38, output38, key38, nonce38, sizeof(input38) }, + { input39, output39, key39, nonce39, sizeof(input39) }, + { input40, output40, key40, nonce40, sizeof(input40) }, + { input41, output41, key41, nonce41, sizeof(input41) }, + { input42, output42, key42, nonce42, sizeof(input42) }, + { input43, output43, key43, nonce43, sizeof(input43) }, + { input44, output44, key44, nonce44, sizeof(input44) }, + { input45, output45, key45, nonce45, sizeof(input45) }, + { input46, output46, key46, nonce46, sizeof(input46) }, + { input47, output47, key47, nonce47, sizeof(input47) }, + { input48, output48, key48, nonce48, sizeof(input48) }, + { input49, output49, key49, nonce49, sizeof(input49) }, + { input50, output50, key50, nonce50, sizeof(input50) }, + { input51, output51, key51, nonce51, sizeof(input51) }, + { input52, output52, key52, nonce52, sizeof(input52) }, + { input53, output53, key53, nonce53, sizeof(input53) }, + { input54, output54, key54, nonce54, sizeof(input54) }, + { input55, output55, key55, nonce55, sizeof(input55) }, + { input56, output56, key56, nonce56, sizeof(input56) }, + { input57, output57, key57, nonce57, sizeof(input57) }, + { input58, output58, key58, nonce58, sizeof(input58) }, + { input59, output59, key59, nonce59, sizeof(input59) }, + { input60, output60, key60, nonce60, sizeof(input60) }, + { input61, output61, key61, nonce61, sizeof(input61) }, + { input62, output62, key62, nonce62, sizeof(input62) }, + { input63, output63, key63, nonce63, sizeof(input63) }, + { input64, output64, key64, nonce64, sizeof(input64) }, + { input65, output65, key65, nonce65, sizeof(input65) }, + { input66, output66, key66, nonce66, sizeof(input66) }, + { input67, output67, key67, nonce67, sizeof(input67) }, + { input68, output68, key68, nonce68, sizeof(input68) }, + { input69, output69, key69, nonce69, sizeof(input69) }, + { input70, output70, key70, nonce70, sizeof(input70) }, + { input71, output71, key71, nonce71, sizeof(input71) }, + { input72, output72, key72, nonce72, sizeof(input72) }, + { input73, output73, key73, nonce73, sizeof(input73) }, + { input74, output74, key74, nonce74, sizeof(input74) } +}; + +static const struct hchacha20_testvec hchacha20_testvecs[] __initconst = {{ + .key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + .nonce = { 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00, 0x31, 0x41, 0x59, 0x27 }, + .output = { 0x82, 0x41, 0x3b, 0x42, 0x27, 0xb2, 0x7b, 0xfe, + 0xd3, 0x0e, 0x42, 0x50, 0x8a, 0x87, 0x7d, 0x73, + 0xa0, 0xf9, 0xe4, 0xd5, 0x8a, 0x74, 0xa8, 0x53, + 0xc1, 0x2e, 0xc4, 0x13, 0x26, 0xd3, 0xec, 0xdc } +}}; + +static bool __init chacha20_selftest(void) +{ + enum { + MAXIMUM_TEST_BUFFER_LEN = 1UL << 10, + OUTRAGEOUSLY_HUGE_BUFFER_LEN = PAGE_SIZE * 35 + 17 /* 143k */ + }; + size_t i, j, k; + u32 derived_key[CHACHA20_KEY_WORDS]; + u8 *offset_input = NULL, *computed_output = NULL, *massive_input = NULL; + u8 offset_key[CHACHA20_KEY_SIZE + 1] + __aligned(__alignof__(unsigned long)); + struct chacha20_ctx state; + bool success = true; + simd_context_t simd_context; + + offset_input = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + massive_input = vzalloc(OUTRAGEOUSLY_HUGE_BUFFER_LEN); + if (!computed_output || !offset_input || !massive_input) { + pr_err("chacha20 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20_testvecs); ++i) { + /* Boring case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Unaligned case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + 1, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + memcpy(offset_key + 1, chacha20_testvecs[i].key, + CHACHA20_KEY_SIZE); + chacha20_init(&state, offset_key + 1, chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + 1, offset_input + 1, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + 1, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned): FAIL\n", + i + 1); + success = false; + } + if (computed_output[0]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen + 1; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Chunked case */ + if (chacha20_testvecs[i].ilen <= CHACHA20_BLOCK_SIZE) + goto next_test; + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + CHACHA20_BLOCK_SIZE, &simd_context); + chacha20(&state, computed_output + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].input + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].ilen - CHACHA20_BLOCK_SIZE, + &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (chunked): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (chunked, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + +next_test: + /* Sliding unaligned case */ + if (chacha20_testvecs[i].ilen > CHACHA20_BLOCK_SIZE + 1 || + !chacha20_testvecs[i].ilen) + continue; + for (j = 1; j < CHACHA20_BLOCK_SIZE; ++j) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + j, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + j, offset_input + j, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + j, + chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu): FAIL\n", + i + 1, j); + success = false; + } + for (k = j; k < j; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + for (k = chacha20_testvecs[i].ilen + j; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + } + } + for (i = 0; i < ARRAY_SIZE(hchacha20_testvecs); ++i) { + memset(&derived_key, 0, sizeof(derived_key)); + hchacha20(derived_key, hchacha20_testvecs[i].nonce, + hchacha20_testvecs[i].key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + if (memcmp(derived_key, hchacha20_testvecs[i].output, + CHACHA20_KEY_SIZE)) { + pr_err("hchacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, &simd_context); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, DONT_USE_SIMD); + for (k = 0; k < OUTRAGEOUSLY_HUGE_BUFFER_LEN; ++k) { + if (massive_input[k]) { + pr_err("chacha20 self-test massive: FAIL\n"); + success = false; + break; + } + } + + simd_put(&simd_context); + +out: + kfree(offset_input); + kfree(computed_output); + vfree(massive_input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c new file mode 100644 index 0000000000000..c58ac6e69d541 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c @@ -0,0 +1,9076 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20poly1305_testvec { + const u8 *input, *output, *assoc, *nonce, *key; + size_t ilen, alen, nlen; + bool failure; +}; + +/* The first of these are the ChaCha20-Poly1305 AEAD test vectors from RFC7539 + * 2.8.2. After they are generated by reference implementations. And the final + * marked ones are taken from wycheproof, but we only do these for the encrypt + * side, because mostly we're stressing the primitives rather than the actual + * chapoly construction. This also requires adding a 96-bit nonce construction, + * just for the purpose of the tests. + */ + +static const u8 enc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 enc_output001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 enc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 enc_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 enc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 enc_input002[] __initconst = { }; +static const u8 enc_output002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 enc_assoc002[] __initconst = { }; +static const u8 enc_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 enc_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 enc_input003[] __initconst = { }; +static const u8 enc_output003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 enc_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 enc_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 enc_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 enc_input004[] __initconst = { + 0xa4 +}; +static const u8 enc_output004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 enc_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 enc_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 enc_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 enc_input005[] __initconst = { + 0x2d +}; +static const u8 enc_output005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 enc_assoc005[] __initconst = { }; +static const u8 enc_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 enc_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 enc_input006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 enc_output006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 enc_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 enc_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 enc_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 enc_input007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 enc_output007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 enc_assoc007[] __initconst = { }; +static const u8 enc_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 enc_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 enc_input008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 enc_output008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 enc_assoc008[] __initconst = { }; +static const u8 enc_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 enc_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 enc_input009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 enc_output009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 enc_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 enc_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 enc_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 enc_input010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 enc_output010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 enc_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 enc_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 enc_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 enc_input011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 enc_output011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 enc_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 enc_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 enc_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 enc_input012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 enc_output012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 enc_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 enc_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 enc_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + +/* wycheproof - misc */ +static const u8 enc_input053[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, + 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 +}; +static const u8 enc_assoc053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key053[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input054[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, + 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce +}; +static const u8 enc_assoc054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key054[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input055[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd, + 0x7a, 0xda, 0x44, 0x42, 0x42, 0x69, 0xbf, 0xfa, + 0x55, 0x27, 0xf2, 0x70, 0xac, 0xf6, 0x85, 0x02, + 0xb7, 0x4c, 0x5a, 0xe2, 0xe6, 0x0c, 0x05, 0x80, + 0x98, 0x1a, 0x49, 0x38, 0x45, 0x93, 0x92, 0xc4, + 0x9b, 0xb2, 0xf2, 0x84, 0xb6, 0x46, 0xef, 0xc7, + 0xf3, 0xf0, 0xb1, 0x36, 0x1d, 0xc3, 0x48, 0xed, + 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, + 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, + 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e +}; +static const u8 enc_assoc055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key055[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input056[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, + 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 +}; +static const u8 enc_assoc056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce056[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key056[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input057[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, + 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 +}; +static const u8 enc_assoc057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce057[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key057[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input058[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42, + 0x85, 0x25, 0xbb, 0xbd, 0xbd, 0x96, 0x40, 0x05, + 0xaa, 0xd8, 0x0d, 0x8f, 0x53, 0x09, 0x7a, 0xfd, + 0x48, 0xb3, 0xa5, 0x1d, 0x19, 0xf3, 0xfa, 0x7f, + 0x67, 0xe5, 0xb6, 0xc7, 0xba, 0x6c, 0x6d, 0x3b, + 0x64, 0x4d, 0x0d, 0x7b, 0x49, 0xb9, 0x10, 0x38, + 0x0c, 0x0f, 0x4e, 0xc9, 0xe2, 0x3c, 0xb7, 0x12, + 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, + 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, + 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 +}; +static const u8 enc_assoc058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce058[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key058[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input059[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e +}; +static const u8 enc_output059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, + 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 +}; +static const u8 enc_assoc059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce059[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key059[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input060[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d +}; +static const u8 enc_output060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, + 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a +}; +static const u8 enc_assoc060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce060[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key060[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input061[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d, + 0x7a, 0xda, 0x44, 0xc2, 0x42, 0x69, 0xbf, 0x7a, + 0x55, 0x27, 0xf2, 0xf0, 0xac, 0xf6, 0x85, 0x82, + 0xb7, 0x4c, 0x5a, 0x62, 0xe6, 0x0c, 0x05, 0x00, + 0x98, 0x1a, 0x49, 0xb8, 0x45, 0x93, 0x92, 0x44, + 0x9b, 0xb2, 0xf2, 0x04, 0xb6, 0x46, 0xef, 0x47, + 0xf3, 0xf0, 0xb1, 0xb6, 0x1d, 0xc3, 0x48, 0x6d, + 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, + 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 +}; +static const u8 enc_output061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, + 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a +}; +static const u8 enc_assoc061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce061[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key061[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input062[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 +}; +static const u8 enc_output062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, + 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 +}; +static const u8 enc_assoc062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce062[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key062[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input063[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 +}; +static const u8 enc_output063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, + 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c +}; +static const u8 enc_assoc063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce063[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key063[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input064[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2, + 0x85, 0x25, 0xbb, 0x3d, 0xbd, 0x96, 0x40, 0x85, + 0xaa, 0xd8, 0x0d, 0x0f, 0x53, 0x09, 0x7a, 0x7d, + 0x48, 0xb3, 0xa5, 0x9d, 0x19, 0xf3, 0xfa, 0xff, + 0x67, 0xe5, 0xb6, 0x47, 0xba, 0x6c, 0x6d, 0xbb, + 0x64, 0x4d, 0x0d, 0xfb, 0x49, 0xb9, 0x10, 0xb8, + 0x0c, 0x0f, 0x4e, 0x49, 0xe2, 0x3c, 0xb7, 0x92, + 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, + 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 +}; +static const u8 enc_output064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, + 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 +}; +static const u8 enc_assoc064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce064[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key064[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input065[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, + 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 +}; +static const u8 enc_assoc065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce065[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key065[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input066[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 +}; +static const u8 enc_output066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, + 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 +}; +static const u8 enc_assoc066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce066[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key066[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input067[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42, + 0x05, 0x25, 0xbb, 0xbd, 0x3d, 0x96, 0x40, 0x05, + 0x2a, 0xd8, 0x0d, 0x8f, 0xd3, 0x09, 0x7a, 0xfd, + 0xc8, 0xb3, 0xa5, 0x1d, 0x99, 0xf3, 0xfa, 0x7f, + 0xe7, 0xe5, 0xb6, 0xc7, 0x3a, 0x6c, 0x6d, 0x3b, + 0xe4, 0x4d, 0x0d, 0x7b, 0xc9, 0xb9, 0x10, 0x38, + 0x8c, 0x0f, 0x4e, 0xc9, 0x62, 0x3c, 0xb7, 0x12, + 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, + 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, + 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 +}; +static const u8 enc_assoc067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce067[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key067[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input068[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, + 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d +}; +static const u8 enc_assoc068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key068[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input069[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, + 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 +}; +static const u8 enc_assoc069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key069[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input070[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42, + 0x7a, 0xda, 0x44, 0x42, 0xbd, 0x96, 0x40, 0x05, + 0x55, 0x27, 0xf2, 0x70, 0x53, 0x09, 0x7a, 0xfd, + 0xb7, 0x4c, 0x5a, 0xe2, 0x19, 0xf3, 0xfa, 0x7f, + 0x98, 0x1a, 0x49, 0x38, 0xba, 0x6c, 0x6d, 0x3b, + 0x9b, 0xb2, 0xf2, 0x84, 0x49, 0xb9, 0x10, 0x38, + 0xf3, 0xf0, 0xb1, 0x36, 0xe2, 0x3c, 0xb7, 0x12, + 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, + 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, + 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 +}; +static const u8 enc_assoc070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key070[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input071[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, + 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 +}; +static const u8 enc_assoc071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce071[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key071[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input072[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, + 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 +}; +static const u8 enc_assoc072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce072[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key072[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input073[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd, + 0x85, 0x25, 0xbb, 0xbd, 0x42, 0x69, 0xbf, 0xfa, + 0xaa, 0xd8, 0x0d, 0x8f, 0xac, 0xf6, 0x85, 0x02, + 0x48, 0xb3, 0xa5, 0x1d, 0xe6, 0x0c, 0x05, 0x80, + 0x67, 0xe5, 0xb6, 0xc7, 0x45, 0x93, 0x92, 0xc4, + 0x64, 0x4d, 0x0d, 0x7b, 0xb6, 0x46, 0xef, 0xc7, + 0x0c, 0x0f, 0x4e, 0xc9, 0x1d, 0xc3, 0x48, 0xed, + 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, + 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, + 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f +}; +static const u8 enc_assoc073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce073[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key073[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input076[] __initconst = { + 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, + 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, + 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, + 0x52, 0x2c, 0xdd, 0x12, 0x06, 0xd2, 0x20, 0xb8, + 0xf8, 0xbd, 0x07, 0xd1, 0xf1, 0xf5, 0xa1, 0xbd, + 0x9a, 0x71, 0xd1, 0x1c, 0x7f, 0x57, 0x9b, 0x85, + 0x58, 0x18, 0xc0, 0x8d, 0x4d, 0xe0, 0x36, 0x39, + 0x31, 0x83, 0xb7, 0xf5, 0x90, 0xb3, 0x35, 0xae, + 0xd8, 0xde, 0x5b, 0x57, 0xb1, 0x3c, 0x5f, 0xed, + 0xe2, 0x44, 0x1c, 0x3e, 0x18, 0x4a, 0xa9, 0xd4, + 0x6e, 0x61, 0x59, 0x85, 0x06, 0xb3, 0xe1, 0x1c, + 0x43, 0xc6, 0x2c, 0xbc, 0xac, 0xec, 0xed, 0x33, + 0x19, 0x08, 0x75, 0xb0, 0x12, 0x21, 0x8b, 0x19, + 0x30, 0xfb, 0x7c, 0x38, 0xec, 0x45, 0xac, 0x11, + 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, + 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 +}; +static const u8 enc_output076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, + 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 +}; +static const u8 enc_assoc076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce076[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 +}; +static const u8 enc_key076[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input077[] __initconst = { + 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, + 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, + 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, + 0x4f, 0x01, 0x46, 0x49, 0x95, 0xa6, 0x9e, 0x61, + 0x76, 0xcb, 0x9e, 0x02, 0xb2, 0x24, 0x7e, 0xd2, + 0x99, 0x89, 0x2f, 0x91, 0x82, 0xa4, 0x5c, 0xaf, + 0x4c, 0x69, 0x40, 0x56, 0x11, 0x76, 0x6e, 0xdf, + 0xaf, 0xdc, 0x28, 0x55, 0x19, 0xea, 0x30, 0x48, + 0x0c, 0x44, 0xf0, 0x5e, 0x78, 0x1e, 0xac, 0xf8, + 0xfc, 0xec, 0xc7, 0x09, 0x0a, 0xbb, 0x28, 0xfa, + 0x5f, 0xd5, 0x85, 0xac, 0x8c, 0xda, 0x7e, 0x87, + 0x72, 0xe5, 0x94, 0xe4, 0xce, 0x6c, 0x88, 0x32, + 0x81, 0x93, 0x2e, 0x0f, 0x89, 0xf8, 0x77, 0xa1, + 0xf0, 0x4d, 0x9c, 0x32, 0xb0, 0x6c, 0xf9, 0x0b, + 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, + 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f +}; +static const u8 enc_output077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, + 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa +}; +static const u8 enc_assoc077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce077[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 +}; +static const u8 enc_key077[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input078[] __initconst = { + 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, + 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, + 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, + 0x0b, 0x96, 0xfb, 0xa0, 0x87, 0x1e, 0x93, 0x2d, + 0xb1, 0xfb, 0xf9, 0x07, 0x61, 0xbe, 0x25, 0xdf, + 0x8d, 0xfa, 0xf9, 0x31, 0xce, 0x57, 0x57, 0xe6, + 0x17, 0xb3, 0xd7, 0xa9, 0xf0, 0xbf, 0x0f, 0xfe, + 0x5d, 0x59, 0x1a, 0x33, 0xc1, 0x43, 0xb8, 0xf5, + 0x3f, 0xd0, 0xb5, 0xa1, 0x96, 0x09, 0xfd, 0x62, + 0xe5, 0xc2, 0x51, 0xa4, 0x28, 0x1a, 0x20, 0x0c, + 0xfd, 0xc3, 0x4f, 0x28, 0x17, 0x10, 0x40, 0x6f, + 0x4e, 0x37, 0x62, 0x54, 0x46, 0xff, 0x6e, 0xf2, + 0x24, 0x91, 0x3d, 0xeb, 0x0d, 0x89, 0xaf, 0x33, + 0x71, 0x28, 0xe3, 0xd1, 0x55, 0xd1, 0x6d, 0x3e, + 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, + 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 +}; +static const u8 enc_output078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, + 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d +}; +static const u8 enc_assoc078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce078[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 +}; +static const u8 enc_key078[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input079[] __initconst = { + 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, + 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, + 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, + 0xb4, 0x0d, 0x50, 0x4e, 0x6c, 0x94, 0x9f, 0x82, + 0xd1, 0x97, 0xc2, 0x5a, 0xd1, 0x7d, 0xc7, 0x21, + 0x65, 0x11, 0x25, 0x78, 0x2a, 0xc7, 0xa7, 0x12, + 0x47, 0xfe, 0xae, 0xf3, 0x2f, 0x1f, 0x25, 0x0c, + 0xe4, 0xbb, 0x8f, 0x79, 0xac, 0xaa, 0x17, 0x9d, + 0x45, 0xa7, 0xb0, 0x54, 0x5f, 0x09, 0x24, 0x32, + 0x5e, 0xfa, 0x87, 0xd5, 0xe4, 0x41, 0xd2, 0x84, + 0x78, 0xc6, 0x1f, 0x22, 0x23, 0xee, 0x67, 0xc3, + 0xb4, 0x1f, 0x43, 0x94, 0x53, 0x5e, 0x2a, 0x24, + 0x36, 0x9a, 0x2e, 0x16, 0x61, 0x3c, 0x45, 0x94, + 0x90, 0xc1, 0x4f, 0xb1, 0xd7, 0x55, 0xfe, 0x53, + 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, + 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd +}; +static const u8 enc_output079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, + 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e +}; +static const u8 enc_assoc079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce079[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a +}; +static const u8 enc_key079[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input080[] __initconst = { + 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, + 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, + 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, + 0x98, 0xd6, 0x69, 0xa8, 0xe6, 0x0e, 0x04, 0xfc, + 0x08, 0xd5, 0xfd, 0x06, 0x9c, 0x36, 0x26, 0x38, + 0xe3, 0x40, 0x0e, 0xf4, 0xcb, 0x24, 0x2e, 0x27, + 0xe2, 0x24, 0x5e, 0x68, 0xcb, 0x9e, 0xc5, 0x83, + 0xda, 0x53, 0x40, 0xb1, 0x2e, 0xdf, 0x42, 0x3b, + 0x73, 0x26, 0xad, 0x20, 0xfe, 0xeb, 0x57, 0xda, + 0xca, 0x2e, 0x04, 0x67, 0xa3, 0x28, 0x99, 0xb4, + 0x2d, 0xf8, 0xe5, 0x6d, 0x84, 0xe0, 0x06, 0xbc, + 0x8a, 0x7a, 0xcc, 0x73, 0x1e, 0x7c, 0x1f, 0x6b, + 0xec, 0xb5, 0x71, 0x9f, 0x70, 0x77, 0xf0, 0xd4, + 0xf4, 0xc6, 0x1a, 0xb1, 0x1e, 0xba, 0xc1, 0x00, + 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, + 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 +}; +static const u8 enc_output080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, + 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 +}; +static const u8 enc_assoc080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce080[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 +}; +static const u8 enc_key080[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input081[] __initconst = { + 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, + 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, + 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, + 0x0b, 0x2a, 0xfa, 0xad, 0x5e, 0xba, 0x35, 0x06, + 0x99, 0x25, 0xa6, 0x03, 0xfd, 0xbc, 0x34, 0x1a, + 0xae, 0xd4, 0x15, 0x05, 0xb1, 0x09, 0x41, 0xfa, + 0x38, 0x56, 0xa7, 0xe2, 0x47, 0xb1, 0x04, 0x07, + 0x09, 0x74, 0x6c, 0xfc, 0x20, 0x96, 0xca, 0xa6, + 0x31, 0xb2, 0xff, 0xf4, 0x1c, 0x25, 0x05, 0x06, + 0xd8, 0x89, 0xc1, 0xc9, 0x06, 0x71, 0xad, 0xe8, + 0x53, 0xee, 0x63, 0x94, 0xc1, 0x91, 0x92, 0xa5, + 0xcf, 0x37, 0x10, 0xd1, 0x07, 0x30, 0x99, 0xe5, + 0xbc, 0x94, 0x65, 0x82, 0xfc, 0x0f, 0xab, 0x9f, + 0x54, 0x3c, 0x71, 0x6a, 0xe2, 0x48, 0x6a, 0x86, + 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, + 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 +}; +static const u8 enc_output081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, + 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 +}; +static const u8 enc_assoc081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce081[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 +}; +static const u8 enc_key081[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input082[] __initconst = { + 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, + 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, + 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, + 0x6e, 0xa6, 0x5f, 0x81, 0xb5, 0x75, 0x74, 0x7d, + 0x03, 0x5b, 0x30, 0x40, 0xfe, 0xde, 0x1e, 0xb9, + 0x45, 0x97, 0x88, 0x66, 0x97, 0x88, 0x40, 0x8e, + 0x00, 0x41, 0x3b, 0x3e, 0x37, 0x6d, 0x15, 0x2d, + 0x20, 0x4a, 0xa2, 0xb7, 0xa8, 0x35, 0x58, 0xfc, + 0xd4, 0x8a, 0x0e, 0xf7, 0xa2, 0x6b, 0x1c, 0xd6, + 0xd3, 0x5d, 0x23, 0xb3, 0xf5, 0xdf, 0xe0, 0xca, + 0x77, 0xa4, 0xce, 0x32, 0xb9, 0x4a, 0xbf, 0x83, + 0xda, 0x2a, 0xef, 0xca, 0xf0, 0x68, 0x38, 0x08, + 0x79, 0xe8, 0x9f, 0xb0, 0xa3, 0x82, 0x95, 0x95, + 0xcf, 0x44, 0xc3, 0x85, 0x2a, 0xe2, 0xcc, 0x66, + 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, + 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 +}; +static const u8 enc_output082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, + 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae +}; +static const u8 enc_assoc082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce082[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 +}; +static const u8 enc_key082[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input083[] __initconst = { + 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, + 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, + 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, + 0xa9, 0x01, 0x58, 0x10, 0x85, 0x96, 0x44, 0xdf, + 0xc0, 0x21, 0x17, 0x4e, 0x0b, 0x06, 0x0a, 0x39, + 0x74, 0x48, 0xde, 0x8b, 0x48, 0x4a, 0x86, 0x03, + 0xbe, 0x68, 0x0a, 0x69, 0x34, 0xc0, 0x90, 0x6f, + 0x30, 0xdd, 0x17, 0xea, 0xe2, 0xd4, 0xc5, 0xfa, + 0xa7, 0x77, 0xf8, 0xca, 0x53, 0x37, 0x0e, 0x08, + 0x33, 0x1b, 0x88, 0xc3, 0x42, 0xba, 0xc9, 0x59, + 0x78, 0x7b, 0xbb, 0x33, 0x93, 0x0e, 0x3b, 0x56, + 0xbe, 0x86, 0xda, 0x7f, 0x2a, 0x6e, 0xb1, 0xf9, + 0x40, 0x89, 0xd1, 0xd1, 0x81, 0x07, 0x4d, 0x43, + 0x02, 0xf8, 0xe0, 0x55, 0x2d, 0x0d, 0xe1, 0xfa, + 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, + 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a +}; +static const u8 enc_output083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, + 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 +}; +static const u8 enc_assoc083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce083[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 +}; +static const u8 enc_key083[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input084[] __initconst = { + 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, + 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, + 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, + 0x4b, 0x72, 0x48, 0x4e, 0x79, 0x91, 0xd2, 0x5c, + 0x29, 0xaa, 0x07, 0x5e, 0xb1, 0xfc, 0x16, 0xde, + 0x93, 0xfe, 0x06, 0x90, 0x58, 0x11, 0x2a, 0xb2, + 0x84, 0xa3, 0xed, 0x18, 0x78, 0x03, 0x26, 0xd1, + 0x25, 0x8a, 0x47, 0x22, 0x2f, 0xa6, 0x33, 0xd8, + 0xb2, 0x9f, 0x3b, 0xd9, 0x15, 0x0b, 0x23, 0x9b, + 0x15, 0x46, 0xc2, 0xbb, 0x9b, 0x9f, 0x41, 0x0f, + 0xeb, 0xea, 0xd3, 0x96, 0x00, 0x0e, 0xe4, 0x77, + 0x70, 0x15, 0x32, 0xc3, 0xd0, 0xf5, 0xfb, 0xf8, + 0x95, 0xd2, 0x80, 0x19, 0x6d, 0x2f, 0x73, 0x7c, + 0x5e, 0x9f, 0xec, 0x50, 0xd9, 0x2b, 0xb0, 0xdf, + 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, + 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee +}; +static const u8 enc_output084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, + 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 +}; +static const u8 enc_assoc084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce084[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 +}; +static const u8 enc_key084[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input085[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x9c, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xd4, 0xd2, 0x06, 0x61, 0x6f, 0x92, 0x93, 0xf6, + 0x5b, 0x45, 0xdb, 0xbc, 0x74, 0xe7, 0xc2, 0xed, + 0xfb, 0xcb, 0xbf, 0x1c, 0xfb, 0x67, 0x9b, 0xb7, + 0x39, 0xa5, 0x86, 0x2d, 0xe2, 0xbc, 0xb9, 0x37, + 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, + 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b +}; +static const u8 enc_output085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, + 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 +}; +static const u8 enc_assoc085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce085[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key085[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input093[] __initconst = { + 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, + 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, + 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x5b, 0x8b, 0x94, 0x50, 0x9e, 0x2b, 0x74, 0xa3, + 0x6d, 0x34, 0x6e, 0x33, 0xd5, 0x72, 0x65, 0x9b, + 0xa9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, + 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c +}; +static const u8 enc_output093[] __initconst = { + 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, + 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, + 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a +}; +static const u8 enc_assoc093[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce093[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key093[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input094[] __initconst = { + 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, + 0xc0, 0x31, 0xf7, 0x52, 0x08, 0x50, 0x67, 0x45, + 0xae, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x49, 0x6d, 0xde, 0xb0, 0x55, 0x09, 0xc6, 0xef, + 0xff, 0xab, 0x75, 0xeb, 0x2d, 0xf4, 0xab, 0x09, + 0x76, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, + 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 +}; +static const u8 enc_output094[] __initconst = { + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, + 0x02, 0xf6, 0xe1, 0xd8, 0x34, 0x20, 0x7b, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, + 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc +}; +static const u8 enc_assoc094[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce094[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key094[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input095[] __initconst = { + 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, + 0x3a, 0x09, 0x57, 0xbe, 0x0f, 0xb8, 0x59, 0x46, + 0x80, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd1, 0x76, 0x9f, 0xe8, 0x06, 0xbb, 0xfe, 0xb6, + 0xf5, 0x90, 0x95, 0x0f, 0x2e, 0xac, 0x9e, 0x0a, + 0x58, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, + 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 +}; +static const u8 enc_output095[] __initconst = { + 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, + 0xf8, 0xce, 0x41, 0x34, 0x33, 0xc8, 0x45, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, + 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 +}; +static const u8 enc_assoc095[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce095[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key095[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input096[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, + 0xd4, 0x59, 0x6f, 0xf4, 0xb3, 0x40, 0x2c, 0x5c, + 0x00, 0xb9, 0xbb, 0x53, 0x70, 0x7a, 0xa6, 0x67, + 0xd3, 0x56, 0xfe, 0x50, 0xc7, 0x19, 0x96, 0x94, + 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, + 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 +}; +static const u8 enc_output096[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, + 0x16, 0x9e, 0x79, 0x7e, 0x8f, 0x30, 0x30, 0x1b, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, + 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 +}; +static const u8 enc_assoc096[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce096[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key096[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input097[] __initconst = { + 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, + 0xe3, 0x84, 0x07, 0xde, 0x2e, 0xc7, 0x5a, 0x47, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x2d, 0x2a, 0xf7, 0xcd, 0x6b, 0x08, 0x05, 0x01, + 0xd3, 0x1b, 0xa5, 0x4f, 0xb2, 0xeb, 0x75, 0x96, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, + 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f +}; +static const u8 enc_output097[] __initconst = { + 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, + 0x21, 0x43, 0x11, 0x54, 0x12, 0xb7, 0x46, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, + 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 +}; +static const u8 enc_assoc097[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce097[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key097[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input098[] __initconst = { + 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, + 0x9e, 0xfe, 0xff, 0x78, 0xd7, 0xd8, 0x6b, 0x45, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xc6, 0xf8, 0x07, 0x8c, 0xc8, 0xef, 0x12, 0xa0, + 0xff, 0x65, 0x7d, 0x6d, 0x08, 0xdb, 0x10, 0xb8, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, + 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 +}; +static const u8 enc_output098[] __initconst = { + 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, + 0x5c, 0x39, 0xe9, 0xf2, 0xeb, 0xa8, 0x77, 0x02, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, + 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b +}; +static const u8 enc_assoc098[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce098[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key098[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input099[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, + 0x8d, 0x7d, 0x1c, 0x57, 0x91, 0x92, 0xb6, 0x98, + 0x63, 0x41, 0x44, 0x15, 0xb6, 0x99, 0x68, 0x95, + 0x9a, 0x72, 0x91, 0xb7, 0xa5, 0xaf, 0x13, 0x48, + 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, + 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 +}; +static const u8 enc_output099[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, + 0x4f, 0xba, 0x0a, 0xdd, 0xad, 0xe2, 0xaa, 0xdf, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, + 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 +}; +static const u8 enc_assoc099[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce099[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key099[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input100[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, + 0xb5, 0x77, 0xf4, 0x41, 0x37, 0xd3, 0xd6, 0xbd, + 0x13, 0x35, 0xf5, 0xeb, 0x44, 0x49, 0x40, 0x77, + 0xb2, 0x64, 0x49, 0xa5, 0x4b, 0x6c, 0x7c, 0x75, + 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, + 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 +}; +static const u8 enc_output100[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, + 0x77, 0xb0, 0xe2, 0xcb, 0x0b, 0xa3, 0xca, 0xfa, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, + 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 +}; +static const u8 enc_assoc100[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce100[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key100[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input101[] __initconst = { + 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, + 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, + 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x16, 0xe9, 0x88, 0x4a, 0x11, 0x4f, 0x0e, 0x92, + 0x66, 0xce, 0xa3, 0x88, 0x5f, 0xe3, 0x6b, 0x9f, + 0xd6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, + 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 +}; +static const u8 enc_output101[] __initconst = { + 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, + 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, + 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 +}; +static const u8 enc_assoc101[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce101[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key101[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input102[] __initconst = { + 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, + 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, + 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x0c, 0x61, 0x2c, 0x5e, 0x8d, 0x89, 0xa8, 0x73, + 0xdb, 0xca, 0xad, 0x5b, 0x73, 0x46, 0x42, 0x9b, + 0xc5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, + 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c +}; +static const u8 enc_output102[] __initconst = { + 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, + 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, + 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 +}; +static const u8 enc_assoc102[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce102[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key102[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input103[] __initconst = { + 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, + 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, + 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x6b, 0x83, 0x94, 0x33, 0x09, 0x21, 0x48, 0x6c, + 0xa1, 0x1d, 0x29, 0x1c, 0x3e, 0x97, 0xee, 0x9a, + 0xd1, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, + 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d +}; +static const u8 enc_output103[] __initconst = { + 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, + 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, + 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 +}; +static const u8 enc_assoc103[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce103[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key103[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input104[] __initconst = { + 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, + 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, + 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9a, 0xde, 0x04, 0xe7, 0x5b, 0xb7, 0x01, 0xd9, + 0x66, 0x06, 0x01, 0xb3, 0x47, 0x65, 0xde, 0x98, + 0xd5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, + 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f +}; +static const u8 enc_output104[] __initconst = { + 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, + 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, + 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d +}; +static const u8 enc_assoc104[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce104[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key104[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input105[] __initconst = { + 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, + 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, + 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x41, 0x2d, 0x96, 0xaf, 0xbe, 0x80, 0xec, 0x3e, + 0x79, 0xd4, 0x51, 0xb0, 0x0a, 0x2d, 0xb2, 0x9a, + 0xc9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, + 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d +}; +static const u8 enc_output105[] __initconst = { + 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, + 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, + 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd +}; +static const u8 enc_assoc105[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce105[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key105[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input106[] __initconst = { + 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, + 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, + 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x17, 0x7c, 0x5f, 0xfe, 0x28, 0x75, 0xf4, 0x68, + 0xf6, 0xc2, 0x96, 0x57, 0x48, 0xf3, 0x59, 0x9a, + 0xd3, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, + 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d +}; +static const u8 enc_output106[] __initconst = { + 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, + 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, + 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a +}; +static const u8 enc_assoc106[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce106[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key106[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input107[] __initconst = { + 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, + 0x9b, 0xce, 0x9a, 0xbe, 0x27, 0x3a, 0xf1, 0x44, + 0xb4, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x48, 0x02, 0x5f, 0x41, 0xfa, 0x4e, 0x33, 0x6c, + 0x78, 0x69, 0x57, 0xa2, 0xa7, 0xc4, 0x93, 0x0a, + 0x6c, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, + 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 +}; +static const u8 enc_output107[] __initconst = { + 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, + 0x59, 0x09, 0x8c, 0x34, 0x1b, 0x4a, 0xed, 0x03, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, + 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 +}; +static const u8 enc_assoc107[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce107[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key107[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input108[] __initconst = { + 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, + 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, + 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x61, 0x95, 0x01, 0x93, 0xb1, 0xbf, 0x03, 0x11, + 0xff, 0x11, 0x79, 0x89, 0xae, 0xd9, 0xa9, 0x99, + 0xb0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, + 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e +}; +static const u8 enc_output108[] __initconst = { + 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, + 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, + 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a +}; +static const u8 enc_assoc108[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce108[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key108[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input109[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, + 0x08, 0xc0, 0x11, 0xd8, 0x50, 0xdd, 0x1d, 0xcb, + 0xcf, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x53, 0x40, 0xb8, 0x5a, 0x9a, 0xa0, 0x82, 0x96, + 0xb7, 0x7a, 0x5f, 0xc3, 0x96, 0x1f, 0x66, 0x0f, + 0x17, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, + 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 +}; +static const u8 enc_output109[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, + 0xca, 0x07, 0x07, 0x52, 0x6c, 0xad, 0x01, 0x8c, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, + 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb +}; +static const u8 enc_assoc109[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce109[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key109[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input110[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, + 0x88, 0x47, 0x7e, 0x99, 0x92, 0x5a, 0x56, 0xd9, + 0xd6, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xfa, 0x84, 0xf0, 0x64, 0x55, 0x36, 0x42, 0x1b, + 0x2b, 0xb9, 0x24, 0x6e, 0xc2, 0x19, 0xed, 0x0b, + 0x0e, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, + 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 +}; +static const u8 enc_output110[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, + 0x4a, 0x80, 0x68, 0x13, 0xae, 0x2a, 0x4a, 0x9e, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, + 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe +}; +static const u8 enc_assoc110[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce110[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key110[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input111[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, + 0xd6, 0x39, 0xa9, 0x2f, 0xc7, 0xec, 0xa6, 0x55, + 0xf7, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x63, 0x48, 0xb8, 0xfd, 0x29, 0xbf, 0x96, 0xd5, + 0x63, 0xa5, 0x17, 0xe2, 0x7d, 0x7b, 0xfc, 0x0f, + 0x2f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, + 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 +}; +static const u8 enc_output111[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, + 0x14, 0xfe, 0xbf, 0xa5, 0xfb, 0x9c, 0xba, 0x12, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, + 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 +}; +static const u8 enc_assoc111[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce111[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key111[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input112[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, + 0xf9, 0x33, 0x81, 0x1e, 0x65, 0x2d, 0xc6, 0x6a, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xb6, 0x71, 0xc8, 0xca, 0xc2, 0x70, 0xc2, 0x65, + 0xa0, 0xac, 0x2f, 0x53, 0x57, 0x99, 0x88, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, + 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 +}; +static const u8 enc_output112[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, + 0x3b, 0xf4, 0x97, 0x94, 0x59, 0x5d, 0xda, 0x2d, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, + 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce +}; +static const u8 enc_assoc112[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce112[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key112[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input113[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, + 0xa1, 0x2d, 0x79, 0x09, 0x8b, 0x6d, 0xe6, 0x0f, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xa7, 0xc7, 0x58, 0x99, 0xf3, 0xe6, 0x0a, 0xf1, + 0xfc, 0xb6, 0xc7, 0x30, 0x7d, 0x87, 0x59, 0x0f, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, + 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 +}; +static const u8 enc_output113[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, + 0x63, 0xea, 0x6f, 0x83, 0xb7, 0x1d, 0xfa, 0x48, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, + 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 +}; +static const u8 enc_assoc113[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce113[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key113[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input114[] __initconst = { + 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, + 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, + 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0xbe, 0x3f, 0xa6, 0x6b, 0x6c, 0xe7, 0x80, 0x8a, + 0xa3, 0xe4, 0x59, 0x49, 0xf9, 0x44, 0x64, 0x9f, + 0xd0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, + 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 +}; +static const u8 enc_output114[] __initconst = { + 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, + 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, + 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 +}; +static const u8 enc_assoc114[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce114[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key114[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input115[] __initconst = { + 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, + 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, + 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x06, 0x53, 0xe7, 0xa3, 0x31, 0x71, 0x88, 0x33, + 0xac, 0xc3, 0xb9, 0xad, 0xff, 0x1c, 0x31, 0x98, + 0xa6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, + 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f +}; +static const u8 enc_output115[] __initconst = { + 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, + 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, + 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 +}; +static const u8 enc_assoc115[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce115[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key115[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input116[] __initconst = { + 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, + 0xcd, 0xd4, 0x41, 0x9d, 0xe6, 0x4c, 0x8f, 0x44, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x77, 0xb5, 0xc9, 0x07, 0xd9, 0xc9, 0xe1, 0xea, + 0x51, 0x85, 0x1a, 0x20, 0x4a, 0xad, 0x9f, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, + 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 +}; +static const u8 enc_output116[] __initconst = { + 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, + 0x0f, 0x13, 0x57, 0x17, 0xda, 0x3c, 0x93, 0x03, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, + 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 +}; +static const u8 enc_assoc116[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce116[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key116[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input117[] __initconst = { + 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, + 0x8c, 0xce, 0x39, 0x2c, 0xc3, 0xee, 0xdb, 0x47, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x60, 0x1b, 0x5a, 0xd2, 0x06, 0x7f, 0x28, 0x06, + 0x6a, 0x8f, 0x32, 0x81, 0x71, 0x5b, 0xa8, 0x08, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, + 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 +}; +static const u8 enc_output117[] __initconst = { + 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, + 0x4e, 0x09, 0x2f, 0xa6, 0xff, 0x9e, 0xc7, 0x00, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, + 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 +}; +static const u8 enc_assoc117[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce117[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key117[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input118[] __initconst = { + 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, + 0xeb, 0xcc, 0x6a, 0xf5, 0x59, 0xec, 0x8b, 0x43, + 0x97, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd8, 0xc8, 0xc3, 0xfa, 0x1a, 0x9e, 0x47, 0x4a, + 0xbe, 0x52, 0xd0, 0x2c, 0x81, 0x87, 0xe9, 0x0f, + 0x4f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, + 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 +}; +static const u8 enc_output118[] __initconst = { + 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, + 0x29, 0x0b, 0x7c, 0x7f, 0x65, 0x9c, 0x97, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, + 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 +}; +static const u8 enc_assoc118[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce118[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key118[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_enc_vectors[] __initconst = { + { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, + sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, + { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, + sizeof(enc_input002), sizeof(enc_assoc002), sizeof(enc_nonce002) }, + { enc_input003, enc_output003, enc_assoc003, enc_nonce003, enc_key003, + sizeof(enc_input003), sizeof(enc_assoc003), sizeof(enc_nonce003) }, + { enc_input004, enc_output004, enc_assoc004, enc_nonce004, enc_key004, + sizeof(enc_input004), sizeof(enc_assoc004), sizeof(enc_nonce004) }, + { enc_input005, enc_output005, enc_assoc005, enc_nonce005, enc_key005, + sizeof(enc_input005), sizeof(enc_assoc005), sizeof(enc_nonce005) }, + { enc_input006, enc_output006, enc_assoc006, enc_nonce006, enc_key006, + sizeof(enc_input006), sizeof(enc_assoc006), sizeof(enc_nonce006) }, + { enc_input007, enc_output007, enc_assoc007, enc_nonce007, enc_key007, + sizeof(enc_input007), sizeof(enc_assoc007), sizeof(enc_nonce007) }, + { enc_input008, enc_output008, enc_assoc008, enc_nonce008, enc_key008, + sizeof(enc_input008), sizeof(enc_assoc008), sizeof(enc_nonce008) }, + { enc_input009, enc_output009, enc_assoc009, enc_nonce009, enc_key009, + sizeof(enc_input009), sizeof(enc_assoc009), sizeof(enc_nonce009) }, + { enc_input010, enc_output010, enc_assoc010, enc_nonce010, enc_key010, + sizeof(enc_input010), sizeof(enc_assoc010), sizeof(enc_nonce010) }, + { enc_input011, enc_output011, enc_assoc011, enc_nonce011, enc_key011, + sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, + { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, + sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, + { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, + sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, + { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, + sizeof(enc_input054), sizeof(enc_assoc054), sizeof(enc_nonce054) }, + { enc_input055, enc_output055, enc_assoc055, enc_nonce055, enc_key055, + sizeof(enc_input055), sizeof(enc_assoc055), sizeof(enc_nonce055) }, + { enc_input056, enc_output056, enc_assoc056, enc_nonce056, enc_key056, + sizeof(enc_input056), sizeof(enc_assoc056), sizeof(enc_nonce056) }, + { enc_input057, enc_output057, enc_assoc057, enc_nonce057, enc_key057, + sizeof(enc_input057), sizeof(enc_assoc057), sizeof(enc_nonce057) }, + { enc_input058, enc_output058, enc_assoc058, enc_nonce058, enc_key058, + sizeof(enc_input058), sizeof(enc_assoc058), sizeof(enc_nonce058) }, + { enc_input059, enc_output059, enc_assoc059, enc_nonce059, enc_key059, + sizeof(enc_input059), sizeof(enc_assoc059), sizeof(enc_nonce059) }, + { enc_input060, enc_output060, enc_assoc060, enc_nonce060, enc_key060, + sizeof(enc_input060), sizeof(enc_assoc060), sizeof(enc_nonce060) }, + { enc_input061, enc_output061, enc_assoc061, enc_nonce061, enc_key061, + sizeof(enc_input061), sizeof(enc_assoc061), sizeof(enc_nonce061) }, + { enc_input062, enc_output062, enc_assoc062, enc_nonce062, enc_key062, + sizeof(enc_input062), sizeof(enc_assoc062), sizeof(enc_nonce062) }, + { enc_input063, enc_output063, enc_assoc063, enc_nonce063, enc_key063, + sizeof(enc_input063), sizeof(enc_assoc063), sizeof(enc_nonce063) }, + { enc_input064, enc_output064, enc_assoc064, enc_nonce064, enc_key064, + sizeof(enc_input064), sizeof(enc_assoc064), sizeof(enc_nonce064) }, + { enc_input065, enc_output065, enc_assoc065, enc_nonce065, enc_key065, + sizeof(enc_input065), sizeof(enc_assoc065), sizeof(enc_nonce065) }, + { enc_input066, enc_output066, enc_assoc066, enc_nonce066, enc_key066, + sizeof(enc_input066), sizeof(enc_assoc066), sizeof(enc_nonce066) }, + { enc_input067, enc_output067, enc_assoc067, enc_nonce067, enc_key067, + sizeof(enc_input067), sizeof(enc_assoc067), sizeof(enc_nonce067) }, + { enc_input068, enc_output068, enc_assoc068, enc_nonce068, enc_key068, + sizeof(enc_input068), sizeof(enc_assoc068), sizeof(enc_nonce068) }, + { enc_input069, enc_output069, enc_assoc069, enc_nonce069, enc_key069, + sizeof(enc_input069), sizeof(enc_assoc069), sizeof(enc_nonce069) }, + { enc_input070, enc_output070, enc_assoc070, enc_nonce070, enc_key070, + sizeof(enc_input070), sizeof(enc_assoc070), sizeof(enc_nonce070) }, + { enc_input071, enc_output071, enc_assoc071, enc_nonce071, enc_key071, + sizeof(enc_input071), sizeof(enc_assoc071), sizeof(enc_nonce071) }, + { enc_input072, enc_output072, enc_assoc072, enc_nonce072, enc_key072, + sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, + { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, + sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, + { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, + sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, + { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, + sizeof(enc_input077), sizeof(enc_assoc077), sizeof(enc_nonce077) }, + { enc_input078, enc_output078, enc_assoc078, enc_nonce078, enc_key078, + sizeof(enc_input078), sizeof(enc_assoc078), sizeof(enc_nonce078) }, + { enc_input079, enc_output079, enc_assoc079, enc_nonce079, enc_key079, + sizeof(enc_input079), sizeof(enc_assoc079), sizeof(enc_nonce079) }, + { enc_input080, enc_output080, enc_assoc080, enc_nonce080, enc_key080, + sizeof(enc_input080), sizeof(enc_assoc080), sizeof(enc_nonce080) }, + { enc_input081, enc_output081, enc_assoc081, enc_nonce081, enc_key081, + sizeof(enc_input081), sizeof(enc_assoc081), sizeof(enc_nonce081) }, + { enc_input082, enc_output082, enc_assoc082, enc_nonce082, enc_key082, + sizeof(enc_input082), sizeof(enc_assoc082), sizeof(enc_nonce082) }, + { enc_input083, enc_output083, enc_assoc083, enc_nonce083, enc_key083, + sizeof(enc_input083), sizeof(enc_assoc083), sizeof(enc_nonce083) }, + { enc_input084, enc_output084, enc_assoc084, enc_nonce084, enc_key084, + sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, + { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, + sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, + { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, + sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, + { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, + sizeof(enc_input094), sizeof(enc_assoc094), sizeof(enc_nonce094) }, + { enc_input095, enc_output095, enc_assoc095, enc_nonce095, enc_key095, + sizeof(enc_input095), sizeof(enc_assoc095), sizeof(enc_nonce095) }, + { enc_input096, enc_output096, enc_assoc096, enc_nonce096, enc_key096, + sizeof(enc_input096), sizeof(enc_assoc096), sizeof(enc_nonce096) }, + { enc_input097, enc_output097, enc_assoc097, enc_nonce097, enc_key097, + sizeof(enc_input097), sizeof(enc_assoc097), sizeof(enc_nonce097) }, + { enc_input098, enc_output098, enc_assoc098, enc_nonce098, enc_key098, + sizeof(enc_input098), sizeof(enc_assoc098), sizeof(enc_nonce098) }, + { enc_input099, enc_output099, enc_assoc099, enc_nonce099, enc_key099, + sizeof(enc_input099), sizeof(enc_assoc099), sizeof(enc_nonce099) }, + { enc_input100, enc_output100, enc_assoc100, enc_nonce100, enc_key100, + sizeof(enc_input100), sizeof(enc_assoc100), sizeof(enc_nonce100) }, + { enc_input101, enc_output101, enc_assoc101, enc_nonce101, enc_key101, + sizeof(enc_input101), sizeof(enc_assoc101), sizeof(enc_nonce101) }, + { enc_input102, enc_output102, enc_assoc102, enc_nonce102, enc_key102, + sizeof(enc_input102), sizeof(enc_assoc102), sizeof(enc_nonce102) }, + { enc_input103, enc_output103, enc_assoc103, enc_nonce103, enc_key103, + sizeof(enc_input103), sizeof(enc_assoc103), sizeof(enc_nonce103) }, + { enc_input104, enc_output104, enc_assoc104, enc_nonce104, enc_key104, + sizeof(enc_input104), sizeof(enc_assoc104), sizeof(enc_nonce104) }, + { enc_input105, enc_output105, enc_assoc105, enc_nonce105, enc_key105, + sizeof(enc_input105), sizeof(enc_assoc105), sizeof(enc_nonce105) }, + { enc_input106, enc_output106, enc_assoc106, enc_nonce106, enc_key106, + sizeof(enc_input106), sizeof(enc_assoc106), sizeof(enc_nonce106) }, + { enc_input107, enc_output107, enc_assoc107, enc_nonce107, enc_key107, + sizeof(enc_input107), sizeof(enc_assoc107), sizeof(enc_nonce107) }, + { enc_input108, enc_output108, enc_assoc108, enc_nonce108, enc_key108, + sizeof(enc_input108), sizeof(enc_assoc108), sizeof(enc_nonce108) }, + { enc_input109, enc_output109, enc_assoc109, enc_nonce109, enc_key109, + sizeof(enc_input109), sizeof(enc_assoc109), sizeof(enc_nonce109) }, + { enc_input110, enc_output110, enc_assoc110, enc_nonce110, enc_key110, + sizeof(enc_input110), sizeof(enc_assoc110), sizeof(enc_nonce110) }, + { enc_input111, enc_output111, enc_assoc111, enc_nonce111, enc_key111, + sizeof(enc_input111), sizeof(enc_assoc111), sizeof(enc_nonce111) }, + { enc_input112, enc_output112, enc_assoc112, enc_nonce112, enc_key112, + sizeof(enc_input112), sizeof(enc_assoc112), sizeof(enc_nonce112) }, + { enc_input113, enc_output113, enc_assoc113, enc_nonce113, enc_key113, + sizeof(enc_input113), sizeof(enc_assoc113), sizeof(enc_nonce113) }, + { enc_input114, enc_output114, enc_assoc114, enc_nonce114, enc_key114, + sizeof(enc_input114), sizeof(enc_assoc114), sizeof(enc_nonce114) }, + { enc_input115, enc_output115, enc_assoc115, enc_nonce115, enc_key115, + sizeof(enc_input115), sizeof(enc_assoc115), sizeof(enc_nonce115) }, + { enc_input116, enc_output116, enc_assoc116, enc_nonce116, enc_key116, + sizeof(enc_input116), sizeof(enc_assoc116), sizeof(enc_nonce116) }, + { enc_input117, enc_output117, enc_assoc117, enc_nonce117, enc_key117, + sizeof(enc_input117), sizeof(enc_assoc117), sizeof(enc_nonce117) }, + { enc_input118, enc_output118, enc_assoc118, enc_nonce118, enc_key118, + sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } +}; + +static const u8 dec_input001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 dec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 dec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 dec_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 dec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 dec_input002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 dec_output002[] __initconst = { }; +static const u8 dec_assoc002[] __initconst = { }; +static const u8 dec_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 dec_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 dec_input003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 dec_output003[] __initconst = { }; +static const u8 dec_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 dec_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 dec_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 dec_input004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 dec_output004[] __initconst = { + 0xa4 +}; +static const u8 dec_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 dec_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 dec_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 dec_input005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 dec_output005[] __initconst = { + 0x2d +}; +static const u8 dec_assoc005[] __initconst = { }; +static const u8 dec_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 dec_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 dec_input006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 dec_output006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 dec_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 dec_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 dec_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 dec_input007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 dec_output007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 dec_assoc007[] __initconst = { }; +static const u8 dec_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 dec_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 dec_input008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 dec_output008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 dec_assoc008[] __initconst = { }; +static const u8 dec_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 dec_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 dec_input009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 dec_output009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 dec_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 dec_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 dec_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 dec_input010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 dec_output010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 dec_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 dec_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 dec_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 dec_input011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 dec_output011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 dec_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 dec_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 dec_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 dec_input012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 dec_output012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const u8 dec_input013[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd7 +}; +static const u8 dec_output013[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc013[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce013[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key013[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_dec_vectors[] __initconst = { + { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, + sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, + { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, + sizeof(dec_input002), sizeof(dec_assoc002), sizeof(dec_nonce002) }, + { dec_input003, dec_output003, dec_assoc003, dec_nonce003, dec_key003, + sizeof(dec_input003), sizeof(dec_assoc003), sizeof(dec_nonce003) }, + { dec_input004, dec_output004, dec_assoc004, dec_nonce004, dec_key004, + sizeof(dec_input004), sizeof(dec_assoc004), sizeof(dec_nonce004) }, + { dec_input005, dec_output005, dec_assoc005, dec_nonce005, dec_key005, + sizeof(dec_input005), sizeof(dec_assoc005), sizeof(dec_nonce005) }, + { dec_input006, dec_output006, dec_assoc006, dec_nonce006, dec_key006, + sizeof(dec_input006), sizeof(dec_assoc006), sizeof(dec_nonce006) }, + { dec_input007, dec_output007, dec_assoc007, dec_nonce007, dec_key007, + sizeof(dec_input007), sizeof(dec_assoc007), sizeof(dec_nonce007) }, + { dec_input008, dec_output008, dec_assoc008, dec_nonce008, dec_key008, + sizeof(dec_input008), sizeof(dec_assoc008), sizeof(dec_nonce008) }, + { dec_input009, dec_output009, dec_assoc009, dec_nonce009, dec_key009, + sizeof(dec_input009), sizeof(dec_assoc009), sizeof(dec_nonce009) }, + { dec_input010, dec_output010, dec_assoc010, dec_nonce010, dec_key010, + sizeof(dec_input010), sizeof(dec_assoc010), sizeof(dec_nonce010) }, + { dec_input011, dec_output011, dec_assoc011, dec_nonce011, dec_key011, + sizeof(dec_input011), sizeof(dec_assoc011), sizeof(dec_nonce011) }, + { dec_input012, dec_output012, dec_assoc012, dec_nonce012, dec_key012, + sizeof(dec_input012), sizeof(dec_assoc012), sizeof(dec_nonce012) }, + { dec_input013, dec_output013, dec_assoc013, dec_nonce013, dec_key013, + sizeof(dec_input013), sizeof(dec_assoc013), sizeof(dec_nonce013), + true } +}; + +static const u8 xenc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xenc_output001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xenc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xenc_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xenc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_enc_vectors[] __initconst = { + { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, + sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } +}; + +static const u8 xdec_input001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xdec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xdec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xdec_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xdec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_dec_vectors[] __initconst = { + { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, + sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } +}; + +static void __init +chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src, + const size_t src_len, const u8 *ad, + const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + + simd_get(&simd_context); + chacha20_init(&chacha20_state, key, 0); + chacha20_state.counter[1] = get_unaligned_le32(nonce + 0); + chacha20_state.counter[2] = get_unaligned_le32(nonce + 4); + chacha20_state.counter[3] = get_unaligned_le32(nonce + 8); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + &simd_context); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + &simd_context); + chacha20(&chacha20_state, dst, src, src_len, &simd_context); + poly1305_update(&poly1305_state, dst, src_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + &simd_context); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + &simd_context); + poly1305_final(&poly1305_state, dst + src_len, &simd_context); + simd_put(&simd_context); + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +static void __init +chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + if (nonce_len == 8) + chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_selftest_encrypt_bignonce(dst, src, src_len, + ad, ad_len, nonce, + key); + else + BUG(); +} + +static bool __init +decryption_success(bool func_ret, bool expect_failure, int memcmp_result) +{ + if (expect_failure) + return !func_ret; + return func_ret && !memcmp_result; +} + +static bool __init chacha20poly1305_selftest(void) +{ + enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; + bool success = true, ret; + simd_context_t simd_context; + struct scatterlist sg_src[3]; + + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { + pr_err("chacha20poly1305 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + chacha20poly1305_enc_vectors[i].nonce, + chacha20poly1305_enc_vectors[i].nlen, + chacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + if (chacha20poly1305_enc_vectors[i].nlen != 8) + continue; + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE); + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), + chacha20poly1305_enc_vectors[i].key, + &simd_context); + if (!ret || memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = chacha20poly1305_decrypt(computed_output, + chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_dec_vectors[i].ilen); + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key, &simd_context); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + xchacha20poly1305_encrypt(computed_output, + xchacha20poly1305_enc_vectors[i].input, + xchacha20poly1305_enc_vectors[i].ilen, + xchacha20poly1305_enc_vectors[i].assoc, + xchacha20poly1305_enc_vectors[i].alen, + xchacha20poly1305_enc_vectors[i].nonce, + xchacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = xchacha20poly1305_decrypt(computed_output, + xchacha20poly1305_dec_vectors[i].input, + xchacha20poly1305_dec_vectors[i].ilen, + xchacha20poly1305_dec_vectors[i].assoc, + xchacha20poly1305_dec_vectors[i].alen, + xchacha20poly1305_dec_vectors[i].nonce, + xchacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + xchacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + xchacha20poly1305_dec_vectors[i].output, + xchacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + + simd_get(&simd_context); + for (total_len = POLY1305_MAC_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + sg_init_table(sg_src, 3); + sg_set_buf(&sg_src[0], input, i); + sg_set_buf(&sg_src[1], input + i, j - i); + sg_set_buf(&sg_src[2], input + j, total_len - j); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_MAC_SIZE, NULL, 0, + 0, enc_key001, &simd_context)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_MAC_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail;; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001, + &simd_context)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + simd_put(&simd_context); + +out: + kfree(computed_output); + kfree(input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/curve25519.c b/net/wireguard/crypto/zinc/selftest/curve25519.c new file mode 100644 index 0000000000000..0e3e3af06ba44 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/curve25519.c @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct curve25519_test_vector { + u8 private[CURVE25519_KEY_SIZE]; + u8 public[CURVE25519_KEY_SIZE]; + u8 result[CURVE25519_KEY_SIZE]; + bool valid; +}; +static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = { + { + .private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d, + 0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45, + 0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a, + 0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a }, + .public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4, + 0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37, + 0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d, + 0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b, + 0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6, + 0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd, + 0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb }, + .public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54, + 0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a, + 0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4, + 0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64, + 0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d, + 0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98, + 0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f, + 0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d, + 0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3, + 0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 }, + .valid = true + }, + { + .private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + { + .private = { 1, 2, 3, 4 }, + .public = { 0 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 2, 4, 6, 8 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f }, + .result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2, + 0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57, + 0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05, + 0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 }, + .valid = true + }, + { + .private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 }, + .result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d, + 0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12, + 0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99, + 0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c }, + .valid = true + }, + /* wycheproof - normal case */ + { + .private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda, + 0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66, + 0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3, + 0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba }, + .public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5, + 0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9, + 0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e, + 0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a }, + .result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5, + 0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38, + 0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e, + 0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4, + 0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5, + 0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49, + 0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 }, + .public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5, + 0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8, + 0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3, + 0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 }, + .result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff, + 0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d, + 0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe, + 0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9, + 0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39, + 0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5, + 0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 }, + .public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f, + 0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b, + 0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c, + 0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 }, + .result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53, + 0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57, + 0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0, + 0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc, + 0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d, + 0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67, + 0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c }, + .public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97, + 0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f, + 0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45, + 0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a }, + .result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93, + 0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2, + 0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44, + 0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1, + 0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95, + 0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99, + 0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d }, + .public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27, + 0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07, + 0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae, + 0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c }, + .result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73, + 0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2, + 0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f, + 0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9, + 0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd, + 0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b, + 0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 }, + .public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5, + 0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52, + 0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8, + 0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 }, + .result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86, + 0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4, + 0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6, + 0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 }, + .valid = true + }, + /* wycheproof - public key = 0 */ + { + .private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11, + 0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac, + 0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b, + 0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key = 1 */ + { + .private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61, + 0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea, + 0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f, + 0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04, + 0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77, + 0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90, + 0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97, + 0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9, + 0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7, + 0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36, + 0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd, + 0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c, + 0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e, + 0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b, + 0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e, + 0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed, + 0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e, + 0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd, + 0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 }, + .public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 }, + .result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f, + 0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1, + 0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10, + 0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3, + 0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d, + 0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00, + 0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 }, + .public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f }, + .result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8, + 0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4, + 0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70, + 0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3, + 0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a, + 0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e, + 0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57, + 0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c, + 0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59, + 0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f, + 0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42, + 0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9, + 0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c, + 0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5, + 0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65, + 0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6, + 0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4, + 0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8, + 0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7, + 0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca, + 0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f, + 0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa, + 0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3, + 0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52, + 0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3, + 0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e, + 0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75, + 0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26, + 0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea, + 0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00, + 0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8, + 0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32, + 0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87, + 0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c, + 0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6, + 0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb, + 0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 }, + .public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f }, + .result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85, + 0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f, + 0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0, + 0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38, + 0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b, + 0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c, + 0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b, + 0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81, + 0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3, + 0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d, + 0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42, + 0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98, + 0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c, + 0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9, + 0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89, + 0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29, + 0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6, + 0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c, + 0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75, + 0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89, + 0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c, + 0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f }, + .valid = true + }, + /* wycheproof - public key with low order */ + { + .private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30, + 0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69, + 0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14, + 0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3, + 0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b, + 0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef, + 0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20, + 0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf, + 0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43, + 0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f, + 0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65, + 0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06, + 0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe, + 0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9, + 0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f, + 0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8, + 0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85, + 0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c, + 0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8, + 0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d, + 0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0, + 0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a, + 0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b, + 0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67, + 0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46, + 0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02, + 0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3, + 0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30, + 0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1, + 0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6, + 0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f, + 0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77, + 0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0, + 0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e, + 0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f, + 0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77, + 0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc, + 0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1, + 0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d, + 0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09, + 0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde, + 0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1, + 0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81, + 0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a, + 0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99, + 0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17, + 0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35, + 0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55, + 0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11, + 0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b, + 0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9, + 0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53, + 0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e, + 0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6, + 0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78, + 0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2, + 0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd, + 0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb, + 0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40, + 0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2, + 0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9, + 0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60, + 0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13, + 0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c, + 0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3, + 0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65, + 0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a, + 0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7, + 0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11, + 0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82, + 0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4, + 0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c, + 0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e, + 0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a, + 0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d, + 0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2, + 0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60, + 0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25, + 0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb, + 0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97, + 0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c, + 0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 }, + .public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23, + 0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8, + 0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69, + 0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a, + 0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23, + 0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b, + 0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 }, + .public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b, + 0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44, + 0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37, + 0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80, + 0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d, + 0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b, + 0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 }, + .public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63, + 0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae, + 0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f, + 0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0, + 0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd, + 0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49, + 0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41, + 0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0, + 0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf, + 0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9, + 0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa, + 0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5, + 0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47, + 0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3, + 0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b, + 0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8, + 0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98, + 0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0, + 0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0, + 0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1, + 0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a, + 0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02, + 0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4, + 0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68, + 0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f, + 0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2, + 0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95, + 0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7, + 0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06, + 0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9, + 0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5, + 0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0, + 0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80, + 0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd, + 0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4, + 0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04, + 0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0, + 0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac, + 0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48, + 0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, + 0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, + 0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4, + 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d }, + .public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, + 0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, + 0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e, + 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 }, + .result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, + 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, + 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, + 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde, + 0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8, + 0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4, + 0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 }, + .result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d, + 0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64, + 0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd, + 0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 }, + .result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8, + 0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf, + 0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94, + 0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d }, + .result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84, + 0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62, + 0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e, + 0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 }, + .result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8, + 0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58, + 0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02, + 0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 }, + .result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9, + 0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a, + 0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44, + 0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b }, + .result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd, + 0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22, + 0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56, + 0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b }, + .result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53, + 0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f, + 0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18, + 0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f }, + .result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55, + 0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b, + 0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79, + 0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f }, + .result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39, + 0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c, + 0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb, + 0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e }, + .result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04, + 0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10, + 0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58, + 0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c }, + .result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3, + 0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c, + 0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88, + 0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 }, + .result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a, + 0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49, + 0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a, + 0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca, + 0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c, + 0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb, + 0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58, + 0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7, + 0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01, + 0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d }, + .result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d, + 0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27, + 0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b, + 0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26, + 0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2, + 0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44, + 0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e }, + .result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6, + 0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d, + 0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e, + 0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61, + 0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67, + 0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e, + 0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c }, + .result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65, + 0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce, + 0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0, + 0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee, + 0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d, + 0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14, + 0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 }, + .result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e, + 0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc, + 0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5, + 0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4, + 0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5, + 0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c, + 0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 }, + .result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b, + 0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93, + 0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f, + 0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 }, + .valid = true + }, + /* wycheproof - private key == -1 (mod order) */ + { + .private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8, + 0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 }, + .public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .valid = true + }, + /* wycheproof - private key == 1 (mod order) on twist */ + { + .private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef, + 0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f }, + .public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .valid = true + } +}; + +static bool __init curve25519_selftest(void) +{ + bool success = true, ret, ret2; + size_t i = 0, j; + u8 in[CURVE25519_KEY_SIZE]; + u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE]; + + for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) { + memset(out, 0, CURVE25519_KEY_SIZE); + ret = curve25519(out, curve25519_test_vectors[i].private, + curve25519_test_vectors[i].public); + if (ret != curve25519_test_vectors[i].valid || + memcmp(out, curve25519_test_vectors[i].result, + CURVE25519_KEY_SIZE)) { + pr_err("curve25519 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < 5; ++i) { + get_random_bytes(in, sizeof(in)); + ret = curve25519_generate_public(out, in); + ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + if (ret != ret2 || memcmp(out, out2, CURVE25519_KEY_SIZE)) { + pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x", + i + 1); + for (j = CURVE25519_KEY_SIZE; j-- > 0;) + printk(KERN_CONT "%02x", in[j]); + printk(KERN_CONT "\n"); + success = false; + } + } + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/poly1305.c b/net/wireguard/crypto/zinc/selftest/poly1305.c new file mode 100644 index 0000000000000..b4d7a9c2f6ec7 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/poly1305.c @@ -0,0 +1,1107 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct poly1305_testvec { + const u8 *input, *output, *key; + size_t ilen; +}; + +/* RFC7539 */ +static const u8 input01[] __initconst = { + 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x67, 0x72, + 0x61, 0x70, 0x68, 0x69, 0x63, 0x20, 0x46, 0x6f, + 0x72, 0x75, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x20, 0x47, 0x72, 0x6f, + 0x75, 0x70 +}; +static const u8 output01[] __initconst = { + 0xa8, 0x06, 0x1d, 0xc1, 0x30, 0x51, 0x36, 0xc6, + 0xc2, 0x2b, 0x8b, 0xaf, 0x0c, 0x01, 0x27, 0xa9 +}; +static const u8 key01[] __initconst = { + 0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33, + 0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8, + 0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd, + 0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b +}; + +/* "The Poly1305-AES message-authentication code" */ +static const u8 input02[] __initconst = { + 0xf3, 0xf6 +}; +static const u8 output02[] __initconst = { + 0xf4, 0xc6, 0x33, 0xc3, 0x04, 0x4f, 0xc1, 0x45, + 0xf8, 0x4f, 0x33, 0x5c, 0xb8, 0x19, 0x53, 0xde +}; +static const u8 key02[] __initconst = { + 0x85, 0x1f, 0xc4, 0x0c, 0x34, 0x67, 0xac, 0x0b, + 0xe0, 0x5c, 0xc2, 0x04, 0x04, 0xf3, 0xf7, 0x00, + 0x58, 0x0b, 0x3b, 0x0f, 0x94, 0x47, 0xbb, 0x1e, + 0x69, 0xd0, 0x95, 0xb5, 0x92, 0x8b, 0x6d, 0xbc +}; + +static const u8 input03[] __initconst = { }; +static const u8 output03[] __initconst = { + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; +static const u8 key03[] __initconst = { + 0xa0, 0xf3, 0x08, 0x00, 0x00, 0xf4, 0x64, 0x00, + 0xd0, 0xc7, 0xe9, 0x07, 0x6c, 0x83, 0x44, 0x03, + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; + +static const u8 input04[] __initconst = { + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output04[] __initconst = { + 0x0e, 0xe1, 0xc1, 0x6b, 0xb7, 0x3f, 0x0f, 0x4f, + 0xd1, 0x98, 0x81, 0x75, 0x3c, 0x01, 0xcd, 0xbe +}; +static const u8 key04[] __initconst = { + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef +}; + +static const u8 input05[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9 +}; +static const u8 output05[] __initconst = { + 0x51, 0x54, 0xad, 0x0d, 0x2c, 0xb2, 0x6e, 0x01, + 0x27, 0x4f, 0xc5, 0x11, 0x48, 0x49, 0x1f, 0x1b +}; +static const u8 key05[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* self-generated vectors exercise "significant" lengths, such that they + * are handled by different code paths */ +static const u8 input06[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf +}; +static const u8 output06[] __initconst = { + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 key06[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input07[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67 +}; +static const u8 output07[] __initconst = { + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 key07[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input08[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output08[] __initconst = { + 0xbb, 0xb6, 0x13, 0xb2, 0xb6, 0xd7, 0x53, 0xba, + 0x07, 0x39, 0x5b, 0x91, 0x6a, 0xae, 0xce, 0x15 +}; +static const u8 key08[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input09[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24 +}; +static const u8 output09[] __initconst = { + 0xc7, 0x94, 0xd7, 0x05, 0x7d, 0x17, 0x78, 0xc4, + 0xbb, 0xee, 0x0a, 0x39, 0xb3, 0xd9, 0x73, 0x42 +}; +static const u8 key09[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input10[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output10[] __initconst = { + 0xff, 0xbc, 0xb9, 0xb3, 0x71, 0x42, 0x31, 0x52, + 0xd7, 0xfc, 0xa5, 0xad, 0x04, 0x2f, 0xba, 0xa9 +}; +static const u8 key10[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input11[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 output11[] __initconst = { + 0x06, 0x9e, 0xd6, 0xb8, 0xef, 0x0f, 0x20, 0x7b, + 0x3e, 0x24, 0x3b, 0xb1, 0x01, 0x9f, 0xe6, 0x32 +}; +static const u8 key11[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input12[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output12[] __initconst = { + 0xcc, 0xa3, 0x39, 0xd9, 0xa4, 0x5f, 0xa2, 0x36, + 0x8c, 0x2c, 0x68, 0xb3, 0xa4, 0x17, 0x91, 0x33 +}; +static const u8 key12[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input13[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output13[] __initconst = { + 0x53, 0xf6, 0xe8, 0x28, 0xa2, 0xf0, 0xfe, 0x0e, + 0xe8, 0x15, 0xbf, 0x0b, 0xd5, 0x84, 0x1a, 0x34 +}; +static const u8 key13[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input14[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output14[] __initconst = { + 0xb8, 0x46, 0xd4, 0x4e, 0x9b, 0xbd, 0x53, 0xce, + 0xdf, 0xfb, 0xfb, 0xb6, 0xb7, 0xfa, 0x49, 0x33 +}; +static const u8 key14[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* 4th power of the key spills to 131th bit in SIMD key setup */ +static const u8 input15[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output15[] __initconst = { + 0x07, 0x14, 0x5a, 0x4c, 0x02, 0xfe, 0x5f, 0xa3, + 0x20, 0x36, 0xde, 0x68, 0xfa, 0xbe, 0x90, 0x66 +}; +static const u8 key15[] __initconst = { + 0xad, 0x62, 0x81, 0x07, 0xe8, 0x35, 0x1d, 0x0f, + 0x2c, 0x23, 0x1a, 0x05, 0xdc, 0x4a, 0x41, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* OpenSSL's poly1305_ieee754.c failed this in final stage */ +static const u8 input16[] __initconst = { + 0x84, 0x23, 0x64, 0xe1, 0x56, 0x33, 0x6c, 0x09, + 0x98, 0xb9, 0x33, 0xa6, 0x23, 0x77, 0x26, 0x18, + 0x0d, 0x9e, 0x3f, 0xdc, 0xbd, 0xe4, 0xcd, 0x5d, + 0x17, 0x08, 0x0f, 0xc3, 0xbe, 0xb4, 0x96, 0x14, + 0xd7, 0x12, 0x2c, 0x03, 0x74, 0x63, 0xff, 0x10, + 0x4d, 0x73, 0xf1, 0x9c, 0x12, 0x70, 0x46, 0x28, + 0xd4, 0x17, 0xc4, 0xc5, 0x4a, 0x3f, 0xe3, 0x0d, + 0x3c, 0x3d, 0x77, 0x14, 0x38, 0x2d, 0x43, 0xb0, + 0x38, 0x2a, 0x50, 0xa5, 0xde, 0xe5, 0x4b, 0xe8, + 0x44, 0xb0, 0x76, 0xe8, 0xdf, 0x88, 0x20, 0x1a, + 0x1c, 0xd4, 0x3b, 0x90, 0xeb, 0x21, 0x64, 0x3f, + 0xa9, 0x6f, 0x39, 0xb5, 0x18, 0xaa, 0x83, 0x40, + 0xc9, 0x42, 0xff, 0x3c, 0x31, 0xba, 0xf7, 0xc9, + 0xbd, 0xbf, 0x0f, 0x31, 0xae, 0x3f, 0xa0, 0x96, + 0xbf, 0x8c, 0x63, 0x03, 0x06, 0x09, 0x82, 0x9f, + 0xe7, 0x2e, 0x17, 0x98, 0x24, 0x89, 0x0b, 0xc8, + 0xe0, 0x8c, 0x31, 0x5c, 0x1c, 0xce, 0x2a, 0x83, + 0x14, 0x4d, 0xbb, 0xff, 0x09, 0xf7, 0x4e, 0x3e, + 0xfc, 0x77, 0x0b, 0x54, 0xd0, 0x98, 0x4a, 0x8f, + 0x19, 0xb1, 0x47, 0x19, 0xe6, 0x36, 0x35, 0x64, + 0x1d, 0x6b, 0x1e, 0xed, 0xf6, 0x3e, 0xfb, 0xf0, + 0x80, 0xe1, 0x78, 0x3d, 0x32, 0x44, 0x54, 0x12, + 0x11, 0x4c, 0x20, 0xde, 0x0b, 0x83, 0x7a, 0x0d, + 0xfa, 0x33, 0xd6, 0xb8, 0x28, 0x25, 0xff, 0xf4, + 0x4c, 0x9a, 0x70, 0xea, 0x54, 0xce, 0x47, 0xf0, + 0x7d, 0xf6, 0x98, 0xe6, 0xb0, 0x33, 0x23, 0xb5, + 0x30, 0x79, 0x36, 0x4a, 0x5f, 0xc3, 0xe9, 0xdd, + 0x03, 0x43, 0x92, 0xbd, 0xde, 0x86, 0xdc, 0xcd, + 0xda, 0x94, 0x32, 0x1c, 0x5e, 0x44, 0x06, 0x04, + 0x89, 0x33, 0x6c, 0xb6, 0x5b, 0xf3, 0x98, 0x9c, + 0x36, 0xf7, 0x28, 0x2c, 0x2f, 0x5d, 0x2b, 0x88, + 0x2c, 0x17, 0x1e, 0x74 +}; +static const u8 output16[] __initconst = { + 0xf2, 0x48, 0x31, 0x2e, 0x57, 0x8d, 0x9d, 0x58, + 0xf8, 0xb7, 0xbb, 0x4d, 0x19, 0x10, 0x54, 0x31 +}; +static const u8 key16[] __initconst = { + 0x95, 0xd5, 0xc0, 0x05, 0x50, 0x3e, 0x51, 0x0d, + 0x8c, 0xd0, 0xaa, 0x07, 0x2c, 0x4a, 0x4d, 0x06, + 0x6e, 0xab, 0xc5, 0x2d, 0x11, 0x65, 0x3d, 0xf4, + 0x7f, 0xbf, 0x63, 0xab, 0x19, 0x8b, 0xcc, 0x26 +}; + +/* AVX2 in OpenSSL's poly1305-x86.pl failed this with 176+32 split */ +static const u8 input17[] __initconst = { + 0x24, 0x8a, 0xc3, 0x10, 0x85, 0xb6, 0xc2, 0xad, + 0xaa, 0xa3, 0x82, 0x59, 0xa0, 0xd7, 0x19, 0x2c, + 0x5c, 0x35, 0xd1, 0xbb, 0x4e, 0xf3, 0x9a, 0xd9, + 0x4c, 0x38, 0xd1, 0xc8, 0x24, 0x79, 0xe2, 0xdd, + 0x21, 0x59, 0xa0, 0x77, 0x02, 0x4b, 0x05, 0x89, + 0xbc, 0x8a, 0x20, 0x10, 0x1b, 0x50, 0x6f, 0x0a, + 0x1a, 0xd0, 0xbb, 0xab, 0x76, 0xe8, 0x3a, 0x83, + 0xf1, 0xb9, 0x4b, 0xe6, 0xbe, 0xae, 0x74, 0xe8, + 0x74, 0xca, 0xb6, 0x92, 0xc5, 0x96, 0x3a, 0x75, + 0x43, 0x6b, 0x77, 0x61, 0x21, 0xec, 0x9f, 0x62, + 0x39, 0x9a, 0x3e, 0x66, 0xb2, 0xd2, 0x27, 0x07, + 0xda, 0xe8, 0x19, 0x33, 0xb6, 0x27, 0x7f, 0x3c, + 0x85, 0x16, 0xbc, 0xbe, 0x26, 0xdb, 0xbd, 0x86, + 0xf3, 0x73, 0x10, 0x3d, 0x7c, 0xf4, 0xca, 0xd1, + 0x88, 0x8c, 0x95, 0x21, 0x18, 0xfb, 0xfb, 0xd0, + 0xd7, 0xb4, 0xbe, 0xdc, 0x4a, 0xe4, 0x93, 0x6a, + 0xff, 0x91, 0x15, 0x7e, 0x7a, 0xa4, 0x7c, 0x54, + 0x44, 0x2e, 0xa7, 0x8d, 0x6a, 0xc2, 0x51, 0xd3, + 0x24, 0xa0, 0xfb, 0xe4, 0x9d, 0x89, 0xcc, 0x35, + 0x21, 0xb6, 0x6d, 0x16, 0xe9, 0xc6, 0x6a, 0x37, + 0x09, 0x89, 0x4e, 0x4e, 0xb0, 0xa4, 0xee, 0xdc, + 0x4a, 0xe1, 0x94, 0x68, 0xe6, 0x6b, 0x81, 0xf2, + 0x71, 0x35, 0x1b, 0x1d, 0x92, 0x1e, 0xa5, 0x51, + 0x04, 0x7a, 0xbc, 0xc6, 0xb8, 0x7a, 0x90, 0x1f, + 0xde, 0x7d, 0xb7, 0x9f, 0xa1, 0x81, 0x8c, 0x11, + 0x33, 0x6d, 0xbc, 0x07, 0x24, 0x4a, 0x40, 0xeb +}; +static const u8 output17[] __initconst = { + 0xbc, 0x93, 0x9b, 0xc5, 0x28, 0x14, 0x80, 0xfa, + 0x99, 0xc6, 0xd6, 0x8c, 0x25, 0x8e, 0xc4, 0x2f +}; +static const u8 key17[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* test vectors from Google */ +static const u8 input18[] __initconst = { }; +static const u8 output18[] __initconst = { + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; +static const u8 key18[] __initconst = { + 0xc8, 0xaf, 0xaa, 0xc3, 0x31, 0xee, 0x37, 0x2c, + 0xd6, 0x08, 0x2d, 0xe1, 0x34, 0x94, 0x3b, 0x17, + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; + +static const u8 input19[] __initconst = { + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, + 0x72, 0x6c, 0x64, 0x21 +}; +static const u8 output19[] __initconst = { + 0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, + 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0 +}; +static const u8 key19[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input20[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output20[] __initconst = { + 0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, + 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07 +}; +static const u8 key20[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input21[] __initconst = { + 0x89, 0xda, 0xb8, 0x0b, 0x77, 0x17, 0xc1, 0xdb, + 0x5d, 0xb4, 0x37, 0x86, 0x0a, 0x3f, 0x70, 0x21, + 0x8e, 0x93, 0xe1, 0xb8, 0xf4, 0x61, 0xfb, 0x67, + 0x7f, 0x16, 0xf3, 0x5f, 0x6f, 0x87, 0xe2, 0xa9, + 0x1c, 0x99, 0xbc, 0x3a, 0x47, 0xac, 0xe4, 0x76, + 0x40, 0xcc, 0x95, 0xc3, 0x45, 0xbe, 0x5e, 0xcc, + 0xa5, 0xa3, 0x52, 0x3c, 0x35, 0xcc, 0x01, 0x89, + 0x3a, 0xf0, 0xb6, 0x4a, 0x62, 0x03, 0x34, 0x27, + 0x03, 0x72, 0xec, 0x12, 0x48, 0x2d, 0x1b, 0x1e, + 0x36, 0x35, 0x61, 0x69, 0x8a, 0x57, 0x8b, 0x35, + 0x98, 0x03, 0x49, 0x5b, 0xb4, 0xe2, 0xef, 0x19, + 0x30, 0xb1, 0x7a, 0x51, 0x90, 0xb5, 0x80, 0xf1, + 0x41, 0x30, 0x0d, 0xf3, 0x0a, 0xdb, 0xec, 0xa2, + 0x8f, 0x64, 0x27, 0xa8, 0xbc, 0x1a, 0x99, 0x9f, + 0xd5, 0x1c, 0x55, 0x4a, 0x01, 0x7d, 0x09, 0x5d, + 0x8c, 0x3e, 0x31, 0x27, 0xda, 0xf9, 0xf5, 0x95 +}; +static const u8 output21[] __initconst = { + 0xc8, 0x5d, 0x15, 0xed, 0x44, 0xc3, 0x78, 0xd6, + 0xb0, 0x0e, 0x23, 0x06, 0x4c, 0x7b, 0xcd, 0x51 +}; +static const u8 key21[] __initconst = { + 0x2d, 0x77, 0x3b, 0xe3, 0x7a, 0xdb, 0x1e, 0x4d, + 0x68, 0x3b, 0xf0, 0x07, 0x5e, 0x79, 0xc4, 0xee, + 0x03, 0x79, 0x18, 0x53, 0x5a, 0x7f, 0x99, 0xcc, + 0xb7, 0x04, 0x0f, 0xb5, 0xf5, 0xf4, 0x3a, 0xea +}; + +static const u8 input22[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, + 0x17, 0x03, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x06, 0xdb, 0x1f, 0x1f, 0x36, 0x8d, 0x69, 0x6a, + 0x81, 0x0a, 0x34, 0x9c, 0x0c, 0x71, 0x4c, 0x9a, + 0x5e, 0x78, 0x50, 0xc2, 0x40, 0x7d, 0x72, 0x1a, + 0xcd, 0xed, 0x95, 0xe0, 0x18, 0xd7, 0xa8, 0x52, + 0x66, 0xa6, 0xe1, 0x28, 0x9c, 0xdb, 0x4a, 0xeb, + 0x18, 0xda, 0x5a, 0xc8, 0xa2, 0xb0, 0x02, 0x6d, + 0x24, 0xa5, 0x9a, 0xd4, 0x85, 0x22, 0x7f, 0x3e, + 0xae, 0xdb, 0xb2, 0xe7, 0xe3, 0x5e, 0x1c, 0x66, + 0xcd, 0x60, 0xf9, 0xab, 0xf7, 0x16, 0xdc, 0xc9, + 0xac, 0x42, 0x68, 0x2d, 0xd7, 0xda, 0xb2, 0x87, + 0xa7, 0x02, 0x4c, 0x4e, 0xef, 0xc3, 0x21, 0xcc, + 0x05, 0x74, 0xe1, 0x67, 0x93, 0xe3, 0x7c, 0xec, + 0x03, 0xc5, 0xbd, 0xa4, 0x2b, 0x54, 0xc1, 0x14, + 0xa8, 0x0b, 0x57, 0xaf, 0x26, 0x41, 0x6c, 0x7b, + 0xe7, 0x42, 0x00, 0x5e, 0x20, 0x85, 0x5c, 0x73, + 0xe2, 0x1d, 0xc8, 0xe2, 0xed, 0xc9, 0xd4, 0x35, + 0xcb, 0x6f, 0x60, 0x59, 0x28, 0x00, 0x11, 0xc2, + 0x70, 0xb7, 0x15, 0x70, 0x05, 0x1c, 0x1c, 0x9b, + 0x30, 0x52, 0x12, 0x66, 0x20, 0xbc, 0x1e, 0x27, + 0x30, 0xfa, 0x06, 0x6c, 0x7a, 0x50, 0x9d, 0x53, + 0xc6, 0x0e, 0x5a, 0xe1, 0xb4, 0x0a, 0xa6, 0xe3, + 0x9e, 0x49, 0x66, 0x92, 0x28, 0xc9, 0x0e, 0xec, + 0xb4, 0xa5, 0x0d, 0xb3, 0x2a, 0x50, 0xbc, 0x49, + 0xe9, 0x0b, 0x4f, 0x4b, 0x35, 0x9a, 0x1d, 0xfd, + 0x11, 0x74, 0x9c, 0xd3, 0x86, 0x7f, 0xcf, 0x2f, + 0xb7, 0xbb, 0x6c, 0xd4, 0x73, 0x8f, 0x6a, 0x4a, + 0xd6, 0xf7, 0xca, 0x50, 0x58, 0xf7, 0x61, 0x88, + 0x45, 0xaf, 0x9f, 0x02, 0x0f, 0x6c, 0x3b, 0x96, + 0x7b, 0x8f, 0x4c, 0xd4, 0xa9, 0x1e, 0x28, 0x13, + 0xb5, 0x07, 0xae, 0x66, 0xf2, 0xd3, 0x5c, 0x18, + 0x28, 0x4f, 0x72, 0x92, 0x18, 0x60, 0x62, 0xe1, + 0x0f, 0xd5, 0x51, 0x0d, 0x18, 0x77, 0x53, 0x51, + 0xef, 0x33, 0x4e, 0x76, 0x34, 0xab, 0x47, 0x43, + 0xf5, 0xb6, 0x8f, 0x49, 0xad, 0xca, 0xb3, 0x84, + 0xd3, 0xfd, 0x75, 0xf7, 0x39, 0x0f, 0x40, 0x06, + 0xef, 0x2a, 0x29, 0x5c, 0x8c, 0x7a, 0x07, 0x6a, + 0xd5, 0x45, 0x46, 0xcd, 0x25, 0xd2, 0x10, 0x7f, + 0xbe, 0x14, 0x36, 0xc8, 0x40, 0x92, 0x4a, 0xae, + 0xbe, 0x5b, 0x37, 0x08, 0x93, 0xcd, 0x63, 0xd1, + 0x32, 0x5b, 0x86, 0x16, 0xfc, 0x48, 0x10, 0x88, + 0x6b, 0xc1, 0x52, 0xc5, 0x32, 0x21, 0xb6, 0xdf, + 0x37, 0x31, 0x19, 0x39, 0x32, 0x55, 0xee, 0x72, + 0xbc, 0xaa, 0x88, 0x01, 0x74, 0xf1, 0x71, 0x7f, + 0x91, 0x84, 0xfa, 0x91, 0x64, 0x6f, 0x17, 0xa2, + 0x4a, 0xc5, 0x5d, 0x16, 0xbf, 0xdd, 0xca, 0x95, + 0x81, 0xa9, 0x2e, 0xda, 0x47, 0x92, 0x01, 0xf0, + 0xed, 0xbf, 0x63, 0x36, 0x00, 0xd6, 0x06, 0x6d, + 0x1a, 0xb3, 0x6d, 0x5d, 0x24, 0x15, 0xd7, 0x13, + 0x51, 0xbb, 0xcd, 0x60, 0x8a, 0x25, 0x10, 0x8d, + 0x25, 0x64, 0x19, 0x92, 0xc1, 0xf2, 0x6c, 0x53, + 0x1c, 0xf9, 0xf9, 0x02, 0x03, 0xbc, 0x4c, 0xc1, + 0x9f, 0x59, 0x27, 0xd8, 0x34, 0xb0, 0xa4, 0x71, + 0x16, 0xd3, 0x88, 0x4b, 0xbb, 0x16, 0x4b, 0x8e, + 0xc8, 0x83, 0xd1, 0xac, 0x83, 0x2e, 0x56, 0xb3, + 0x91, 0x8a, 0x98, 0x60, 0x1a, 0x08, 0xd1, 0x71, + 0x88, 0x15, 0x41, 0xd5, 0x94, 0xdb, 0x39, 0x9c, + 0x6a, 0xe6, 0x15, 0x12, 0x21, 0x74, 0x5a, 0xec, + 0x81, 0x4c, 0x45, 0xb0, 0xb0, 0x5b, 0x56, 0x54, + 0x36, 0xfd, 0x6f, 0x13, 0x7a, 0xa1, 0x0a, 0x0c, + 0x0b, 0x64, 0x37, 0x61, 0xdb, 0xd6, 0xf9, 0xa9, + 0xdc, 0xb9, 0x9b, 0x1a, 0x6e, 0x69, 0x08, 0x54, + 0xce, 0x07, 0x69, 0xcd, 0xe3, 0x97, 0x61, 0xd8, + 0x2f, 0xcd, 0xec, 0x15, 0xf0, 0xd9, 0x2d, 0x7d, + 0x8e, 0x94, 0xad, 0xe8, 0xeb, 0x83, 0xfb, 0xe0 +}; +static const u8 output22[] __initconst = { + 0x26, 0x37, 0x40, 0x8f, 0xe1, 0x30, 0x86, 0xea, + 0x73, 0xf9, 0x71, 0xe3, 0x42, 0x5e, 0x28, 0x20 +}; +static const u8 key22[] __initconst = { + 0x99, 0xe5, 0x82, 0x2d, 0xd4, 0x17, 0x3c, 0x99, + 0x5e, 0x3d, 0xae, 0x0d, 0xde, 0xfb, 0x97, 0x74, + 0x3f, 0xde, 0x3b, 0x08, 0x01, 0x34, 0xb3, 0x9f, + 0x76, 0xe9, 0xbf, 0x8d, 0x0e, 0x88, 0xd5, 0x46 +}; + +/* test vectors from Hanno Böck */ +static const u8 input23[] __initconst = { + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0x80, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xc5, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe3, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xac, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe6, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x00, 0x00, 0x00, + 0xaf, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xff, 0xff, 0xff, 0xf5, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xe7, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x71, 0x92, 0x05, 0xa8, 0x52, 0x1d, + 0xfc +}; +static const u8 output23[] __initconst = { + 0x85, 0x59, 0xb8, 0x76, 0xec, 0xee, 0xd6, 0x6e, + 0xb3, 0x77, 0x98, 0xc0, 0x45, 0x7b, 0xaf, 0xf9 +}; +static const u8 key23[] __initconst = { + 0x7f, 0x1b, 0x02, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc +}; + +static const u8 input24[] __initconst = { + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x64 +}; +static const u8 output24[] __initconst = { + 0x00, 0xbd, 0x12, 0x58, 0x97, 0x8e, 0x20, 0x54, + 0x44, 0xc9, 0xaa, 0xaa, 0x82, 0x00, 0x6f, 0xed +}; +static const u8 key24[] __initconst = { + 0xe0, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa +}; + +static const u8 input25[] __initconst = { + 0x02, 0xfc +}; +static const u8 output25[] __initconst = { + 0x06, 0x12, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; +static const u8 key25[] __initconst = { + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; + +static const u8 input26[] __initconst = { + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x5c, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x6e, 0x7b, 0x00, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x5c, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x6e, 0x7b, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x7a, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc +}; +static const u8 output26[] __initconst = { + 0x33, 0x20, 0x5b, 0xbf, 0x9e, 0x9f, 0x8f, 0x72, + 0x12, 0xab, 0x9e, 0x2a, 0xb9, 0xb7, 0xe4, 0xa5 +}; +static const u8 key26[] __initconst = { + 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x7b +}; + +static const u8 input27[] __initconst = { + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0xff, 0xff, 0xff, 0xe9, + 0xe9, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x00, 0x00, 0xac, 0xac, + 0xec, 0x01, 0x00, 0xac, 0xac, 0xac, 0x2c, 0xac, + 0xa2, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x64, 0xf2 +}; +static const u8 output27[] __initconst = { + 0x02, 0xee, 0x7c, 0x8c, 0x54, 0x6d, 0xde, 0xb1, + 0xa4, 0x67, 0xe4, 0xc3, 0x98, 0x11, 0x58, 0xb9 +}; +static const u8 key27[] __initconst = { + 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x7f, + 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xcf, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77 +}; + +/* nacl */ +static const u8 input28[] __initconst = { + 0x8e, 0x99, 0x3b, 0x9f, 0x48, 0x68, 0x12, 0x73, + 0xc2, 0x96, 0x50, 0xba, 0x32, 0xfc, 0x76, 0xce, + 0x48, 0x33, 0x2e, 0xa7, 0x16, 0x4d, 0x96, 0xa4, + 0x47, 0x6f, 0xb8, 0xc5, 0x31, 0xa1, 0x18, 0x6a, + 0xc0, 0xdf, 0xc1, 0x7c, 0x98, 0xdc, 0xe8, 0x7b, + 0x4d, 0xa7, 0xf0, 0x11, 0xec, 0x48, 0xc9, 0x72, + 0x71, 0xd2, 0xc2, 0x0f, 0x9b, 0x92, 0x8f, 0xe2, + 0x27, 0x0d, 0x6f, 0xb8, 0x63, 0xd5, 0x17, 0x38, + 0xb4, 0x8e, 0xee, 0xe3, 0x14, 0xa7, 0xcc, 0x8a, + 0xb9, 0x32, 0x16, 0x45, 0x48, 0xe5, 0x26, 0xae, + 0x90, 0x22, 0x43, 0x68, 0x51, 0x7a, 0xcf, 0xea, + 0xbd, 0x6b, 0xb3, 0x73, 0x2b, 0xc0, 0xe9, 0xda, + 0x99, 0x83, 0x2b, 0x61, 0xca, 0x01, 0xb6, 0xde, + 0x56, 0x24, 0x4a, 0x9e, 0x88, 0xd5, 0xf9, 0xb3, + 0x79, 0x73, 0xf6, 0x22, 0xa4, 0x3d, 0x14, 0xa6, + 0x59, 0x9b, 0x1f, 0x65, 0x4c, 0xb4, 0x5a, 0x74, + 0xe3, 0x55, 0xa5 +}; +static const u8 output28[] __initconst = { + 0xf3, 0xff, 0xc7, 0x70, 0x3f, 0x94, 0x00, 0xe5, + 0x2a, 0x7d, 0xfb, 0x4b, 0x3d, 0x33, 0x05, 0xd9 +}; +static const u8 key28[] __initconst = { + 0xee, 0xa6, 0xa7, 0x25, 0x1c, 0x1e, 0x72, 0x91, + 0x6d, 0x11, 0xc2, 0xcb, 0x21, 0x4d, 0x3c, 0x25, + 0x25, 0x39, 0x12, 0x1d, 0x8e, 0x23, 0x4e, 0x65, + 0x2d, 0x65, 0x1f, 0xa4, 0xc8, 0xcf, 0xf8, 0x80 +}; + +/* wrap 2^130-5 */ +static const u8 input29[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output29[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key29[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* wrap 2^128 */ +static const u8 input30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output30[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +/* limb carry */ +static const u8 input31[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output31[] __initconst = { + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key31[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-5 */ +static const u8 input32[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfb, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; +static const u8 output32[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key32[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-6 */ +static const u8 input33[] __initconst = { + 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output33[] __initconst = { + 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 key33[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction intermediate */ +static const u8 input34[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output34[] __initconst = { + 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key34[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction final */ +static const u8 input35[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output35[] __initconst = { + 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key35[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const struct poly1305_testvec poly1305_testvecs[] __initconst = { + { input01, output01, key01, sizeof(input01) }, + { input02, output02, key02, sizeof(input02) }, + { input03, output03, key03, sizeof(input03) }, + { input04, output04, key04, sizeof(input04) }, + { input05, output05, key05, sizeof(input05) }, + { input06, output06, key06, sizeof(input06) }, + { input07, output07, key07, sizeof(input07) }, + { input08, output08, key08, sizeof(input08) }, + { input09, output09, key09, sizeof(input09) }, + { input10, output10, key10, sizeof(input10) }, + { input11, output11, key11, sizeof(input11) }, + { input12, output12, key12, sizeof(input12) }, + { input13, output13, key13, sizeof(input13) }, + { input14, output14, key14, sizeof(input14) }, + { input15, output15, key15, sizeof(input15) }, + { input16, output16, key16, sizeof(input16) }, + { input17, output17, key17, sizeof(input17) }, + { input18, output18, key18, sizeof(input18) }, + { input19, output19, key19, sizeof(input19) }, + { input20, output20, key20, sizeof(input20) }, + { input21, output21, key21, sizeof(input21) }, + { input22, output22, key22, sizeof(input22) }, + { input23, output23, key23, sizeof(input23) }, + { input24, output24, key24, sizeof(input24) }, + { input25, output25, key25, sizeof(input25) }, + { input26, output26, key26, sizeof(input26) }, + { input27, output27, key27, sizeof(input27) }, + { input28, output28, key28, sizeof(input28) }, + { input29, output29, key29, sizeof(input29) }, + { input30, output30, key30, sizeof(input30) }, + { input31, output31, key31, sizeof(input31) }, + { input32, output32, key32, sizeof(input32) }, + { input33, output33, key33, sizeof(input33) }, + { input34, output34, key34, sizeof(input34) }, + { input35, output35, key35, sizeof(input35) } +}; + +static bool __init poly1305_selftest(void) +{ + simd_context_t simd_context; + bool success = true; + size_t i, j; + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(poly1305_testvecs); ++i) { + struct poly1305_ctx poly1305; + u8 out[POLY1305_MAC_SIZE]; + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + poly1305_testvecs[i].ilen, &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu: FAIL\n", i + 1); + success = false; + } + simd_relax(&simd_context); + + if (poly1305_testvecs[i].ilen <= 1) + continue; + + for (j = 1; j < poly1305_testvecs[i].ilen - 1; ++j) { + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu): FAIL\n", + i + 1, j); + success = false; + } + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + DONT_USE_SIMD); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n", + i + 1, j); + success = false; + } + simd_relax(&simd_context); + } + } + simd_put(&simd_context); + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/run.h b/net/wireguard/crypto/zinc/selftest/run.h new file mode 100644 index 0000000000000..4ffaf6089eeaf --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/run.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_SELFTEST_RUN_H +#define _ZINC_SELFTEST_RUN_H + +#include +#include +#include + +static inline bool selftest_run(const char *name, bool (*selftest)(void), + bool *const nobs[], unsigned int nobs_len) +{ + unsigned long set = 0, subset = 0, largest_subset = 0; + unsigned int i; + + BUILD_BUG_ON(!__builtin_constant_p(nobs_len) || + nobs_len >= BITS_PER_LONG); + + if (!IS_ENABLED(CONFIG_ZINC_SELFTEST)) + return true; + + for (i = 0; i < nobs_len; ++i) + set |= ((unsigned long)*nobs[i]) << i; + + do { + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & subset; + if (selftest()) + largest_subset = max(subset, largest_subset); + else + pr_err("%s self-test combination 0x%lx: FAIL\n", name, + subset); + subset = (subset - set) & set; + } while (subset); + + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & largest_subset; + + if (largest_subset == set) + pr_info("%s self-tests: pass\n", name); + + return !WARN_ON(largest_subset != set); +} + +#endif diff --git a/net/wireguard/device.c b/net/wireguard/device.c new file mode 100644 index 0000000000000..e888ac37f9a89 --- /dev/null +++ b/net/wireguard/device.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "socket.h" +#include "timers.h" +#include "device.h" +#include "ratelimiter.h" +#include "peer.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(device_list); + +static int wg_open(struct net_device *dev) +{ + struct in_device *dev_v4 = __in_dev_get_rtnl(dev); +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + struct inet6_dev *dev_v6 = __in6_dev_get(dev); +#endif + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + int ret; + + if (dev_v4) { + /* At some point we might put this check near the ip_rt_send_ + * redirect call of ip_forward in net/ipv4/ip_forward.c, similar + * to the current secpath check. + */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + if (dev_v6) +#ifndef COMPAT_CANNOT_USE_DEV_CNF + dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#else + dev_v6->addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#endif +#endif + + ret = wg_socket_init(wg, wg->incoming_port); + if (ret < 0) + return ret; + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_send_staged_packets(peer); + if (peer->persistent_keepalive_interval) + wg_packet_send_keepalive(peer); + } + mutex_unlock(&wg->device_update_lock); + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int wg_pm_notification(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct wg_device *wg; + struct wg_peer *peer; + + /* If the machine is constantly suspending and resuming, as part of + * its normal operation rather than as a somewhat rare event, then we + * don't actually want to clear keys. + */ + if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) + return 0; + + if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) + return 0; + + rtnl_lock(); + list_for_each_entry(wg, &device_list, device_list) { + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + del_timer(&peer->timer_zero_key_material); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + } + mutex_unlock(&wg->device_update_lock); + } + rtnl_unlock(); + rcu_barrier(); + return 0; +} + +static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; +#endif + +static int wg_stop(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_purge_staged_packets(peer); + wg_timers_stop(peer); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + } + mutex_unlock(&wg->device_update_lock); + skb_queue_purge(&wg->incoming_handshakes); + wg_socket_reinit(wg, NULL, NULL); + return 0; +} + +static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct sk_buff_head packets; + struct wg_peer *peer; + struct sk_buff *next; + sa_family_t family; + u32 mtu; + int ret; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) { + ret = -EPROTONOSUPPORT; + net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); + goto err; + } + + peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); + if (unlikely(!peer)) { + ret = -ENOKEY; + if (skb->protocol == htons(ETH_P_IP)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", + dev->name, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", + dev->name, &ipv6_hdr(skb)->daddr); + goto err; + } + + family = READ_ONCE(peer->endpoint.addr.sa_family); + if (unlikely(family != AF_INET && family != AF_INET6)) { + ret = -EDESTADDRREQ; + net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", + dev->name, peer->internal_id); + goto err_peer; + } + + mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; + + __skb_queue_head_init(&packets); + if (!skb_is_gso(skb)) { + skb_mark_not_on_list(skb); + } else { + struct sk_buff *segs = skb_gso_segment(skb, 0); + + if (unlikely(IS_ERR(segs))) { + ret = PTR_ERR(segs); + goto err_peer; + } + dev_kfree_skb(skb); + skb = segs; + } + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + + skb = skb_share_check(skb, GFP_ATOMIC); + if (unlikely(!skb)) + continue; + + /* We only need to keep the original dst around for icmp, + * so at this point we're in a position to drop it. + */ + skb_dst_drop(skb); + + PACKET_CB(skb)->mtu = mtu; + + __skb_queue_tail(&packets, skb); + } + + spin_lock_bh(&peer->staged_packet_queue.lock); + /* If the queue is getting too big, we start removing the oldest packets + * until it's small again. We do this before adding the new packet, so + * we don't remove GSO segments that are in excess. + */ + while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { + dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); + ++dev->stats.tx_dropped; + } + skb_queue_splice_tail(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + wg_packet_send_staged_packets(peer); + + wg_peer_put(peer); + return NETDEV_TX_OK; + +err_peer: + wg_peer_put(peer); +err: + ++dev->stats.tx_errors; + if (skb->protocol == htons(ETH_P_IP)) + icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); + else if (skb->protocol == htons(ETH_P_IPV6)) + icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); + kfree_skb(skb); + return ret; +} + +static const struct net_device_ops netdev_ops = { + .ndo_open = wg_open, + .ndo_stop = wg_stop, + .ndo_start_xmit = wg_xmit, + .ndo_get_stats64 = ip_tunnel_get_stats64 +}; + +static void wg_destruct(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + + rtnl_lock(); + list_del(&wg->device_list); + rtnl_unlock(); + mutex_lock(&wg->device_update_lock); + wg->incoming_port = 0; + wg_socket_reinit(wg, NULL, NULL); + /* The final references are cleared in the below calls to destroy_workqueue. */ + wg_peer_remove_all(wg); + destroy_workqueue(wg->handshake_receive_wq); + destroy_workqueue(wg->handshake_send_wq); + destroy_workqueue(wg->packet_crypt_wq); + wg_packet_queue_free(&wg->decrypt_queue, true); + wg_packet_queue_free(&wg->encrypt_queue, true); + rcu_barrier(); /* Wait for all the peers to be actually freed. */ + wg_ratelimiter_uninit(); + memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); + skb_queue_purge(&wg->incoming_handshakes); + free_percpu(dev->tstats); + free_percpu(wg->incoming_handshakes_worker); + if (wg->have_creating_net_ref) + put_net(wg->creating_net); + kvfree(wg->index_hashtable); + kvfree(wg->peer_hashtable); + mutex_unlock(&wg->device_update_lock); + + pr_debug("%s: Interface deleted\n", dev->name); + free_netdev(dev); +} + +static const struct device_type device_type = { .name = KBUILD_MODNAME }; + +static void wg_setup(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | + NETIF_F_SG | NETIF_F_GSO | + NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; + + dev->netdev_ops = &netdev_ops; + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->needed_headroom = DATA_PACKET_HEAD_ROOM; + dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); + dev->type = ARPHRD_NONE; + dev->flags = IFF_POINTOPOINT | IFF_NOARP; +#ifndef COMPAT_CANNOT_USE_IFF_NO_QUEUE + dev->priv_flags |= IFF_NO_QUEUE; +#else + dev->tx_queue_len = 0; +#endif + dev->features |= NETIF_F_LLTX; + dev->features |= WG_NETDEV_FEATURES; + dev->hw_features |= WG_NETDEV_FEATURES; + dev->hw_enc_features |= WG_NETDEV_FEATURES; + dev->mtu = ETH_DATA_LEN - MESSAGE_MINIMUM_LENGTH - + sizeof(struct udphdr) - + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + + SET_NETDEV_DEVTYPE(dev, &device_type); + + /* We need to keep the dst around in case of icmp replies. */ + netif_keep_dst(dev); + + memset(wg, 0, sizeof(*wg)); + wg->dev = dev; +} + +static int wg_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct wg_device *wg = netdev_priv(dev); + int ret = -ENOMEM; + + wg->creating_net = src_net; + init_rwsem(&wg->static_identity.lock); + mutex_init(&wg->socket_update_lock); + mutex_init(&wg->device_update_lock); + skb_queue_head_init(&wg->incoming_handshakes); + wg_allowedips_init(&wg->peer_allowedips); + wg_cookie_checker_init(&wg->cookie_checker, wg); + INIT_LIST_HEAD(&wg->peer_list); + wg->device_update_gen = 1; + + wg->peer_hashtable = wg_pubkey_hashtable_alloc(); + if (!wg->peer_hashtable) + return ret; + + wg->index_hashtable = wg_index_hashtable_alloc(); + if (!wg->index_hashtable) + goto err_free_peer_hashtable; + + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_free_index_hashtable; + + wg->incoming_handshakes_worker = + wg_packet_percpu_multicore_worker_alloc( + wg_packet_handshake_receive_worker, wg); + if (!wg->incoming_handshakes_worker) + goto err_free_tstats; + + wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", + WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_receive_wq) + goto err_free_incoming_handshakes; + + wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", + WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_send_wq) + goto err_destroy_handshake_receive; + + wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); + if (!wg->packet_crypt_wq) + goto err_destroy_handshake_send; + + ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_destroy_packet_crypt; + + ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_free_encrypt_queue; + + ret = wg_ratelimiter_init(); + if (ret < 0) + goto err_free_decrypt_queue; + + ret = register_netdevice(dev); + if (ret < 0) + goto err_uninit_ratelimiter; + + list_add(&wg->device_list, &device_list); + + /* We wait until the end to assign priv_destructor, so that + * register_netdevice doesn't call it for us if it fails. + */ + dev->priv_destructor = wg_destruct; + + pr_debug("%s: Interface created\n", dev->name); + return ret; + +err_uninit_ratelimiter: + wg_ratelimiter_uninit(); +err_free_decrypt_queue: + wg_packet_queue_free(&wg->decrypt_queue, true); +err_free_encrypt_queue: + wg_packet_queue_free(&wg->encrypt_queue, true); +err_destroy_packet_crypt: + destroy_workqueue(wg->packet_crypt_wq); +err_destroy_handshake_send: + destroy_workqueue(wg->handshake_send_wq); +err_destroy_handshake_receive: + destroy_workqueue(wg->handshake_receive_wq); +err_free_incoming_handshakes: + free_percpu(wg->incoming_handshakes_worker); +err_free_tstats: + free_percpu(dev->tstats); +err_free_index_hashtable: + kvfree(wg->index_hashtable); +err_free_peer_hashtable: + kvfree(wg->peer_hashtable); + return ret; +} + +static struct rtnl_link_ops link_ops __read_mostly = { + .kind = KBUILD_MODNAME, + .priv_size = sizeof(struct wg_device), + .setup = wg_setup, + .newlink = wg_newlink, +}; + +static int wg_netdevice_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct net_device *dev = ((struct netdev_notifier_info *)data)->dev; + struct wg_device *wg = netdev_priv(dev); + + ASSERT_RTNL(); + + if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops) + return 0; + + if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) { + put_net(wg->creating_net); + wg->have_creating_net_ref = false; + } else if (dev_net(dev) != wg->creating_net && + !wg->have_creating_net_ref) { + wg->have_creating_net_ref = true; + get_net(wg->creating_net); + } + return 0; +} + +static struct notifier_block netdevice_notifier = { + .notifier_call = wg_netdevice_notification +}; + +int __init wg_device_init(void) +{ + int ret; + +#ifdef CONFIG_PM_SLEEP + ret = register_pm_notifier(&pm_notifier); + if (ret) + return ret; +#endif + + ret = register_netdevice_notifier(&netdevice_notifier); + if (ret) + goto error_pm; + + ret = rtnl_link_register(&link_ops); + if (ret) + goto error_netdevice; + + return 0; + +error_netdevice: + unregister_netdevice_notifier(&netdevice_notifier); +error_pm: +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + return ret; +} + +void wg_device_uninit(void) +{ + rtnl_link_unregister(&link_ops); + unregister_netdevice_notifier(&netdevice_notifier); +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + rcu_barrier(); +} diff --git a/net/wireguard/device.h b/net/wireguard/device.h new file mode 100644 index 0000000000000..c91f3051c5c78 --- /dev/null +++ b/net/wireguard/device.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_DEVICE_H +#define _WG_DEVICE_H + +#include "noise.h" +#include "allowedips.h" +#include "peerlookup.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include + +struct wg_device; + +struct multicore_worker { + void *ptr; + struct work_struct work; +}; + +struct crypt_queue { + struct ptr_ring ring; + union { + struct { + struct multicore_worker __percpu *worker; + int last_cpu; + }; + struct work_struct work; + }; +}; + +struct wg_device { + struct net_device *dev; + struct crypt_queue encrypt_queue, decrypt_queue; + struct sock __rcu *sock4, *sock6; + struct net *creating_net; + struct noise_static_identity static_identity; + struct workqueue_struct *handshake_receive_wq, *handshake_send_wq; + struct workqueue_struct *packet_crypt_wq; + struct sk_buff_head incoming_handshakes; + int incoming_handshake_cpu; + struct multicore_worker __percpu *incoming_handshakes_worker; + struct cookie_checker cookie_checker; + struct pubkey_hashtable *peer_hashtable; + struct index_hashtable *index_hashtable; + struct allowedips peer_allowedips; + struct mutex device_update_lock, socket_update_lock; + struct list_head device_list, peer_list; + unsigned int num_peers, device_update_gen; + u32 fwmark; + u16 incoming_port; + bool have_creating_net_ref; +}; + +int wg_device_init(void); +void wg_device_uninit(void); + +/* Later after the dust settles, this can be moved into include/linux/skbuff.h, + * where virtually all code that deals with GSO segs can benefit, around ~30 + * drivers as of writing. + */ +#define skb_list_walk_safe(first, skb, next) \ + for (skb = first, next = skb->next; skb; \ + skb = next, next = skb ? skb->next : NULL) + +#endif /* _WG_DEVICE_H */ diff --git a/net/wireguard/main.c b/net/wireguard/main.c new file mode 100644 index 0000000000000..9ca4195bc9850 --- /dev/null +++ b/net/wireguard/main.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "version.h" +#include "device.h" +#include "noise.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "netlink.h" +#include "uapi/wireguard.h" +#include "crypto/zinc.h" + +#include +#include +#include +#include + +static int __init mod_init(void) +{ + int ret; + + if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) || + (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) || + (ret = curve25519_mod_init())) + return ret; + +#ifdef DEBUG + if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || + !wg_ratelimiter_selftest()) + return -ENOTRECOVERABLE; +#endif + wg_noise_init(); + + ret = wg_device_init(); + if (ret < 0) + goto err_device; + + ret = wg_genetlink_init(); + if (ret < 0) + goto err_netlink; + + pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); + pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + + return 0; + +err_netlink: + wg_device_uninit(); +err_device: + return ret; +} + +static void __exit mod_exit(void) +{ + wg_genetlink_uninit(); + wg_device_uninit(); +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("WireGuard secure network tunnel"); +MODULE_AUTHOR("Jason A. Donenfeld "); +MODULE_VERSION(WIREGUARD_VERSION); +MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); +MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); diff --git a/net/wireguard/messages.h b/net/wireguard/messages.h new file mode 100644 index 0000000000000..f415cdd2e90a9 --- /dev/null +++ b/net/wireguard/messages.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_MESSAGES_H +#define _WG_MESSAGES_H + +#include +#include +#include + +#include +#include +#include + +enum noise_lengths { + NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, + NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, + NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32), + NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, + NOISE_HASH_LEN = BLAKE2S_HASH_SIZE +}; + +#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN) + +enum cookie_values { + COOKIE_SECRET_MAX_AGE = 2 * 60, + COOKIE_SECRET_LATENCY = 5, + COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, + COOKIE_LEN = 16 +}; + +enum counter_values { + COUNTER_BITS_TOTAL = 2048, + COUNTER_REDUNDANT_BITS = BITS_PER_LONG, + COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS +}; + +enum limits { + REKEY_AFTER_MESSAGES = 1ULL << 60, + REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1, + REKEY_TIMEOUT = 5, + REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3, + REKEY_AFTER_TIME = 120, + REJECT_AFTER_TIME = 180, + INITIATIONS_PER_SECOND = 50, + MAX_PEERS_PER_DEVICE = 1U << 20, + KEEPALIVE_TIMEOUT = 10, + MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT, + MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */ + MAX_STAGED_PACKETS = 128, + MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */ +}; + +enum message_type { + MESSAGE_INVALID = 0, + MESSAGE_HANDSHAKE_INITIATION = 1, + MESSAGE_HANDSHAKE_RESPONSE = 2, + MESSAGE_HANDSHAKE_COOKIE = 3, + MESSAGE_DATA = 4 +}; + +struct message_header { + /* The actual layout of this that we want is: + * u8 type + * u8 reserved_zero[3] + * + * But it turns out that by encoding this as little endian, + * we achieve the same thing, and it makes checking faster. + */ + __le32 type; +}; + +struct message_macs { + u8 mac1[COOKIE_LEN]; + u8 mac2[COOKIE_LEN]; +}; + +struct message_handshake_initiation { + struct message_header header; + __le32 sender_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; + u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; + struct message_macs macs; +}; + +struct message_handshake_response { + struct message_header header; + __le32 sender_index; + __le32 receiver_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_nothing[noise_encrypted_len(0)]; + struct message_macs macs; +}; + +struct message_handshake_cookie { + struct message_header header; + __le32 receiver_index; + u8 nonce[COOKIE_NONCE_LEN]; + u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)]; +}; + +struct message_data { + struct message_header header; + __le32 key_idx; + __le64 counter; + u8 encrypted_data[]; +}; + +#define message_data_len(plain_len) \ + (noise_encrypted_len(plain_len) + sizeof(struct message_data)) + +enum message_alignments { + MESSAGE_PADDING_MULTIPLE = 16, + MESSAGE_MINIMUM_LENGTH = message_data_len(0) +}; + +#define SKB_HEADER_LEN \ + (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ + sizeof(struct udphdr) + NET_SKB_PAD) +#define DATA_PACKET_HEAD_ROOM \ + ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4) + +enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ }; + +#endif /* _WG_MESSAGES_H */ diff --git a/net/wireguard/netlink.c b/net/wireguard/netlink.c new file mode 100644 index 0000000000000..9bf2f84edda81 --- /dev/null +++ b/net/wireguard/netlink.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "netlink.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" +#include "uapi/wireguard.h" +#include +#include +#include +#include + +static struct genl_family genl_family; + +static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { + [WGDEVICE_A_IFINDEX] = { .type = NLA_U32 }, + [WGDEVICE_A_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, + [WGDEVICE_A_PRIVATE_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, + [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, + [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, + [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } +}; + +static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { + [WGPEER_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGPEER_A_PRESHARED_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN }, + [WGPEER_A_FLAGS] = { .type = NLA_U32 }, + [WGPEER_A_ENDPOINT] = { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) }, + [WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL] = { .type = NLA_U16 }, + [WGPEER_A_LAST_HANDSHAKE_TIME] = { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) }, + [WGPEER_A_RX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_TX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_ALLOWEDIPS] = { .type = NLA_NESTED }, + [WGPEER_A_PROTOCOL_VERSION] = { .type = NLA_U32 } +}; + +static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { + [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, + [WGALLOWEDIP_A_IPADDR] = { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) }, + [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } +}; + +static struct wg_device *lookup_interface(struct nlattr **attrs, + struct sk_buff *skb) +{ + struct net_device *dev = NULL; + + if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) + return ERR_PTR(-EBADR); + if (attrs[WGDEVICE_A_IFINDEX]) + dev = dev_get_by_index(sock_net(skb->sk), + nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); + else if (attrs[WGDEVICE_A_IFNAME]) + dev = dev_get_by_name(sock_net(skb->sk), + nla_data(attrs[WGDEVICE_A_IFNAME])); + if (!dev) + return ERR_PTR(-ENODEV); + if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || + strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) { + dev_put(dev); + return ERR_PTR(-EOPNOTSUPP); + } + return netdev_priv(dev); +} + +static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, + int family) +{ + struct nlattr *allowedip_nest; + + allowedip_nest = nla_nest_start(skb, 0); + if (!allowedip_nest) + return -EMSGSIZE; + + if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || + nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || + nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? + sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { + nla_nest_cancel(skb, allowedip_nest); + return -EMSGSIZE; + } + + nla_nest_end(skb, allowedip_nest); + return 0; +} + +struct dump_ctx { + struct wg_device *wg; + struct wg_peer *next_peer; + u64 allowedips_seq; + struct allowedips_node *next_allowedip; +}; + +#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) + +static int +get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) +{ + + struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); + struct allowedips_node *allowedips_node = ctx->next_allowedip; + bool fail; + + if (!peer_nest) + return -EMSGSIZE; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, + peer->handshake.remote_static); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (!allowedips_node) { + const struct __kernel_timespec last_handshake = { + .tv_sec = peer->walltime_last_handshake.tv_sec, + .tv_nsec = peer->walltime_last_handshake.tv_nsec + }; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, + NOISE_SYMMETRIC_KEY_LEN, + peer->handshake.preshared_key); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, + sizeof(last_handshake), &last_handshake) || + nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + peer->persistent_keepalive_interval) || + nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) + goto err; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr4), + &peer->endpoint.addr4); + else if (peer->endpoint.addr.sa_family == AF_INET6) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr6), + &peer->endpoint.addr6); + read_unlock_bh(&peer->endpoint_lock); + if (fail) + goto err; + allowedips_node = + list_first_entry_or_null(&peer->allowedips_list, + struct allowedips_node, peer_list); + } + if (!allowedips_node) + goto no_allowedips; + if (!ctx->allowedips_seq) + ctx->allowedips_seq = peer->device->peer_allowedips.seq; + else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq) + goto no_allowedips; + + allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS); + if (!allowedips_nest) + goto err; + + list_for_each_entry_from(allowedips_node, &peer->allowedips_list, + peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family; + + family = wg_allowedips_read_node(allowedips_node, ip, &cidr); + if (get_allowedips(skb, ip, cidr, family)) { + nla_nest_end(skb, allowedips_nest); + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = allowedips_node; + return -EMSGSIZE; + } + } + nla_nest_end(skb, allowedips_nest); +no_allowedips: + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = NULL; + ctx->allowedips_seq = 0; + return 0; +err: + nla_nest_cancel(skb, peer_nest); + return -EMSGSIZE; +} + +static int wg_get_device_start(struct netlink_callback *cb) +{ + struct wg_device *wg; + + wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); + if (IS_ERR(wg)) + return PTR_ERR(wg); + DUMP_CTX(cb)->wg = wg; + return 0; +} + +static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct wg_peer *peer, *next_peer_cursor; + struct dump_ctx *ctx = DUMP_CTX(cb); + struct wg_device *wg = ctx->wg; + struct nlattr *peers_nest; + int ret = -EMSGSIZE; + bool done = true; + void *hdr; + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + cb->seq = wg->device_update_gen; + next_peer_cursor = ctx->next_peer; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); + if (!hdr) + goto out; + genl_dump_check_consistent(cb, hdr); + + if (!ctx->next_peer) { + if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, + wg->incoming_port) || + nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || + nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || + nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) + goto out; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity) { + if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_private) || + nla_put(skb, WGDEVICE_A_PUBLIC_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_public)) { + up_read(&wg->static_identity.lock); + goto out; + } + } + up_read(&wg->static_identity.lock); + } + + peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); + if (!peers_nest) + goto out; + ret = 0; + /* If the last cursor was removed via list_del_init in peer_remove, then + * we just treat this the same as there being no more peers left. The + * reason is that seq_nr should indicate to userspace that this isn't a + * coherent dump anyway, so they'll try again. + */ + if (list_empty(&wg->peer_list) || + (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) { + nla_nest_cancel(skb, peers_nest); + goto out; + } + lockdep_assert_held(&wg->device_update_lock); + peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list); + list_for_each_entry_continue(peer, &wg->peer_list, peer_list) { + if (get_peer(peer, skb, ctx)) { + done = false; + break; + } + next_peer_cursor = peer; + } + nla_nest_end(skb, peers_nest); + +out: + if (!ret && !done && next_peer_cursor) + wg_peer_get(next_peer_cursor); + wg_peer_put(ctx->next_peer); + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + + if (ret) { + genlmsg_cancel(skb, hdr); + return ret; + } + genlmsg_end(skb, hdr); + if (done) { + ctx->next_peer = NULL; + return 0; + } + ctx->next_peer = next_peer_cursor; + return skb->len; + + /* At this point, we can't really deal ourselves with safely zeroing out + * the private key material after usage. This will need an additional API + * in the kernel for marking skbs as zero_on_free. + */ +} + +static int wg_get_device_done(struct netlink_callback *cb) +{ + struct dump_ctx *ctx = DUMP_CTX(cb); + + if (ctx->wg) + dev_put(ctx->wg->dev); + wg_peer_put(ctx->next_peer); + return 0; +} + +static int set_port(struct wg_device *wg, u16 port) +{ + struct wg_peer *peer; + + if (wg->incoming_port == port) + return 0; + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + if (!netif_running(wg->dev)) { + wg->incoming_port = port; + return 0; + } + return wg_socket_init(wg, port); +} + +static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) +{ + int ret = -EINVAL; + u16 family; + u8 cidr; + + if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || + !attrs[WGALLOWEDIP_A_CIDR_MASK]) + return ret; + family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); + cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); + + if (family == AF_INET && cidr <= 32 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) + ret = wg_allowedips_insert_v4( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else if (family == AF_INET6 && cidr <= 128 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) + ret = wg_allowedips_insert_v6( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + + return ret; +} + +static int set_peer(struct wg_device *wg, struct nlattr **attrs) +{ + u8 *public_key = NULL, *preshared_key = NULL; + struct wg_peer *peer = NULL; + u32 flags = 0; + int ret; + + ret = -EINVAL; + if (attrs[WGPEER_A_PUBLIC_KEY] && + nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) + public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]); + else + goto out; + if (attrs[WGPEER_A_PRESHARED_KEY] && + nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN) + preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]); + + if (attrs[WGPEER_A_FLAGS]) + flags = nla_get_u32(attrs[WGPEER_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGPEER_F_ALL) + goto out; + + ret = -EPFNOSUPPORT; + if (attrs[WGPEER_A_PROTOCOL_VERSION]) { + if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1) + goto out; + } + + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + nla_data(attrs[WGPEER_A_PUBLIC_KEY])); + ret = 0; + if (!peer) { /* Peer doesn't exist yet. Add a new one. */ + if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) + goto out; + + /* The peer is new, so there aren't allowed IPs to remove. */ + flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity && + !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), + wg->static_identity.static_public, + NOISE_PUBLIC_KEY_LEN)) { + /* We silently ignore peers that have the same public + * key as the device. The reason we do it silently is + * that we'd like for people to be able to reuse the + * same set of API calls across peers. + */ + up_read(&wg->static_identity.lock); + ret = 0; + goto out; + } + up_read(&wg->static_identity.lock); + + peer = wg_peer_create(wg, public_key, preshared_key); + if (IS_ERR(peer)) { + /* Similar to the above, if the key is invalid, we skip + * it without fanfare, so that services don't need to + * worry about doing key validation themselves. + */ + ret = PTR_ERR(peer) == -EKEYREJECTED ? 0 : PTR_ERR(peer); + peer = NULL; + goto out; + } + /* Take additional reference, as though we've just been + * looked up. + */ + wg_peer_get(peer); + } + + if (flags & WGPEER_F_REMOVE_ME) { + wg_peer_remove(peer); + goto out; + } + + if (preshared_key) { + down_write(&peer->handshake.lock); + memcpy(&peer->handshake.preshared_key, preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + up_write(&peer->handshake.lock); + } + + if (attrs[WGPEER_A_ENDPOINT]) { + struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); + size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); + + if ((len == sizeof(struct sockaddr_in) && + addr->sa_family == AF_INET) || + (len == sizeof(struct sockaddr_in6) && + addr->sa_family == AF_INET6)) { + struct endpoint endpoint = { { { 0 } } }; + + memcpy(&endpoint.addr, addr, len); + wg_socket_set_peer_endpoint(peer, &endpoint); + } + } + + if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) + wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, + &wg->device_update_lock); + + if (attrs[WGPEER_A_ALLOWEDIPS]) { + struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { + ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, + attr, allowedip_policy, NULL); + if (ret < 0) + goto out; + ret = set_allowedip(peer, allowedip); + if (ret < 0) + goto out; + } + } + + if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { + const u16 persistent_keepalive_interval = nla_get_u16( + attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); + const bool send_keepalive = + !peer->persistent_keepalive_interval && + persistent_keepalive_interval && + netif_running(wg->dev); + + peer->persistent_keepalive_interval = persistent_keepalive_interval; + if (send_keepalive) + wg_packet_send_keepalive(peer); + } + + if (netif_running(wg->dev)) + wg_packet_send_staged_packets(peer); + +out: + wg_peer_put(peer); + if (attrs[WGPEER_A_PRESHARED_KEY]) + memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), + nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + return ret; +} + +static int wg_set_device(struct sk_buff *skb, struct genl_info *info) +{ + struct wg_device *wg = lookup_interface(info->attrs, skb); + u32 flags = 0; + int ret; + + if (IS_ERR(wg)) { + ret = PTR_ERR(wg); + goto out_nodev; + } + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + + if (info->attrs[WGDEVICE_A_FLAGS]) + flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGDEVICE_F_ALL) + goto out; + + ret = -EPERM; + if ((info->attrs[WGDEVICE_A_LISTEN_PORT] || + info->attrs[WGDEVICE_A_FWMARK]) && + !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN)) + goto out; + + ++wg->device_update_gen; + + if (info->attrs[WGDEVICE_A_FWMARK]) { + struct wg_peer *peer; + + wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + } + + if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { + ret = set_port(wg, + nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); + if (ret) + goto out; + } + + if (flags & WGDEVICE_F_REPLACE_PEERS) + wg_peer_remove_all(wg); + + if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == + NOISE_PUBLIC_KEY_LEN) { + u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); + u8 public_key[NOISE_PUBLIC_KEY_LEN]; + struct wg_peer *peer, *temp; + + if (!crypto_memneq(wg->static_identity.static_private, + private_key, NOISE_PUBLIC_KEY_LEN)) + goto skip_set_private_key; + + /* We remove before setting, to prevent race, which means doing + * two 25519-genpub ops. + */ + if (curve25519_generate_public(public_key, private_key)) { + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + public_key); + if (peer) { + wg_peer_put(peer); + wg_peer_remove(peer); + } + } + + down_write(&wg->static_identity.lock); + wg_noise_set_static_identity_private_key(&wg->static_identity, + private_key); + list_for_each_entry_safe(peer, temp, &wg->peer_list, + peer_list) { + if (wg_noise_precompute_static_static(peer)) + wg_noise_expire_current_peer_keypairs(peer); + else + wg_peer_remove(peer); + } + wg_cookie_checker_precompute_device_keys(&wg->cookie_checker); + up_write(&wg->static_identity.lock); + } +skip_set_private_key: + + if (info->attrs[WGDEVICE_A_PEERS]) { + struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { + ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, + peer_policy, NULL); + if (ret < 0) + goto out; + ret = set_peer(wg, peer); + if (ret < 0) + goto out; + } + } + ret = 0; + +out: + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + dev_put(wg->dev); +out_nodev: + if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) + memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + return ret; +} + +#ifndef COMPAT_CANNOT_USE_CONST_GENL_OPS +static const +#else +static +#endif +struct genl_ops genl_ops[] = { + { + .cmd = WG_CMD_GET_DEVICE, +#ifndef COMPAT_CANNOT_USE_NETLINK_START + .start = wg_get_device_start, +#endif + .dumpit = wg_get_device_dump, + .done = wg_get_device_done, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + }, { + .cmd = WG_CMD_SET_DEVICE, + .doit = wg_set_device, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + } +}; + +static struct genl_family genl_family +#ifndef COMPAT_CANNOT_USE_GENL_NOPS +__ro_after_init = { + .ops = genl_ops, + .n_ops = ARRAY_SIZE(genl_ops), +#else += { +#endif + .name = WG_GENL_NAME, + .version = WG_GENL_VERSION, + .maxattr = WGDEVICE_A_MAX, + .module = THIS_MODULE, +#ifndef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .netnsok = true +}; + +int __init wg_genetlink_init(void) +{ + return genl_register_family(&genl_family); +} + +void __exit wg_genetlink_uninit(void) +{ + genl_unregister_family(&genl_family); +} diff --git a/net/wireguard/netlink.h b/net/wireguard/netlink.h new file mode 100644 index 0000000000000..15100d92e2e32 --- /dev/null +++ b/net/wireguard/netlink.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_NETLINK_H +#define _WG_NETLINK_H + +int wg_genetlink_init(void); +void wg_genetlink_uninit(void); + +#endif /* _WG_NETLINK_H */ diff --git a/net/wireguard/noise.c b/net/wireguard/noise.c new file mode 100644 index 0000000000000..269b69faf11a4 --- /dev/null +++ b/net/wireguard/noise.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "noise.h" +#include "device.h" +#include "peer.h" +#include "messages.h" +#include "queueing.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +/* This implements Noise_IKpsk2: + * + * <- s + * ****** + * -> e, es, s, ss, {t} + * <- e, ee, se, psk, {} + */ + +static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; +static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; +static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; +static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; +static atomic64_t keypair_counter = ATOMIC64_INIT(0); + +void __init wg_noise_init(void) +{ + struct blake2s_state blake; + + blake2s(handshake_init_chaining_key, handshake_name, NULL, + NOISE_HASH_LEN, sizeof(handshake_name), 0); + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s_update(&blake, identifier_name, sizeof(identifier_name)); + blake2s_final(&blake, handshake_init_hash); +} + +/* Must hold peer->handshake.static_identity->lock */ +bool wg_noise_precompute_static_static(struct wg_peer *peer) +{ + bool ret = true; + + down_write(&peer->handshake.lock); + if (peer->handshake.static_identity->has_identity) + ret = curve25519( + peer->handshake.precomputed_static_static, + peer->handshake.static_identity->static_private, + peer->handshake.remote_static); + else + memset(peer->handshake.precomputed_static_static, 0, + NOISE_PUBLIC_KEY_LEN); + up_write(&peer->handshake.lock); + return ret; +} + +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer) +{ + memset(handshake, 0, sizeof(*handshake)); + init_rwsem(&handshake->lock); + handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; + handshake->entry.peer = peer; + memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); + if (peer_preshared_key) + memcpy(handshake->preshared_key, peer_preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + handshake->static_identity = static_identity; + handshake->state = HANDSHAKE_ZEROED; + return wg_noise_precompute_static_static(peer); +} + +static void handshake_zero(struct noise_handshake *handshake) +{ + memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->hash, 0, NOISE_HASH_LEN); + memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); + handshake->remote_index = 0; + handshake->state = HANDSHAKE_ZEROED; +} + +void wg_noise_handshake_clear(struct noise_handshake *handshake) +{ + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + down_write(&handshake->lock); + handshake_zero(handshake); + up_write(&handshake->lock); + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); +} + +static struct noise_keypair *keypair_create(struct wg_peer *peer) +{ + struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); + + if (unlikely(!keypair)) + return NULL; + keypair->internal_id = atomic64_inc_return(&keypair_counter); + keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; + keypair->entry.peer = peer; + kref_init(&keypair->refcount); + return keypair; +} + +static void keypair_free_rcu(struct rcu_head *rcu) +{ + kzfree(container_of(rcu, struct noise_keypair, rcu)); +} + +static void keypair_free_kref(struct kref *kref) +{ + struct noise_keypair *keypair = + container_of(kref, struct noise_keypair, refcount); + + net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", + keypair->entry.peer->device->dev->name, + keypair->internal_id, + keypair->entry.peer->internal_id); + wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, + &keypair->entry); + call_rcu(&keypair->rcu, keypair_free_rcu); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) +{ + if (unlikely(!keypair)) + return; + if (unlikely(unreference_now)) + wg_index_hashtable_remove( + keypair->entry.peer->device->index_hashtable, + &keypair->entry); + kref_put(&keypair->refcount, keypair_free_kref); +} + +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking noise keypair reference without holding the RCU BH read lock"); + if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) + return NULL; + return keypair; +} + +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) +{ + struct noise_keypair *old; + + spin_lock_bh(&keypairs->keypair_update_lock); + + /* We zero the next_keypair before zeroing the others, so that + * wg_noise_received_with_keypair returns early before subsequent ones + * are zeroed. + */ + old = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->current_keypair, NULL); + wg_noise_keypair_put(old, true); + + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + + wg_noise_handshake_clear(&peer->handshake); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + + spin_lock_bh(&peer->keypairs.keypair_update_lock); + keypair = rcu_dereference_protected(peer->keypairs.next_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + keypair = rcu_dereference_protected(peer->keypairs.current_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + spin_unlock_bh(&peer->keypairs.keypair_update_lock); +} + +static void add_new_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *new_keypair) +{ + struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; + + spin_lock_bh(&keypairs->keypair_update_lock); + previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + next_keypair = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + current_keypair = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + if (new_keypair->i_am_the_initiator) { + /* If we're the initiator, it means we've sent a handshake, and + * received a confirmation response, which means this new + * keypair can now be used. + */ + if (next_keypair) { + /* If there already was a next keypair pending, we + * demote it to be the previous keypair, and free the + * existing current. Note that this means KCI can result + * in this transition. It would perhaps be more sound to + * always just get rid of the unused next keypair + * instead of putting it in the previous slot, but this + * might be a bit less robust. Something to think about + * for the future. + */ + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + rcu_assign_pointer(keypairs->previous_keypair, + next_keypair); + wg_noise_keypair_put(current_keypair, true); + } else /* If there wasn't an existing next keypair, we replace + * the previous with the current one. + */ + rcu_assign_pointer(keypairs->previous_keypair, + current_keypair); + /* At this point we can get rid of the old previous keypair, and + * set up the new keypair. + */ + wg_noise_keypair_put(previous_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, new_keypair); + } else { + /* If we're the responder, it means we can't use the new keypair + * until we receive confirmation via the first data packet, so + * we get rid of the existing previous one, the possibly + * existing next one, and slide in the new next one. + */ + rcu_assign_pointer(keypairs->next_keypair, new_keypair); + wg_noise_keypair_put(next_keypair, true); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(previous_keypair, true); + } + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair) +{ + struct noise_keypair *old_keypair; + bool key_is_new; + + /* We first check without taking the spinlock. */ + key_is_new = received_keypair == + rcu_access_pointer(keypairs->next_keypair); + if (likely(!key_is_new)) + return false; + + spin_lock_bh(&keypairs->keypair_update_lock); + /* After locking, we double check that things didn't change from + * beneath us. + */ + if (unlikely(received_keypair != + rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)))) { + spin_unlock_bh(&keypairs->keypair_update_lock); + return false; + } + + /* When we've finally received the confirmation, we slide the next + * into the current, the current into the previous, and get rid of + * the old previous. + */ + old_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + rcu_assign_pointer(keypairs->previous_keypair, + rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock))); + wg_noise_keypair_put(old_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, received_keypair); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + + spin_unlock_bh(&keypairs->keypair_update_lock); + return true; +} + +/* Must hold static_identity->lock */ +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(static_identity->static_private, private_key, + NOISE_PUBLIC_KEY_LEN); + curve25519_clamp_secret(static_identity->static_private); + static_identity->has_identity = curve25519_generate_public( + static_identity->static_public, private_key); +} + +/* This is Hugo Krawczyk's HKDF: + * - https://eprint.iacr.org/2010/264.pdf + * - https://tools.ietf.org/html/rfc5869 + */ +static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, + size_t first_len, size_t second_len, size_t third_len, + size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) +{ + u8 output[BLAKE2S_HASH_SIZE + 1]; + u8 secret[BLAKE2S_HASH_SIZE]; + + WARN_ON(IS_ENABLED(DEBUG) && + (first_len > BLAKE2S_HASH_SIZE || + second_len > BLAKE2S_HASH_SIZE || + third_len > BLAKE2S_HASH_SIZE || + ((second_len || second_dst || third_len || third_dst) && + (!first_len || !first_dst)) || + ((third_len || third_dst) && (!second_len || !second_dst)))); + + /* Extract entropy from data into secret */ + blake2s_hmac(secret, data, chaining_key, BLAKE2S_HASH_SIZE, data_len, + NOISE_HASH_LEN); + + if (!first_dst || !first_len) + goto out; + + /* Expand first key: key = secret, data = 0x1 */ + output[0] = 1; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, 1, + BLAKE2S_HASH_SIZE); + memcpy(first_dst, output, first_len); + + if (!second_dst || !second_len) + goto out; + + /* Expand second key: key = secret, data = first-key || 0x2 */ + output[BLAKE2S_HASH_SIZE] = 2; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(second_dst, output, second_len); + + if (!third_dst || !third_len) + goto out; + + /* Expand third key: key = secret, data = second-key || 0x3 */ + output[BLAKE2S_HASH_SIZE] = 3; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(third_dst, output, third_len); + +out: + /* Clear sensitive data from stack */ + memzero_explicit(secret, BLAKE2S_HASH_SIZE); + memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); +} + +static void symmetric_key_init(struct noise_symmetric_key *key) +{ + spin_lock_init(&key->counter.receive.lock); + atomic64_set(&key->counter.counter, 0); + memset(key->counter.receive.backtrack, 0, + sizeof(key->counter.receive.backtrack)); + key->birthdate = ktime_get_coarse_boottime_ns(); + key->is_valid = true; +} + +static void derive_keys(struct noise_symmetric_key *first_dst, + struct noise_symmetric_key *second_dst, + const u8 chaining_key[NOISE_HASH_LEN]) +{ + kdf(first_dst->key, second_dst->key, NULL, NULL, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, + chaining_key); + symmetric_key_init(first_dst); + symmetric_key_init(second_dst); +} + +static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 private[NOISE_PUBLIC_KEY_LEN], + const u8 public[NOISE_PUBLIC_KEY_LEN]) +{ + u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; + + if (unlikely(!curve25519(dh_calculation, private, public))) + return false; + kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); + memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); + return true; +} + +static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, hash, NOISE_HASH_LEN); + blake2s_update(&blake, src, src_len); + blake2s_final(&blake, hash); +} + +static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) +{ + u8 temp_hash[NOISE_HASH_LEN]; + + kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); + mix_hash(hash, temp_hash, NOISE_HASH_LEN); + memzero_explicit(temp_hash, NOISE_HASH_LEN); +} + +static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN], + const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); + mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); +} + +static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, + NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key); + mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); +} + +static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, + hash, NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key)) + return false; + mix_hash(hash, src_ciphertext, src_len); + return true; +} + +static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], + const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], + u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (ephemeral_dst != ephemeral_src) + memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, + NOISE_PUBLIC_KEY_LEN, chaining_key); +} + +static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) +{ + struct timespec64 now; + + ktime_get_real_ts64(&now); + + /* In order to prevent some sort of infoleak from precise timers, we + * round down the nanoseconds part to the closest rounded-down power of + * two to the maximum initiations per second allowed anyway by the + * implementation. + */ + now.tv_nsec = ALIGN_DOWN(now.tv_nsec, + rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); + + /* https://cr.yp.to/libtai/tai64.html */ + *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); + *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); +} + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake) +{ + u8 timestamp[NOISE_TIMESTAMP_LEN]; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (unlikely(!handshake->static_identity->has_identity)) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); + + handshake_init(handshake->chaining_key, handshake->hash, + handshake->remote_static); + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* es */ + if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* s */ + message_encrypt(dst->encrypted_static, + handshake->static_identity->static_public, + NOISE_PUBLIC_KEY_LEN, key, handshake->hash); + + /* ss */ + kdf(handshake->chaining_key, key, NULL, + handshake->precomputed_static_static, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + handshake->chaining_key); + + /* {t} */ + tai64n_now(timestamp); + message_encrypt(dst->encrypted_timestamp, timestamp, + NOISE_TIMESTAMP_LEN, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_INITIATION; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + bool replay_attack, flood_attack; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 s[NOISE_PUBLIC_KEY_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 t[NOISE_TIMESTAMP_LEN]; + u64 initiation_consumption; + + down_read(&wg->static_identity.lock); + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake_init(chaining_key, hash, wg->static_identity.static_public); + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* es */ + if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) + goto out; + + /* s */ + if (!message_decrypt(s, src->encrypted_static, + sizeof(src->encrypted_static), key, hash)) + goto out; + + /* Lookup which peer we're actually talking to */ + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); + if (!peer) + goto out; + handshake = &peer->handshake; + + /* ss */ + kdf(chaining_key, key, NULL, handshake->precomputed_static_static, + NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + chaining_key); + + /* {t} */ + if (!message_decrypt(t, src->encrypted_timestamp, + sizeof(src->encrypted_timestamp), key, hash)) + goto out; + + down_read(&handshake->lock); + replay_attack = memcmp(t, handshake->latest_timestamp, + NOISE_TIMESTAMP_LEN) <= 0; + flood_attack = (s64)handshake->last_initiation_consumption + + NSEC_PER_SEC / INITIATIONS_PER_SECOND > + (s64)ktime_get_coarse_boottime_ns(); + up_read(&handshake->lock); + if (replay_attack || flood_attack) + goto out; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) + memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + if ((s64)(handshake->last_initiation_consumption - + (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0) + handshake->last_initiation_consumption = initiation_consumption; + handshake->state = HANDSHAKE_CONSUMED_INITIATION; + up_write(&handshake->lock); + ret_peer = peer; + +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + up_read(&wg->static_identity.lock); + if (!ret_peer) + wg_peer_put(peer); + return ret_peer; +} + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake) +{ + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); + dst->receiver_index = handshake->remote_index; + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* ee */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_ephemeral)) + goto out; + + /* se */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* psk */ + mix_psk(handshake->chaining_key, handshake->hash, key, + handshake->preshared_key); + + /* {} */ + message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_RESPONSE; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg) +{ + enum noise_handshake_state state = HANDSHAKE_ZEROED; + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + + down_read(&wg->static_identity.lock); + + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake = (struct noise_handshake *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, + src->receiver_index, &peer); + if (unlikely(!handshake)) + goto out; + + down_read(&handshake->lock); + state = handshake->state; + memcpy(hash, handshake->hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); + memcpy(ephemeral_private, handshake->ephemeral_private, + NOISE_PUBLIC_KEY_LEN); + up_read(&handshake->lock); + + if (state != HANDSHAKE_CREATED_INITIATION) + goto fail; + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* ee */ + if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + goto fail; + + /* se */ + if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) + goto fail; + + /* psk */ + mix_psk(chaining_key, hash, key, handshake->preshared_key); + + /* {} */ + if (!message_decrypt(NULL, src->encrypted_nothing, + sizeof(src->encrypted_nothing), key, hash)) + goto fail; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + /* It's important to check that the state is still the same, while we + * have an exclusive lock. + */ + if (handshake->state != state) { + up_write(&handshake->lock); + goto fail; + } + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + handshake->state = HANDSHAKE_CONSUMED_RESPONSE; + up_write(&handshake->lock); + ret_peer = peer; + goto out; + +fail: + wg_peer_put(peer); +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); + memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); + up_read(&wg->static_identity.lock); + return ret_peer; +} + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs) +{ + struct noise_keypair *new_keypair; + bool ret = false; + + down_write(&handshake->lock); + if (handshake->state != HANDSHAKE_CREATED_RESPONSE && + handshake->state != HANDSHAKE_CONSUMED_RESPONSE) + goto out; + + new_keypair = keypair_create(handshake->entry.peer); + if (!new_keypair) + goto out; + new_keypair->i_am_the_initiator = handshake->state == + HANDSHAKE_CONSUMED_RESPONSE; + new_keypair->remote_index = handshake->remote_index; + + if (new_keypair->i_am_the_initiator) + derive_keys(&new_keypair->sending, &new_keypair->receiving, + handshake->chaining_key); + else + derive_keys(&new_keypair->receiving, &new_keypair->sending, + handshake->chaining_key); + + handshake_zero(handshake); + rcu_read_lock_bh(); + if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, + handshake)->is_dead))) { + add_new_keypair(keypairs, new_keypair); + net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", + handshake->entry.peer->device->dev->name, + new_keypair->internal_id, + handshake->entry.peer->internal_id); + ret = wg_index_hashtable_replace( + handshake->entry.peer->device->index_hashtable, + &handshake->entry, &new_keypair->entry); + } else { + kzfree(new_keypair); + } + rcu_read_unlock_bh(); + +out: + up_write(&handshake->lock); + return ret; +} diff --git a/net/wireguard/noise.h b/net/wireguard/noise.h new file mode 100644 index 0000000000000..138a07bb817ce --- /dev/null +++ b/net/wireguard/noise.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ +#ifndef _WG_NOISE_H +#define _WG_NOISE_H + +#include "messages.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +union noise_counter { + struct { + u64 counter; + unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG]; + spinlock_t lock; + } receive; + atomic64_t counter; +}; + +struct noise_symmetric_key { + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + union noise_counter counter; + u64 birthdate; + bool is_valid; +}; + +struct noise_keypair { + struct index_hashtable_entry entry; + struct noise_symmetric_key sending; + struct noise_symmetric_key receiving; + __le32 remote_index; + bool i_am_the_initiator; + struct kref refcount; + struct rcu_head rcu; + u64 internal_id; +}; + +struct noise_keypairs { + struct noise_keypair __rcu *current_keypair; + struct noise_keypair __rcu *previous_keypair; + struct noise_keypair __rcu *next_keypair; + spinlock_t keypair_update_lock; +}; + +struct noise_static_identity { + u8 static_public[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + struct rw_semaphore lock; + bool has_identity; +}; + +enum noise_handshake_state { + HANDSHAKE_ZEROED, + HANDSHAKE_CREATED_INITIATION, + HANDSHAKE_CONSUMED_INITIATION, + HANDSHAKE_CREATED_RESPONSE, + HANDSHAKE_CONSUMED_RESPONSE +}; + +struct noise_handshake { + struct index_hashtable_entry entry; + + enum noise_handshake_state state; + u64 last_initiation_consumption; + + struct noise_static_identity *static_identity; + + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 remote_static[NOISE_PUBLIC_KEY_LEN]; + u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN]; + + u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; + + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + + u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; + __le32 remote_index; + + /* Protects all members except the immutable (after noise_handshake_ + * init): remote_static, precomputed_static_static, static_identity. + */ + struct rw_semaphore lock; +}; + +struct wg_device; + +void wg_noise_init(void); +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer); +void wg_noise_handshake_clear(struct noise_handshake *handshake); +static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns) +{ + atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() - + (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now); +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair); +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs); +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair); +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer); + +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]); +bool wg_noise_precompute_static_static(struct wg_peer *peer); + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg); + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg); + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs); + +#endif /* _WG_NOISE_H */ diff --git a/net/wireguard/peer.c b/net/wireguard/peer.c new file mode 100644 index 0000000000000..071eedf33f5aa --- /dev/null +++ b/net/wireguard/peer.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peer.h" +#include "device.h" +#include "queueing.h" +#include "timers.h" +#include "peerlookup.h" +#include "noise.h" + +#include +#include +#include +#include + +static atomic64_t peer_counter = ATOMIC64_INIT(0); + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) +{ + struct wg_peer *peer; + int ret = -ENOMEM; + + lockdep_assert_held(&wg->device_update_lock); + + if (wg->num_peers >= MAX_PEERS_PER_DEVICE) + return ERR_PTR(ret); + + peer = kzalloc(sizeof(*peer), GFP_KERNEL); + if (unlikely(!peer)) + return ERR_PTR(ret); + peer->device = wg; + + if (!wg_noise_handshake_init(&peer->handshake, &wg->static_identity, + public_key, preshared_key, peer)) { + ret = -EKEYREJECTED; + goto err_1; + } + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) + goto err_1; + if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, + MAX_QUEUED_PACKETS)) + goto err_2; + if (wg_packet_queue_init(&peer->rx_queue, NULL, false, + MAX_QUEUED_PACKETS)) + goto err_3; + + peer->internal_id = atomic64_inc_return(&peer_counter); + peer->serial_work_cpu = nr_cpumask_bits; + wg_cookie_init(&peer->latest_cookie); + wg_timers_init(peer); + wg_cookie_checker_precompute_peer_keys(peer); + spin_lock_init(&peer->keypairs.keypair_update_lock); + INIT_WORK(&peer->transmit_handshake_work, + wg_packet_handshake_send_worker); + rwlock_init(&peer->endpoint_lock); + kref_init(&peer->refcount); + skb_queue_head_init(&peer->staged_packet_queue); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); + netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, + NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); + list_add_tail(&peer->peer_list, &wg->peer_list); + INIT_LIST_HEAD(&peer->allowedips_list); + wg_pubkey_hashtable_add(wg->peer_hashtable, peer); + ++wg->num_peers; + pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); + return peer; + +err_3: + wg_packet_queue_free(&peer->tx_queue, false); +err_2: + dst_cache_destroy(&peer->endpoint_cache); +err_1: + kfree(peer); + return ERR_PTR(ret); +} + +struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking peer reference without holding the RCU read lock"); + if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) + return NULL; + return peer; +} + +static void peer_make_dead(struct wg_peer *peer) +{ + /* Remove from configuration-time lookup structures. */ + list_del_init(&peer->peer_list); + wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, + &peer->device->device_update_lock); + wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); + + /* Mark as dead, so that we don't allow jumping contexts after. */ + WRITE_ONCE(peer->is_dead, true); + + /* The caller must now synchronize_rcu() for this to take effect. */ +} + +static void peer_remove_after_dead(struct wg_peer *peer) +{ + WARN_ON(!peer->is_dead); + + /* No more keypairs can be created for this peer, since is_dead protects + * add_new_keypair, so we can now destroy existing ones. + */ + wg_noise_keypairs_clear(&peer->keypairs); + + /* Destroy all ongoing timers that were in-flight at the beginning of + * this function. + */ + wg_timers_stop(peer); + + /* The transition between packet encryption/decryption queues isn't + * guarded by is_dead, but each reference's life is strictly bounded by + * two generations: once for parallel crypto and once for serial + * ingestion, so we can simply flush twice, and be sure that we no + * longer have references inside these queues. + */ + + /* a) For encrypt/decrypt. */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.1) For send (but not receive, since that's napi). */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.2.1) For receive (but not send, since that's wq). */ + napi_disable(&peer->napi); + /* b.2.1) It's now safe to remove the napi struct, which must be done + * here from process context. + */ + netif_napi_del(&peer->napi); + + /* Ensure any workstructs we own (like transmit_handshake_work or + * clear_peer_work) no longer are in use. + */ + flush_workqueue(peer->device->handshake_send_wq); + + /* After the above flushes, a peer might still be active in a few + * different contexts: 1) from xmit(), before hitting is_dead and + * returning, 2) from wg_packet_consume_data(), before hitting is_dead + * and returning, 3) from wg_receive_handshake_packet() after a point + * where it has processed an incoming handshake packet, but where + * all calls to pass it off to timers fails because of is_dead. We won't + * have new references in (1) eventually, because we're removed from + * allowedips; we won't have new references in (2) eventually, because + * wg_index_hashtable_lookup will always return NULL, since we removed + * all existing keypairs and no more can be created; we won't have new + * references in (3) eventually, because we're removed from the pubkey + * hash table, which allows for a maximum of one handshake response, + * via the still-uncleared index hashtable entry, but not more than one, + * and in wg_cookie_message_consume, the lookup eventually gets a peer + * with a refcount of zero, so no new reference is taken. + */ + + --peer->device->num_peers; + wg_peer_put(peer); +} + +/* We have a separate "remove" function make sure that all active places where + * a peer is currently operating will eventually come to an end and not pass + * their reference onto another context. + */ +void wg_peer_remove(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + lockdep_assert_held(&peer->device->device_update_lock); + + peer_make_dead(peer); + synchronize_rcu(); + peer_remove_after_dead(peer); +} + +void wg_peer_remove_all(struct wg_device *wg) +{ + struct wg_peer *peer, *temp; + LIST_HEAD(dead_peers); + + lockdep_assert_held(&wg->device_update_lock); + + /* Avoid having to traverse individually for each one. */ + wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); + + list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { + peer_make_dead(peer); + list_add_tail(&peer->peer_list, &dead_peers); + } + synchronize_rcu(); + list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) + peer_remove_after_dead(peer); +} + +static void rcu_release(struct rcu_head *rcu) +{ + struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); + + dst_cache_destroy(&peer->endpoint_cache); + wg_packet_queue_free(&peer->rx_queue, false); + wg_packet_queue_free(&peer->tx_queue, false); + + /* The final zeroing takes care of clearing any remaining handshake key + * material and other potentially sensitive information. + */ + kzfree(peer); +} + +static void kref_release(struct kref *refcount) +{ + struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); + + pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + /* Remove ourself from dynamic runtime lookup structures, now that the + * last reference is gone. + */ + wg_index_hashtable_remove(peer->device->index_hashtable, + &peer->handshake.entry); + + /* Remove any lingering packets that didn't have a chance to be + * transmitted. + */ + wg_packet_purge_staged_packets(peer); + + /* Free the memory used. */ + call_rcu(&peer->rcu, rcu_release); +} + +void wg_peer_put(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + kref_put(&peer->refcount, kref_release); +} diff --git a/net/wireguard/peer.h b/net/wireguard/peer.h new file mode 100644 index 0000000000000..23af409229972 --- /dev/null +++ b/net/wireguard/peer.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEER_H +#define _WG_PEER_H + +#include "device.h" +#include "noise.h" +#include "cookie.h" + +#include +#include +#include +#include +#include + +struct wg_device; + +struct endpoint { + union { + struct sockaddr addr; + struct sockaddr_in addr4; + struct sockaddr_in6 addr6; + }; + union { + struct { + struct in_addr src4; + /* Essentially the same as addr6->scope_id */ + int src_if4; + }; + struct in6_addr src6; + }; +}; + +struct wg_peer { + struct wg_device *device; + struct crypt_queue tx_queue, rx_queue; + struct sk_buff_head staged_packet_queue; + int serial_work_cpu; + struct noise_keypairs keypairs; + struct endpoint endpoint; + struct dst_cache endpoint_cache; + rwlock_t endpoint_lock; + struct noise_handshake handshake; + atomic64_t last_sent_handshake; + struct work_struct transmit_handshake_work, clear_peer_work; + struct cookie latest_cookie; + struct hlist_node pubkey_hash; + u64 rx_bytes, tx_bytes; + struct timer_list timer_retransmit_handshake, timer_send_keepalive; + struct timer_list timer_new_handshake, timer_zero_key_material; + struct timer_list timer_persistent_keepalive; + unsigned int timer_handshake_attempts; + u16 persistent_keepalive_interval; + bool timer_need_another_keepalive; + bool sent_lastminute_handshake; + struct timespec64 walltime_last_handshake; + struct kref refcount; + struct rcu_head rcu; + struct list_head peer_list; + struct list_head allowedips_list; + u64 internal_id; + struct napi_struct napi; + bool is_dead; +}; + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); + +struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer); +static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) +{ + kref_get(&peer->refcount); + return peer; +} +void wg_peer_put(struct wg_peer *peer); +void wg_peer_remove(struct wg_peer *peer); +void wg_peer_remove_all(struct wg_device *wg); + +#endif /* _WG_PEER_H */ diff --git a/net/wireguard/peerlookup.c b/net/wireguard/peerlookup.c new file mode 100644 index 0000000000000..e4deb331476b3 --- /dev/null +++ b/net/wireguard/peerlookup.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peerlookup.h" +#include "peer.h" +#include "noise.h" + +static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + /* siphash gives us a secure 64bit number based on a random key. Since + * the bits are uniformly distributed, we can then mask off to get the + * bits we need. + */ + const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key); + + return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)]; +} + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void) +{ + struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + get_random_bytes(&table->key, sizeof(table->key)); + hash_init(table->hashtable); + mutex_init(&table->lock); + return table; +} + +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_add_head_rcu(&peer->pubkey_hash, + pubkey_bucket(table, peer->handshake.remote_static)); + mutex_unlock(&table->lock); +} + +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_del_init_rcu(&peer->pubkey_hash); + mutex_unlock(&table->lock); +} + +/* Returns a strong reference to a peer */ +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + struct wg_peer *iter_peer, *peer = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey), + pubkey_hash) { + if (!memcmp(pubkey, iter_peer->handshake.remote_static, + NOISE_PUBLIC_KEY_LEN)) { + peer = iter_peer; + break; + } + } + peer = wg_peer_get_maybe_zero(peer); + rcu_read_unlock_bh(); + return peer; +} + +static struct hlist_head *index_bucket(struct index_hashtable *table, + const __le32 index) +{ + /* Since the indices are random and thus all bits are uniformly + * distributed, we can find its bucket simply by masking. + */ + return &table->hashtable[(__force u32)index & + (HASH_SIZE(table->hashtable) - 1)]; +} + +struct index_hashtable *wg_index_hashtable_alloc(void) +{ + struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + hash_init(table->hashtable); + spin_lock_init(&table->lock); + return table; +} + +/* At the moment, we limit ourselves to 2^20 total peers, which generally might + * amount to 2^20*3 items in this hashtable. The algorithm below works by + * picking a random number and testing it. We can see that these limits mean we + * usually succeed pretty quickly: + * + * >>> def calculation(tries, size): + * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) + * ... + * >>> calculation(1, 2**20 * 3) + * 0.999267578125 + * >>> calculation(2, 2**20 * 3) + * 0.0007318854331970215 + * >>> calculation(3, 2**20 * 3) + * 5.360489012673497e-07 + * >>> calculation(4, 2**20 * 3) + * 3.9261394135792216e-10 + * + * At the moment, we don't do any masking, so this algorithm isn't exactly + * constant time in either the random guessing or in the hash list lookup. We + * could require a minimum of 3 tries, which would successfully mask the + * guessing. this would not, however, help with the growing hash lengths, which + * is another thing to consider moving forward. + */ + +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + struct index_hashtable_entry *existing_entry; + + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); + + rcu_read_lock_bh(); + +search_unused_slot: + /* First we try to find an unused slot, randomly, while unlocked. */ + entry->index = (__force __le32)get_random_u32(); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) + /* If it's already in use, we continue searching. */ + goto search_unused_slot; + } + + /* Once we've found an unused slot, we lock it, and then double-check + * that nobody else stole it from us. + */ + spin_lock_bh(&table->lock); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) { + spin_unlock_bh(&table->lock); + /* If it was stolen, we start over. */ + goto search_unused_slot; + } + } + /* Otherwise, we know we have it exclusively (since we're locked), + * so we insert. + */ + hlist_add_head_rcu(&entry->index_hash, + index_bucket(table, entry->index)); + spin_unlock_bh(&table->lock); + + rcu_read_unlock_bh(); + + return entry->index; +} + +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new) +{ + if (unlikely(hlist_unhashed(&old->index_hash))) + return false; + spin_lock_bh(&table->lock); + new->index = old->index; + hlist_replace_rcu(&old->index_hash, &new->index_hash); + + /* Calling init here NULLs out index_hash, and in fact after this + * function returns, it's theoretically possible for this to get + * reinserted elsewhere. That means the RCU lookup below might either + * terminate early or jump between buckets, in which case the packet + * simply gets dropped, which isn't terrible. + */ + INIT_HLIST_NODE(&old->index_hash); + spin_unlock_bh(&table->lock); + return true; +} + +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); +} + +/* Returns a strong reference to a entry->peer */ +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer) +{ + struct index_hashtable_entry *iter_entry, *entry = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index), + index_hash) { + if (iter_entry->index == index) { + if (likely(iter_entry->type & type_mask)) + entry = iter_entry; + break; + } + } + if (likely(entry)) { + entry->peer = wg_peer_get_maybe_zero(entry->peer); + if (likely(entry->peer)) + *peer = entry->peer; + else + entry = NULL; + } + rcu_read_unlock_bh(); + return entry; +} diff --git a/net/wireguard/peerlookup.h b/net/wireguard/peerlookup.h new file mode 100644 index 0000000000000..ced8117976800 --- /dev/null +++ b/net/wireguard/peerlookup.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEERLOOKUP_H +#define _WG_PEERLOOKUP_H + +#include "messages.h" + +#include +#include +#include + +struct wg_peer; + +struct pubkey_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 11); + siphash_key_t key; + struct mutex lock; +}; + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void); +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer); +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer); +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]); + +struct index_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 13); + spinlock_t lock; +}; + +enum index_hashtable_type { + INDEX_HASHTABLE_HANDSHAKE = 1U << 0, + INDEX_HASHTABLE_KEYPAIR = 1U << 1 +}; + +struct index_hashtable_entry { + struct wg_peer *peer; + struct hlist_node index_hash; + enum index_hashtable_type type; + __le32 index; +}; + +struct index_hashtable *wg_index_hashtable_alloc(void); +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry); +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new); +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry); +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer); + +#endif /* _WG_PEERLOOKUP_H */ diff --git a/net/wireguard/queueing.c b/net/wireguard/queueing.c new file mode 100644 index 0000000000000..5c964fcb994ec --- /dev/null +++ b/net/wireguard/queueing.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" + +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) +{ + int cpu; + struct multicore_worker __percpu *worker = + alloc_percpu(struct multicore_worker); + + if (!worker) + return NULL; + + for_each_possible_cpu(cpu) { + per_cpu_ptr(worker, cpu)->ptr = ptr; + INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function); + } + return worker; +} + +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len) +{ + int ret; + + memset(queue, 0, sizeof(*queue)); + ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); + if (ret) + return ret; + if (function) { + if (multicore) { + queue->worker = wg_packet_percpu_multicore_worker_alloc( + function, queue); + if (!queue->worker) + return -ENOMEM; + } else { + INIT_WORK(&queue->work, function); + } + } + return 0; +} + +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) +{ + if (multicore) + free_percpu(queue->worker); + WARN_ON(!__ptr_ring_empty(&queue->ring)); + ptr_ring_cleanup(&queue->ring, NULL); +} diff --git a/net/wireguard/queueing.h b/net/wireguard/queueing.h new file mode 100644 index 0000000000000..32b3af3c22769 --- /dev/null +++ b/net/wireguard/queueing.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_QUEUEING_H +#define _WG_QUEUEING_H + +#include "peer.h" +#include +#include +#include +#include + +struct wg_device; +struct wg_peer; +struct multicore_worker; +struct crypt_queue; +struct sk_buff; + +/* queueing.c APIs: */ +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len); +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); + +/* receive.c APIs: */ +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); +void wg_packet_handshake_receive_worker(struct work_struct *work); +/* NAPI poll function: */ +int wg_packet_rx_poll(struct napi_struct *napi, int budget); +/* Workqueue worker: */ +void wg_packet_decrypt_worker(struct work_struct *work); + +/* send.c APIs: */ +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry); +void wg_packet_send_handshake_response(struct wg_peer *peer); +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index); +void wg_packet_send_keepalive(struct wg_peer *peer); +void wg_packet_purge_staged_packets(struct wg_peer *peer); +void wg_packet_send_staged_packets(struct wg_peer *peer); +/* Workqueue workers: */ +void wg_packet_handshake_send_worker(struct work_struct *work); +void wg_packet_tx_worker(struct work_struct *work); +void wg_packet_encrypt_worker(struct work_struct *work); + +enum packet_state { + PACKET_STATE_UNCRYPTED, + PACKET_STATE_CRYPTED, + PACKET_STATE_DEAD +}; + +struct packet_cb { + u64 nonce; + struct noise_keypair *keypair; + atomic_t state; + u32 mtu; + u8 ds; +}; + +#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) +#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer) + +/* Returns either the correct skb->protocol value, or 0 if invalid. */ +static inline __be16 wg_skb_examine_untrusted_ip_hdr(struct sk_buff *skb) +{ + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct iphdr)) <= + skb_tail_pointer(skb) && + ip_hdr(skb)->version == 4) + return htons(ETH_P_IP); + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct ipv6hdr)) <= + skb_tail_pointer(skb) && + ipv6_hdr(skb)->version == 6) + return htons(ETH_P_IPV6); + return 0; +} + +static inline void wg_reset_packet(struct sk_buff *skb) +{ + const int pfmemalloc = skb->pfmemalloc; + + skb_scrub_packet(skb, true); + memset(&skb->headers_start, 0, + offsetof(struct sk_buff, headers_end) - + offsetof(struct sk_buff, headers_start)); + skb->pfmemalloc = pfmemalloc; + skb->queue_mapping = 0; + skb->nohdr = 0; + skb->peeked = 0; + skb->mac_len = 0; + skb->dev = NULL; +#ifdef CONFIG_NET_SCHED + skb->tc_index = 0; + skb_reset_redirect(skb); +#endif + skb->hdr_len = skb_headroom(skb); + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb_probe_transport_header(skb); + skb_reset_inner_headers(skb); +} + +static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) +{ + unsigned int cpu = *stored_cpu, cpu_index, i; + + if (unlikely(cpu == nr_cpumask_bits || + !cpumask_test_cpu(cpu, cpu_online_mask))) { + cpu_index = id % cpumask_weight(cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < cpu_index; ++i) + cpu = cpumask_next(cpu, cpu_online_mask); + *stored_cpu = cpu; + } + return cpu; +} + +/* This function is racy, in the sense that next is unlocked, so it could return + * the same CPU twice. A race-free version of this would be to instead store an + * atomic sequence number, do an increment-and-return, and then iterate through + * every possible CPU until we get to that index -- choose_cpu. However that's + * a bit slower, and it doesn't seem like this potential race actually + * introduces any performance loss, so we live with it. + */ +static inline int wg_cpumask_next_online(int *next) +{ + int cpu = *next; + + while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask))) + cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + return cpu; +} + +static inline int wg_queue_enqueue_per_device_and_peer( + struct crypt_queue *device_queue, struct crypt_queue *peer_queue, + struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) +{ + int cpu; + + atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); + /* We first queue this up for the peer ingestion, but the consumer + * will wait for the state to change to CRYPTED or DEAD before. + */ + if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) + return -ENOSPC; + /* Then we queue it up in the device queue, which consumes the + * packet as soon as it can. + */ + cpu = wg_cpumask_next_online(next_cpu); + if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) + return -EPIPE; + queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); + return 0; +} + +static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, + struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, + peer->internal_id), + peer->device->packet_crypt_wq, &queue->work); + wg_peer_put(peer); +} + +static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + wg_peer_put(peer); +} + +#ifdef DEBUG +bool wg_packet_counter_selftest(void); +#endif + +#endif /* _WG_QUEUEING_H */ diff --git a/net/wireguard/ratelimiter.c b/net/wireguard/ratelimiter.c new file mode 100644 index 0000000000000..e33ec72a96421 --- /dev/null +++ b/net/wireguard/ratelimiter.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU +/* We normally alias all non-_bh functions to the _bh ones in the compat layer, + * but that's not appropriate here, where we actually do want non-_bh ones. + */ +#undef synchronize_rcu +#define synchronize_rcu old_synchronize_rcu +#undef call_rcu +#define call_rcu old_call_rcu +#undef rcu_barrier +#define rcu_barrier old_rcu_barrier +#endif + +#include "ratelimiter.h" +#include +#include +#include +#include + +static struct kmem_cache *entry_cache; +static hsiphash_key_t key; +static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); +static DEFINE_MUTEX(init_lock); +static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ +static atomic_t total_entries = ATOMIC_INIT(0); +static unsigned int max_entries, table_size; +static void wg_ratelimiter_gc_entries(struct work_struct *); +static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); +static struct hlist_head *table_v4; +#if IS_ENABLED(CONFIG_IPV6) +static struct hlist_head *table_v6; +#endif + +struct ratelimiter_entry { + u64 last_time_ns, tokens, ip; + void *net; + spinlock_t lock; + struct hlist_node hash; + struct rcu_head rcu; +}; + +enum { + PACKETS_PER_SECOND = 20, + PACKETS_BURSTABLE = 5, + PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, + TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE +}; + +static void entry_free(struct rcu_head *rcu) +{ + kmem_cache_free(entry_cache, + container_of(rcu, struct ratelimiter_entry, rcu)); + atomic_dec(&total_entries); +} + +static void entry_uninit(struct ratelimiter_entry *entry) +{ + hlist_del_rcu(&entry->hash); + call_rcu(&entry->rcu, entry_free); +} + +/* Calling this function with a NULL work uninits all entries. */ +static void wg_ratelimiter_gc_entries(struct work_struct *work) +{ + const u64 now = ktime_get_coarse_boottime_ns(); + struct ratelimiter_entry *entry; + struct hlist_node *temp; + unsigned int i; + + for (i = 0; i < table_size; ++i) { + spin_lock(&table_lock); + hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#if IS_ENABLED(CONFIG_IPV6) + hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#endif + spin_unlock(&table_lock); + if (likely(work)) + cond_resched(); + } + if (likely(work)) + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); +} + +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) +{ + /* We only take the bottom half of the net pointer, so that we can hash + * 3 words in the end. This way, siphash's len param fits into the final + * u32, and we don't incur an extra round. + */ + const u32 net_word = (unsigned long)net; + struct ratelimiter_entry *entry; + struct hlist_head *bucket; + u64 ip; + + if (skb->protocol == htons(ETH_P_IP)) { + ip = (u64 __force)ip_hdr(skb)->saddr; + bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & + (table_size - 1)]; + } +#if IS_ENABLED(CONFIG_IPV6) + else if (skb->protocol == htons(ETH_P_IPV6)) { + /* Only use 64 bits, so as to ratelimit the whole /64. */ + memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); + bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & + (table_size - 1)]; + } +#endif + else + return false; + rcu_read_lock(); + hlist_for_each_entry_rcu(entry, bucket, hash) { + if (entry->net == net && entry->ip == ip) { + u64 now, tokens; + bool ret; + /* Quasi-inspired by nft_limit.c, but this is actually a + * slightly different algorithm. Namely, we incorporate + * the burst as part of the maximum tokens, rather than + * as part of the rate. + */ + spin_lock(&entry->lock); + now = ktime_get_coarse_boottime_ns(); + tokens = min_t(u64, TOKEN_MAX, + entry->tokens + now - + entry->last_time_ns); + entry->last_time_ns = now; + ret = tokens >= PACKET_COST; + entry->tokens = ret ? tokens - PACKET_COST : tokens; + spin_unlock(&entry->lock); + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + if (atomic_inc_return(&total_entries) > max_entries) + goto err_oom; + + entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); + if (unlikely(!entry)) + goto err_oom; + + entry->net = net; + entry->ip = ip; + INIT_HLIST_NODE(&entry->hash); + spin_lock_init(&entry->lock); + entry->last_time_ns = ktime_get_coarse_boottime_ns(); + entry->tokens = TOKEN_MAX - PACKET_COST; + spin_lock(&table_lock); + hlist_add_head_rcu(&entry->hash, bucket); + spin_unlock(&table_lock); + return true; + +err_oom: + atomic_dec(&total_entries); + return false; +} + +int wg_ratelimiter_init(void) +{ + mutex_lock(&init_lock); + if (++init_refcnt != 1) + goto out; + + entry_cache = KMEM_CACHE(ratelimiter_entry, 0); + if (!entry_cache) + goto err; + + /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, + * but what it shares in common is that it uses a massive hashtable. So, + * we borrow their wisdom about good table sizes on different systems + * dependent on RAM. This calculation here comes from there. + */ + table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : + max_t(unsigned long, 16, roundup_pow_of_two( + (totalram_pages() << PAGE_SHIFT) / + (1U << 14) / sizeof(struct hlist_head))); + max_entries = table_size * 8; + + table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL); + if (unlikely(!table_v4)) + goto err_kmemcache; + +#if IS_ENABLED(CONFIG_IPV6) + table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL); + if (unlikely(!table_v6)) { + kvfree(table_v4); + goto err_kmemcache; + } +#endif + + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); + get_random_bytes(&key, sizeof(key)); +out: + mutex_unlock(&init_lock); + return 0; + +err_kmemcache: + kmem_cache_destroy(entry_cache); +err: + --init_refcnt; + mutex_unlock(&init_lock); + return -ENOMEM; +} + +void wg_ratelimiter_uninit(void) +{ + mutex_lock(&init_lock); + if (!init_refcnt || --init_refcnt) + goto out; + + cancel_delayed_work_sync(&gc_work); + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + kvfree(table_v4); +#if IS_ENABLED(CONFIG_IPV6) + kvfree(table_v6); +#endif + kmem_cache_destroy(entry_cache); +out: + mutex_unlock(&init_lock); +} + +#include "selftest/ratelimiter.c" diff --git a/net/wireguard/ratelimiter.h b/net/wireguard/ratelimiter.h new file mode 100644 index 0000000000000..83067f71ea99e --- /dev/null +++ b/net/wireguard/ratelimiter.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_RATELIMITER_H +#define _WG_RATELIMITER_H + +#include + +int wg_ratelimiter_init(void); +void wg_ratelimiter_uninit(void); +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net); + +#ifdef DEBUG +bool wg_ratelimiter_selftest(void); +#endif + +#endif /* _WG_RATELIMITER_H */ diff --git a/net/wireguard/receive.c b/net/wireguard/receive.c new file mode 100644 index 0000000000000..e00f0f4b09827 --- /dev/null +++ b/net/wireguard/receive.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "device.h" +#include "peer.h" +#include "timers.h" +#include "messages.h" +#include "cookie.h" +#include "socket.h" + +#include +#include +#include +#include +#include + +/* Must be called with bh disabled. */ +static void update_rx_stats(struct wg_peer *peer, size_t len) +{ + struct pcpu_sw_netstats *tstats = + get_cpu_ptr(peer->device->dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + ++tstats->rx_packets; + tstats->rx_bytes += len; + peer->rx_bytes += len; + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); +} + +#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) + +static size_t validate_header_len(struct sk_buff *skb) +{ + if (unlikely(skb->len < sizeof(struct message_header))) + return 0; + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && + skb->len >= MESSAGE_MINIMUM_LENGTH) + return sizeof(struct message_data); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && + skb->len == sizeof(struct message_handshake_initiation)) + return sizeof(struct message_handshake_initiation); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && + skb->len == sizeof(struct message_handshake_response)) + return sizeof(struct message_handshake_response); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && + skb->len == sizeof(struct message_handshake_cookie)) + return sizeof(struct message_handshake_cookie); + return 0; +} + +static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) +{ + size_t data_offset, data_len, header_len; + struct udphdr *udp; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol || + skb_transport_header(skb) < skb->head || + (skb_transport_header(skb) + sizeof(struct udphdr)) > + skb_tail_pointer(skb))) + return -EINVAL; /* Bogus IP header */ + udp = udp_hdr(skb); + data_offset = (u8 *)udp - skb->data; + if (unlikely(data_offset > U16_MAX || + data_offset + sizeof(struct udphdr) > skb->len)) + /* Packet has offset at impossible location or isn't big enough + * to have UDP fields. + */ + return -EINVAL; + data_len = ntohs(udp->len); + if (unlikely(data_len < sizeof(struct udphdr) || + data_len > skb->len - data_offset)) + /* UDP packet is reporting too small of a size or lying about + * its size. + */ + return -EINVAL; + data_len -= sizeof(struct udphdr); + data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; + if (unlikely(!pskb_may_pull(skb, + data_offset + sizeof(struct message_header)) || + pskb_trim(skb, data_len + data_offset) < 0)) + return -EINVAL; + skb_pull(skb, data_offset); + if (unlikely(skb->len != data_len)) + /* Final len does not agree with calculated len */ + return -EINVAL; + header_len = validate_header_len(skb); + if (unlikely(!header_len)) + return -EINVAL; + __skb_push(skb, data_offset); + if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) + return -EINVAL; + __skb_pull(skb, data_offset); + return 0; +} + +static void wg_receive_handshake_packet(struct wg_device *wg, + struct sk_buff *skb) +{ + enum cookie_mac_state mac_state; + struct wg_peer *peer = NULL; + /* This is global, so that our load calculation applies to the whole + * system. We don't care about races with it at all. + */ + static u64 last_under_load; + bool packet_needs_cookie; + bool under_load; + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { + net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", + wg->dev->name, skb); + wg_cookie_message_consume( + (struct message_handshake_cookie *)skb->data, wg); + return; + } + + under_load = skb_queue_len(&wg->incoming_handshakes) >= + MAX_QUEUED_INCOMING_HANDSHAKES / 8; + if (under_load) + last_under_load = ktime_get_coarse_boottime_ns(); + else if (last_under_load) + under_load = !wg_birthdate_has_expired(last_under_load, 1); + mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, + under_load); + if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || + (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { + packet_needs_cookie = false; + } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { + packet_needs_cookie = true; + } else { + net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", + wg->dev->name, skb); + return; + } + + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { + struct message_handshake_initiation *message = + (struct message_handshake_initiation *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_initiation(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + wg_packet_send_handshake_response(peer); + break; + } + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { + struct message_handshake_response *message = + (struct message_handshake_response *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_response(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_handshake_complete(peer); + /* Calling this function will either send any existing + * packets in the queue and not send a keepalive, which + * is the best case, Or, if there's nothing in the + * queue, it will send a keepalive, in order to give + * immediate confirmation of the session. + */ + wg_packet_send_keepalive(peer); + } + break; + } + } + + if (unlikely(!peer)) { + WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); + return; + } + + local_bh_disable(); + update_rx_stats(peer, skb->len); + local_bh_enable(); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_peer_put(peer); +} + +void wg_packet_handshake_receive_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *skb; + + while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { + wg_receive_handshake_packet(wg, skb); + dev_kfree_skb(skb); + cond_resched(); + } +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + if (peer->sent_lastminute_handshake) + return; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) + send = true; + rcu_read_unlock_bh(); + + if (send) { + peer->sent_lastminute_handshake = true; + wg_packet_send_queued_handshake_initiation(peer, false); + } +} + +static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key, + simd_context_t *simd_context) +{ + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct sk_buff *trailer; + unsigned int offset; + int num_frags; + + if (unlikely(!key)) + return false; + + if (unlikely(!READ_ONCE(key->is_valid) || + wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || + key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { + WRITE_ONCE(key->is_valid, false); + return false; + } + + PACKET_CB(skb)->nonce = + le64_to_cpu(((struct message_data *)skb->data)->counter); + + /* We ensure that the network header is part of the packet before we + * call skb_cow_data, so that there's no chance that data is removed + * from the skb, so that later we can extract the original endpoint. + */ + offset = skb->data - skb_network_header(skb); + skb_push(skb, offset); + num_frags = skb_cow_data(skb, 0, &trailer); + offset += sizeof(struct message_data); + skb_pull(skb, offset); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) + return false; + + if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, + PACKET_CB(skb)->nonce, key->key, + simd_context)) + return false; + + /* Another ugly situation of pushing and pulling the header so as to + * keep endpoint information intact. + */ + skb_push(skb, offset); + if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) + return false; + skb_pull(skb, offset); + + return true; +} + +/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ +static bool counter_validate(union noise_counter *counter, u64 their_counter) +{ + unsigned long index, index_current, top, i; + bool ret = false; + + spin_lock_bh(&counter->receive.lock); + + if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || + their_counter >= REJECT_AFTER_MESSAGES)) + goto out; + + ++their_counter; + + if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < + counter->receive.counter)) + goto out; + + index = their_counter >> ilog2(BITS_PER_LONG); + + if (likely(their_counter > counter->receive.counter)) { + index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); + top = min_t(unsigned long, index - index_current, + COUNTER_BITS_TOTAL / BITS_PER_LONG); + for (i = 1; i <= top; ++i) + counter->receive.backtrack[(i + index_current) & + ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; + counter->receive.counter = their_counter; + } + + index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; + ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), + &counter->receive.backtrack[index]); + +out: + spin_unlock_bh(&counter->receive.lock); + return ret; +} + +#include "selftest/counter.c" + +static void wg_packet_consume_data_done(struct wg_peer *peer, + struct sk_buff *skb, + struct endpoint *endpoint) +{ + struct net_device *dev = peer->device->dev; + unsigned int len, len_before_trim; + struct wg_peer *routed_peer; + + wg_socket_set_peer_endpoint(peer, endpoint); + + if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, + PACKET_CB(skb)->keypair))) { + wg_timers_handshake_complete(peer); + wg_packet_send_staged_packets(peer); + } + + keep_key_fresh(peer); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + + /* A packet with length 0 is a keepalive packet */ + if (unlikely(!skb->len)) { + update_rx_stats(peer, message_data_len(0)); + net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + goto packet_processed; + } + + wg_timers_data_received(peer); + + if (unlikely(skb_network_header(skb) < skb->head)) + goto dishonest_packet_size; + if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && + (ip_hdr(skb)->version == 4 || + (ip_hdr(skb)->version == 6 && + pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) + goto dishonest_packet_type; + + skb->dev = dev; + /* We've already verified the Poly1305 auth tag, which means this packet + * was not modified in transit. We can therefore tell the networking + * stack that all checksums of every layer of encapsulation have already + * been checked "by the hardware" and therefore is unnecessary to check + * again in software. + */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +#ifndef COMPAT_CANNOT_USE_CSUM_LEVEL + skb->csum_level = ~0; /* All levels */ +#endif + skb->protocol = wg_skb_examine_untrusted_ip_hdr(skb); + if (skb->protocol == htons(ETH_P_IP)) { + len = ntohs(ip_hdr(skb)->tot_len); + if (unlikely(len < sizeof(struct iphdr))) + goto dishonest_packet_size; + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP_ECN_set_ce(ip_hdr(skb)); + } else if (skb->protocol == htons(ETH_P_IPV6)) { + len = ntohs(ipv6_hdr(skb)->payload_len) + + sizeof(struct ipv6hdr); + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP6_ECN_set_ce(skb, ipv6_hdr(skb)); + } else { + goto dishonest_packet_type; + } + + if (unlikely(len > skb->len)) + goto dishonest_packet_size; + len_before_trim = skb->len; + if (unlikely(pskb_trim(skb, len))) + goto packet_processed; + + routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, + skb); + wg_peer_put(routed_peer); /* We don't need the extra reference. */ + + if (unlikely(routed_peer != peer)) + goto dishonest_packet_peer; + + if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { + ++dev->stats.rx_dropped; + net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + } else { + update_rx_stats(peer, message_data_len(len_before_trim)); + } + return; + +dishonest_packet_peer: + net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", + dev->name, skb, peer->internal_id, + &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_type: + net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_size: + net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_length_errors; + goto packet_processed; +packet_processed: + dev_kfree_skb(skb); +} + +int wg_packet_rx_poll(struct napi_struct *napi, int budget) +{ + struct wg_peer *peer = container_of(napi, struct wg_peer, napi); + struct crypt_queue *queue = &peer->rx_queue; + struct noise_keypair *keypair; + struct endpoint endpoint; + enum packet_state state; + struct sk_buff *skb; + int work_done = 0; + bool free; + + if (unlikely(budget <= 0)) + return 0; + + while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(skb); + keypair = PACKET_CB(skb)->keypair; + free = true; + + if (unlikely(state != PACKET_STATE_CRYPTED)) + goto next; + + if (unlikely(!counter_validate(&keypair->receiving.counter, + PACKET_CB(skb)->nonce))) { + net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", + peer->device->dev->name, + PACKET_CB(skb)->nonce, + keypair->receiving.counter.receive.counter); + goto next; + } + + if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) + goto next; + + wg_reset_packet(skb); + wg_packet_consume_data_done(peer, skb, &endpoint); + free = false; + +next: + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + if (unlikely(free)) + dev_kfree_skb(skb); + + if (++work_done >= budget) + break; + } + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; +} + +void wg_packet_decrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + simd_context_t simd_context; + struct sk_buff *skb; + + simd_get(&simd_context); + while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = likely(decrypt_packet(skb, + &PACKET_CB(skb)->keypair->receiving, + &simd_context)) ? + PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; + wg_queue_enqueue_per_peer_napi(skb, state); + simd_relax(&simd_context); + } + + simd_put(&simd_context); +} + +static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) +{ + __le32 idx = ((struct message_data *)skb->data)->key_idx; + struct wg_peer *peer = NULL; + int ret; + + rcu_read_lock_bh(); + PACKET_CB(skb)->keypair = + (struct noise_keypair *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, + &peer); + if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) + goto err_keypair; + + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, + &peer->rx_queue, skb, + wg->packet_crypt_wq, + &wg->decrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + if (likely(!ret || ret == -EPIPE)) { + rcu_read_unlock_bh(); + return; + } +err: + wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); +err_keypair: + rcu_read_unlock_bh(); + wg_peer_put(peer); + dev_kfree_skb(skb); +} + +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) +{ + if (unlikely(prepare_skb_header(skb, wg) < 0)) + goto err; + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): + case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { + int cpu; + + if (skb_queue_len(&wg->incoming_handshakes) > + MAX_QUEUED_INCOMING_HANDSHAKES || + unlikely(!rng_is_initialized())) { + net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + skb_queue_tail(&wg->incoming_handshakes, skb); + /* Queues up a call to packet_process_queued_handshake_ + * packets(skb): + */ + cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); + queue_work_on(cpu, wg->handshake_receive_wq, + &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); + break; + } + case cpu_to_le32(MESSAGE_DATA): + PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); + wg_packet_consume_data(wg, skb); + break; + default: + net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + return; + +err: + dev_kfree_skb(skb); +} diff --git a/net/wireguard/selftest/allowedips.c b/net/wireguard/selftest/allowedips.c new file mode 100644 index 0000000000000..846db14cb046b --- /dev/null +++ b/net/wireguard/selftest/allowedips.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This contains some basic static unit tests for the allowedips data structure. + * It also has two additional modes that are disabled and meant to be used by + * folks directly playing with this file. If you define the macro + * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in + * memory, it will be printed out as KERN_DEBUG in a format that can be passed + * to graphviz (the dot command) to visualize it. If you define the macro + * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of + * randomized tests done against a trivial implementation, which may take + * upwards of a half-hour to complete. There's no set of users who should be + * enabling these, and the only developers that should go anywhere near these + * nobs are the ones who are reading this comment. + */ + +#ifdef DEBUG + +#include + +static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits, + u8 cidr) +{ + swap_endian(dst, src, bits); + memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8); + if (cidr) + dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8); +} + +static __init void print_node(struct allowedips_node *node, u8 bits) +{ + char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n"; + char *fmt_declaration = KERN_DEBUG + "\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n"; + char *style = "dotted"; + u8 ip1[16], ip2[16]; + u32 color = 0; + + if (bits == 32) { + fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n"; + } else if (bits == 128) { + fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n"; + } + if (node->peer) { + hsiphash_key_t key = { { 0 } }; + + memcpy(&key, &node->peer, sizeof(node->peer)); + color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 | + hsiphash_1u32(0xbabecafe, &key) % 200 << 8 | + hsiphash_1u32(0xabad1dea, &key) % 200; + style = "bold"; + } + swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr); + printk(fmt_declaration, ip1, node->cidr, style, color); + if (node->bit[0]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[0])->bits, bits, + node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[0])->cidr); + print_node(rcu_dereference_raw(node->bit[0]), bits); + } + if (node->bit[1]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[1])->bits, + bits, node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[1])->cidr); + print_node(rcu_dereference_raw(node->bit[1]), bits); + } +} + +static __init void print_tree(struct allowedips_node __rcu *top, u8 bits) +{ + printk(KERN_DEBUG "digraph trie {\n"); + print_node(rcu_dereference_raw(top), bits); + printk(KERN_DEBUG "}\n"); +} + +enum { + NUM_PEERS = 2000, + NUM_RAND_ROUTES = 400, + NUM_MUTATED_ROUTES = 100, + NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30 +}; + +struct horrible_allowedips { + struct hlist_head head; +}; + +struct horrible_allowedips_node { + struct hlist_node table; + union nf_inet_addr ip; + union nf_inet_addr mask; + u8 ip_version; + void *value; +}; + +static __init void horrible_allowedips_init(struct horrible_allowedips *table) +{ + INIT_HLIST_HEAD(&table->head); +} + +static __init void horrible_allowedips_free(struct horrible_allowedips *table) +{ + struct horrible_allowedips_node *node; + struct hlist_node *h; + + hlist_for_each_entry_safe(node, h, &table->head, table) { + hlist_del(&node->table); + kfree(node); + } +} + +static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr) +{ + union nf_inet_addr mask; + + memset(&mask, 0x00, 128 / 8); + memset(&mask, 0xff, cidr / 8); + if (cidr % 32) + mask.all[cidr / 32] = (__force u32)htonl( + (0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL); + return mask; +} + +static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet) +{ + return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) + + hweight32(subnet.all[2]) + hweight32(subnet.all[3]); +} + +static __init inline void +horrible_mask_self(struct horrible_allowedips_node *node) +{ + if (node->ip_version == 4) { + node->ip.ip &= node->mask.ip; + } else if (node->ip_version == 6) { + node->ip.ip6[0] &= node->mask.ip6[0]; + node->ip.ip6[1] &= node->mask.ip6[1]; + node->ip.ip6[2] &= node->mask.ip6[2]; + node->ip.ip6[3] &= node->mask.ip6[3]; + } +} + +static __init inline bool +horrible_match_v4(const struct horrible_allowedips_node *node, + struct in_addr *ip) +{ + return (ip->s_addr & node->mask.ip) == node->ip.ip; +} + +static __init inline bool +horrible_match_v6(const struct horrible_allowedips_node *node, + struct in6_addr *ip) +{ + return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) == + node->ip.ip6[0] && + (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) == + node->ip.ip6[1] && + (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) == + node->ip.ip6[2] && + (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3]; +} + +static __init void +horrible_insert_ordered(struct horrible_allowedips *table, + struct horrible_allowedips_node *node) +{ + struct horrible_allowedips_node *other = NULL, *where = NULL; + u8 my_cidr = horrible_mask_to_cidr(node->mask); + + hlist_for_each_entry(other, &table->head, table) { + if (!memcmp(&other->mask, &node->mask, + sizeof(union nf_inet_addr)) && + !memcmp(&other->ip, &node->ip, + sizeof(union nf_inet_addr)) && + other->ip_version == node->ip_version) { + other->value = node->value; + kfree(node); + return; + } + where = other; + if (horrible_mask_to_cidr(other->mask) <= my_cidr) + break; + } + if (!other && !where) + hlist_add_head(&node->table, &table->head); + else if (!other) + hlist_add_behind(&node->table, &where->table); + else + hlist_add_before(&node->table, &where->table); +} + +static __init int +horrible_allowedips_insert_v4(struct horrible_allowedips *table, + struct in_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 4; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init int +horrible_allowedips_insert_v6(struct horrible_allowedips *table, + struct in6_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in6 = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 6; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init void * +horrible_allowedips_lookup_v4(struct horrible_allowedips *table, + struct in_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 4) + continue; + if (horrible_match_v4(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init void * +horrible_allowedips_lookup_v6(struct horrible_allowedips *table, + struct in6_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 6) + continue; + if (horrible_match_v6(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init bool randomized_test(void) +{ + unsigned int i, j, k, mutate_amount, cidr; + u8 ip[16], mutate_mask[16], mutated[16]; + struct wg_peer **peers, *peer; + struct horrible_allowedips h; + DEFINE_MUTEX(mutex); + struct allowedips t; + bool ret = false; + + mutex_init(&mutex); + + wg_allowedips_init(&t); + horrible_allowedips_init(&h); + + peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL); + if (unlikely(!peers)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + for (i = 0; i < NUM_PEERS; ++i) { + peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL); + if (unlikely(!peers[i])) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + kref_init(&peers[i]->refcount); + } + + mutex_lock(&mutex); + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 4); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 4); + prandom_bytes(mutate_mask, 4); + mutate_amount = prandom_u32_max(32); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, + (struct in_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, + (struct in_addr *)mutated, cidr, peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 16); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 16); + prandom_bytes(mutate_mask, 16); + mutate_amount = prandom_u32_max(128); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, + (struct in6_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6( + &h, (struct in6_addr *)mutated, cidr, + peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + mutex_unlock(&mutex); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 4); + if (lookup(t.root4, 32, ip) != + horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 16); + if (lookup(t.root6, 128, ip) != + horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + ret = true; + +free: + mutex_lock(&mutex); +free_locked: + wg_allowedips_free(&t, &mutex); + mutex_unlock(&mutex); + horrible_allowedips_free(&h); + if (peers) { + for (i = 0; i < NUM_PEERS; ++i) + kfree(peers[i]); + } + kfree(peers); + return ret; +} + +static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d) +{ + static struct in_addr ip; + u8 *split = (u8 *)&ip; + + split[0] = a; + split[1] = b; + split[2] = c; + split[3] = d; + return &ip; +} + +static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d) +{ + static struct in6_addr ip; + __be32 *split = (__be32 *)&ip; + + split[0] = cpu_to_be32(a); + split[1] = cpu_to_be32(b); + split[2] = cpu_to_be32(c); + split[3] = cpu_to_be32(d); + return &ip; +} + +static __init struct wg_peer *init_peer(void) +{ + struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL); + + if (!peer) + return NULL; + kref_init(&peer->refcount); + INIT_LIST_HEAD(&peer->allowedips_list); + return peer; +} + +#define insert(version, mem, ipa, ipb, ipc, ipd, cidr) \ + wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ + cidr, mem, &mutex) + +#define maybe_fail() do { \ + ++i; \ + if (!_s) { \ + pr_info("allowedips self-test %zu: FAIL\n", i); \ + success = false; \ + } \ + } while (0) + +#define test(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) == (mem); \ + maybe_fail(); \ + } while (0) + +#define test_negative(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) != (mem); \ + maybe_fail(); \ + } while (0) + +#define test_boolean(cond) do { \ + bool _s = (cond); \ + maybe_fail(); \ + } while (0) + +bool __init wg_allowedips_selftest(void) +{ + bool found_a = false, found_b = false, found_c = false, found_d = false, + found_e = false, found_other = false; + struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(), + *d = init_peer(), *e = init_peer(), *f = init_peer(), + *g = init_peer(), *h = init_peer(); + struct allowedips_node *iter_node; + bool success = false; + struct allowedips t; + DEFINE_MUTEX(mutex); + struct in6_addr ip; + size_t i = 0, count = 0; + __be64 part; + + mutex_init(&mutex); + mutex_lock(&mutex); + wg_allowedips_init(&t); + + if (!a || !b || !c || !d || !e || !f || !g || !h) { + pr_err("allowedips self-test malloc: FAIL\n"); + goto free; + } + + insert(4, a, 192, 168, 4, 0, 24); + insert(4, b, 192, 168, 4, 4, 32); + insert(4, c, 192, 168, 0, 0, 16); + insert(4, d, 192, 95, 5, 64, 27); + /* replaces previous entry, and maskself is required */ + insert(4, c, 192, 95, 5, 65, 27); + insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64); + insert(4, e, 0, 0, 0, 0, 0); + insert(6, e, 0, 0, 0, 0, 0); + /* replaces previous entry */ + insert(6, f, 0, 0, 0, 0, 0); + insert(6, g, 0x24046800, 0, 0, 0, 32); + /* maskself is required */ + insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); + insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); + insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + insert(4, g, 64, 15, 112, 0, 20); + /* maskself is required */ + insert(4, h, 64, 15, 123, 211, 25); + insert(4, a, 10, 0, 0, 0, 25); + insert(4, b, 10, 0, 0, 128, 25); + insert(4, a, 10, 1, 0, 0, 30); + insert(4, b, 10, 1, 0, 4, 30); + insert(4, c, 10, 1, 0, 8, 29); + insert(4, d, 10, 1, 0, 16, 29); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + success = true; + + test(4, a, 192, 168, 4, 20); + test(4, a, 192, 168, 4, 0); + test(4, b, 192, 168, 4, 4); + test(4, c, 192, 168, 200, 182); + test(4, c, 192, 95, 5, 68); + test(4, e, 192, 95, 5, 96); + test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543); + test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee); + test(6, f, 0x26075300, 0x60006b01, 0, 0); + test(6, g, 0x24046800, 0x40040806, 0, 0x1006); + test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0, 0); + test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010); + test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); + test(4, g, 64, 15, 116, 26); + test(4, g, 64, 15, 127, 3); + test(4, g, 64, 15, 123, 1); + test(4, h, 64, 15, 123, 128); + test(4, h, 64, 15, 123, 129); + test(4, a, 10, 0, 0, 52); + test(4, b, 10, 0, 0, 220); + test(4, a, 10, 1, 0, 2); + test(4, b, 10, 1, 0, 6); + test(4, c, 10, 1, 0, 10); + test(4, d, 10, 1, 0, 20); + + insert(4, a, 1, 0, 0, 0, 32); + insert(4, a, 64, 0, 0, 0, 32); + insert(4, a, 128, 0, 0, 0, 32); + insert(4, a, 192, 0, 0, 0, 32); + insert(4, a, 255, 0, 0, 0, 32); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 1, 0, 0, 0); + test_negative(4, a, 64, 0, 0, 0); + test_negative(4, a, 128, 0, 0, 0); + test_negative(4, a, 192, 0, 0, 0); + test_negative(4, a, 255, 0, 0, 0); + + wg_allowedips_free(&t, &mutex); + wg_allowedips_init(&t); + insert(4, a, 192, 168, 0, 0, 16); + insert(4, a, 192, 168, 0, 0, 24); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 192, 168, 0, 1); + + /* These will hit the WARN_ON(len >= 128) in free_node if something + * goes wrong. + */ + for (i = 0; i < 128; ++i) { + part = cpu_to_be64(~(1LLU << (i % 64))); + memset(&ip, 0xff, 16); + memcpy((u8 *)&ip + (i < 64) * 8, &part, 8); + wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex); + } + + wg_allowedips_free(&t, &mutex); + + wg_allowedips_init(&t); + insert(4, a, 192, 95, 5, 93, 27); + insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(4, a, 10, 1, 0, 20, 29); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); + list_for_each_entry(iter_node, &a->allowedips_list, peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family = wg_allowedips_read_node(iter_node, ip, &cidr); + + count++; + + if (cidr == 27 && family == AF_INET && + !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr))) + found_a = true; + else if (cidr == 128 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), + sizeof(struct in6_addr))) + found_b = true; + else if (cidr == 29 && family == AF_INET && + !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr))) + found_c = true; + else if (cidr == 83 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), + sizeof(struct in6_addr))) + found_d = true; + else if (cidr == 21 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075000, 0, 0, 0), + sizeof(struct in6_addr))) + found_e = true; + else + found_other = true; + } + test_boolean(count == 5); + test_boolean(found_a); + test_boolean(found_b); + test_boolean(found_c); + test_boolean(found_d); + test_boolean(found_e); + test_boolean(!found_other); + + if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success) + success = randomized_test(); + + if (success) + pr_info("allowedips self-tests: pass\n"); + +free: + wg_allowedips_free(&t, &mutex); + kfree(a); + kfree(b); + kfree(c); + kfree(d); + kfree(e); + kfree(f); + kfree(g); + kfree(h); + mutex_unlock(&mutex); + + return success; +} + +#undef test_negative +#undef test +#undef remove +#undef insert +#undef init_peer + +#endif diff --git a/net/wireguard/selftest/counter.c b/net/wireguard/selftest/counter.c new file mode 100644 index 0000000000000..f4fbb9072ed73 --- /dev/null +++ b/net/wireguard/selftest/counter.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG +bool __init wg_packet_counter_selftest(void) +{ + unsigned int test_num = 0, i; + union noise_counter counter; + bool success = true; + +#define T_INIT do { \ + memset(&counter, 0, sizeof(union noise_counter)); \ + spin_lock_init(&counter.receive.lock); \ + } while (0) +#define T_LIM (COUNTER_WINDOW_SIZE + 1) +#define T(n, v) do { \ + ++test_num; \ + if (counter_validate(&counter, n) != (v)) { \ + pr_err("nonce counter self-test %u: FAIL\n", \ + test_num); \ + success = false; \ + } \ + } while (0) + + T_INIT; + /* 1 */ T(0, true); + /* 2 */ T(1, true); + /* 3 */ T(1, false); + /* 4 */ T(9, true); + /* 5 */ T(8, true); + /* 6 */ T(7, true); + /* 7 */ T(7, false); + /* 8 */ T(T_LIM, true); + /* 9 */ T(T_LIM - 1, true); + /* 10 */ T(T_LIM - 1, false); + /* 11 */ T(T_LIM - 2, true); + /* 12 */ T(2, true); + /* 13 */ T(2, false); + /* 14 */ T(T_LIM + 16, true); + /* 15 */ T(3, false); + /* 16 */ T(T_LIM + 16, false); + /* 17 */ T(T_LIM * 4, true); + /* 18 */ T(T_LIM * 4 - (T_LIM - 1), true); + /* 19 */ T(10, false); + /* 20 */ T(T_LIM * 4 - T_LIM, false); + /* 21 */ T(T_LIM * 4 - (T_LIM + 1), false); + /* 22 */ T(T_LIM * 4 - (T_LIM - 2), true); + /* 23 */ T(T_LIM * 4 + 1 - T_LIM, false); + /* 24 */ T(0, false); + /* 25 */ T(REJECT_AFTER_MESSAGES, false); + /* 26 */ T(REJECT_AFTER_MESSAGES - 1, true); + /* 27 */ T(REJECT_AFTER_MESSAGES, false); + /* 28 */ T(REJECT_AFTER_MESSAGES - 1, false); + /* 29 */ T(REJECT_AFTER_MESSAGES - 2, true); + /* 30 */ T(REJECT_AFTER_MESSAGES + 1, false); + /* 31 */ T(REJECT_AFTER_MESSAGES + 2, false); + /* 32 */ T(REJECT_AFTER_MESSAGES - 2, false); + /* 33 */ T(REJECT_AFTER_MESSAGES - 3, true); + /* 34 */ T(0, false); + + T_INIT; + for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) + T(i, true); + T(0, true); + T(0, false); + + T_INIT; + for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) + T(i, true); + T(1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) + T(i, true); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) + T(i, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(COUNTER_WINDOW_SIZE + 1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(0, true); + T(COUNTER_WINDOW_SIZE + 1, true); + +#undef T +#undef T_LIM +#undef T_INIT + + if (success) + pr_info("nonce counter self-tests: pass\n"); + return success; +} +#endif diff --git a/net/wireguard/selftest/ratelimiter.c b/net/wireguard/selftest/ratelimiter.c new file mode 100644 index 0000000000000..bcd6462e45401 --- /dev/null +++ b/net/wireguard/selftest/ratelimiter.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG + +#include + +static const struct { + bool result; + unsigned int msec_to_sleep_before; +} expected_results[] __initconst = { + [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, + [PACKETS_BURSTABLE] = { false, 0 }, + [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 2] = { false, 0 }, + [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, + [PACKETS_BURSTABLE + 4] = { true, 0 }, + [PACKETS_BURSTABLE + 5] = { false, 0 } +}; + +static __init unsigned int maximum_jiffies_at_index(int index) +{ + unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; + int i; + + for (i = 0; i <= index; ++i) + total_msecs += expected_results[i].msec_to_sleep_before; + return msecs_to_jiffies(total_msecs); +} + +static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, + struct sk_buff *skb6, struct ipv6hdr *hdr6, + int *test) +{ + unsigned long loop_start_time; + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + loop_start_time = jiffies; + + for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { + if (expected_results[i].msec_to_sleep_before) + msleep(expected_results[i].msec_to_sleep_before); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb4, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb4, &init_net)) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1); + +#if IS_ENABLED(CONFIG_IPV6) + hdr6->saddr.in6_u.u6_addr32[2] = htonl(i); + hdr6->saddr.in6_u.u6_addr32[3] = htonl(i); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb6, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb6, &init_net)) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; +#endif + } + return 0; +} + +static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4, + int *test) +{ + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + + if (atomic_read(&total_entries)) + return -EXFULL; + ++(*test); + + for (i = 0; i <= max_entries; ++i) { + hdr4->saddr = htonl(i); + if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries)) + return -EXFULL; + ++(*test); + } + return 0; +} + +bool __init wg_ratelimiter_selftest(void) +{ + enum { TRIALS_BEFORE_GIVING_UP = 5000 }; + bool success = false; + int test = 0, trials; + struct sk_buff *skb4, *skb6; + struct iphdr *hdr4; + struct ipv6hdr *hdr6; + + if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) + return true; + + BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); + + if (wg_ratelimiter_init()) + goto out; + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + goto out; + } + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + goto out; + } + ++test; + + skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL); + if (unlikely(!skb4)) + goto err_nofree; + skb4->protocol = htons(ETH_P_IP); + hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4)); + hdr4->saddr = htonl(8182); + skb_reset_network_header(skb4); + ++test; + +#if IS_ENABLED(CONFIG_IPV6) + skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL); + if (unlikely(!skb6)) { + kfree_skb(skb4); + goto err_nofree; + } + skb6->protocol = htons(ETH_P_IPV6); + hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6)); + hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212); + hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188); + skb_reset_network_header(skb6); + ++test; +#endif + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0, ret; + + ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); + if (ret == -ETIMEDOUT) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(500); + continue; + } else if (ret < 0) { + test += test_count; + goto err; + } else { + test += test_count; + break; + } + } + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0; + + if (capacity_test(skb4, hdr4, &test_count) < 0) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(50); + continue; + } + test += test_count; + break; + } + + success = true; + +err: + kfree_skb(skb4); +#if IS_ENABLED(CONFIG_IPV6) + kfree_skb(skb6); +#endif +err_nofree: + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + /* Uninit one extra time to check underflow detection. */ + wg_ratelimiter_uninit(); +out: + if (success) + pr_info("ratelimiter self-tests: pass\n"); + else + pr_err("ratelimiter self-test %d: FAIL\n", test); + + return success; +} +#endif diff --git a/net/wireguard/send.c b/net/wireguard/send.c new file mode 100644 index 0000000000000..85b83fafbe4f6 --- /dev/null +++ b/net/wireguard/send.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include +#include + +static void wg_packet_send_handshake_initiation(struct wg_peer *peer) +{ + struct message_handshake_initiation packet; + + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT)) + return; /* This function is rate limited. */ + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), + HANDSHAKE_DSCP); + wg_timers_handshake_initiated(peer); + } +} + +void wg_packet_handshake_send_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + transmit_handshake_work); + + wg_packet_send_handshake_initiation(peer); + wg_peer_put(peer); +} + +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry) +{ + if (!is_retry) + peer->timer_handshake_attempts = 0; + + rcu_read_lock_bh(); + /* We check last_sent_handshake here in addition to the actual function + * we're queueing up, so that we don't queue things if not strictly + * necessary: + */ + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT) || + unlikely(READ_ONCE(peer->is_dead))) + goto out; + + wg_peer_get(peer); + /* Queues up calling packet_send_queued_handshakes(peer), where we do a + * peer_put(peer) after: + */ + if (!queue_work(peer->device->handshake_send_wq, + &peer->transmit_handshake_work)) + /* If the work was already queued, we want to drop the + * extra reference: + */ + wg_peer_put(peer); +out: + rcu_read_unlock_bh(); +} + +void wg_packet_send_handshake_response(struct wg_peer *peer) +{ + struct message_handshake_response packet; + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, + sizeof(packet), + HANDSHAKE_DSCP); + } + } +} + +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index) +{ + struct message_handshake_cookie packet; + + net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", + wg->dev->name, initiating_skb); + wg_cookie_message_create(&packet, initiating_skb, sender_index, + &wg->cookie_checker); + wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, + sizeof(packet)); +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + (unlikely(atomic64_read(&keypair->sending.counter.counter) > + REKEY_AFTER_MESSAGES) || + (keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REKEY_AFTER_TIME))))) + send = true; + rcu_read_unlock_bh(); + + if (send) + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static unsigned int calculate_skb_padding(struct sk_buff *skb) +{ + /* We do this modulo business with the MTU, just in case the networking + * layer gives us a packet that's bigger than the MTU. In that case, we + * wouldn't want the final subtraction to overflow in the case of the + * padded_size being clamped. + */ + unsigned int last_unit = skb->len % PACKET_CB(skb)->mtu; + unsigned int padded_size = ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE); + + if (padded_size > PACKET_CB(skb)->mtu) + padded_size = PACKET_CB(skb)->mtu; + return padded_size - last_unit; +} + +static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair, + simd_context_t *simd_context) +{ + unsigned int padding_len, plaintext_len, trailer_len; + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct message_data *header; + struct sk_buff *trailer; + int num_frags; + + /* Calculate lengths. */ + padding_len = calculate_skb_padding(skb); + trailer_len = padding_len + noise_encrypted_len(0); + plaintext_len = skb->len + padding_len; + + /* Expand data section to have room for padding and auth tag. */ + num_frags = skb_cow_data(skb, trailer_len, &trailer); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + /* Set the padding to zeros, and make sure it and the auth tag are part + * of the skb. + */ + memset(skb_tail_pointer(trailer), 0, padding_len); + + /* Expand head section to have room for our header and the network + * stack's headers. + */ + if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) + return false; + + /* Finalize checksum calculation for the inner packet, if required. */ + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_help(skb))) + return false; + + /* Only after checksumming can we safely add on the padding at the end + * and the header. + */ + skb_set_inner_network_header(skb, 0); + header = (struct message_data *)skb_push(skb, sizeof(*header)); + header->header.type = cpu_to_le32(MESSAGE_DATA); + header->key_idx = keypair->remote_index; + header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); + pskb_put(skb, trailer, trailer_len); + + /* Now we can encrypt the scattergather segments */ + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, sizeof(struct message_data), + noise_encrypted_len(plaintext_len)) <= 0) + return false; + return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + keypair->sending.key, + simd_context); +} + +void wg_packet_send_keepalive(struct wg_peer *peer) +{ + struct sk_buff *skb; + + if (skb_queue_empty(&peer->staged_packet_queue)) { + skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, + GFP_ATOMIC); + if (unlikely(!skb)) + return; + skb_reserve(skb, DATA_PACKET_HEAD_ROOM); + skb->dev = peer->device->dev; + PACKET_CB(skb)->mtu = skb->dev->mtu; + skb_queue_tail(&peer->staged_packet_queue, skb); + net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + } + + wg_packet_send_staged_packets(peer); +} + +static void wg_packet_create_data_done(struct sk_buff *first, + struct wg_peer *peer) +{ + struct sk_buff *skb, *next; + bool is_keepalive, data_sent = false; + + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + skb_list_walk_safe(first, skb, next) { + is_keepalive = skb->len == message_data_len(0); + if (likely(!wg_socket_send_skb_to_peer(peer, skb, + PACKET_CB(skb)->ds) && !is_keepalive)) + data_sent = true; + } + + if (likely(data_sent)) + wg_timers_data_sent(peer); + + keep_key_fresh(peer); +} + +void wg_packet_tx_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct crypt_queue, + work); + struct noise_keypair *keypair; + enum packet_state state; + struct sk_buff *first; + struct wg_peer *peer; + + while ((first = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(first)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(first); + keypair = PACKET_CB(first)->keypair; + + if (likely(state == PACKET_STATE_CRYPTED)) + wg_packet_create_data_done(first, peer); + else + kfree_skb_list(first); + + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + } +} + +void wg_packet_encrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *first, *skb, *next; + simd_context_t simd_context; + + simd_get(&simd_context); + while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = PACKET_STATE_CRYPTED; + + skb_list_walk_safe(first, skb, next) { + if (likely(encrypt_packet(skb, + PACKET_CB(first)->keypair, + &simd_context))) { + wg_reset_packet(skb); + } else { + state = PACKET_STATE_DEAD; + break; + } + } + wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, + state); + + simd_relax(&simd_context); + } + simd_put(&simd_context); +} + +static void wg_packet_create_data(struct sk_buff *first) +{ + struct wg_peer *peer = PACKET_PEER(first); + struct wg_device *wg = peer->device; + int ret = -EINVAL; + + rcu_read_lock_bh(); + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, + &peer->tx_queue, first, + wg->packet_crypt_wq, + &wg->encrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer(&peer->tx_queue, first, + PACKET_STATE_DEAD); +err: + rcu_read_unlock_bh(); + if (likely(!ret || ret == -EPIPE)) + return; + wg_noise_keypair_put(PACKET_CB(first)->keypair, false); + wg_peer_put(peer); + kfree_skb_list(first); +} + +void wg_packet_purge_staged_packets(struct wg_peer *peer) +{ + spin_lock_bh(&peer->staged_packet_queue.lock); + peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; + __skb_queue_purge(&peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); +} + +void wg_packet_send_staged_packets(struct wg_peer *peer) +{ + struct noise_symmetric_key *key; + struct noise_keypair *keypair; + struct sk_buff_head packets; + struct sk_buff *skb; + + /* Steal the current queue into our local one. */ + __skb_queue_head_init(&packets); + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice_init(&peer->staged_packet_queue, &packets); + spin_unlock_bh(&peer->staged_packet_queue.lock); + if (unlikely(skb_queue_empty(&packets))) + return; + + /* First we make sure we have a valid reference to a valid key. */ + rcu_read_lock_bh(); + keypair = wg_noise_keypair_get( + rcu_dereference_bh(peer->keypairs.current_keypair)); + rcu_read_unlock_bh(); + if (unlikely(!keypair)) + goto out_nokey; + key = &keypair->sending; + if (unlikely(!READ_ONCE(key->is_valid))) + goto out_nokey; + if (unlikely(wg_birthdate_has_expired(key->birthdate, + REJECT_AFTER_TIME))) + goto out_invalid; + + /* After we know we have a somewhat valid key, we now try to assign + * nonces to all of the packets in the queue. If we can't assign nonces + * for all of them, we just consider it a failure and wait for the next + * handshake. + */ + skb_queue_walk(&packets, skb) { + /* 0 for no outer TOS: no leak. TODO: at some later point, we + * might consider using flowi->tos as outer instead. + */ + PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); + PACKET_CB(skb)->nonce = + atomic64_inc_return(&key->counter.counter) - 1; + if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) + goto out_invalid; + } + + packets.prev->next = NULL; + wg_peer_get(keypair->entry.peer); + PACKET_CB(packets.next)->keypair = keypair; + wg_packet_create_data(packets.next); + return; + +out_invalid: + WRITE_ONCE(key->is_valid, false); +out_nokey: + wg_noise_keypair_put(keypair, false); + + /* We orphan the packets if we're waiting on a handshake, so that they + * don't block a socket's pool. + */ + skb_queue_walk(&packets, skb) + skb_orphan(skb); + /* Then we put them back on the top of the queue. We're not too + * concerned about accidentally getting things a little out of order if + * packets are being added really fast, because this queue is for before + * packets can even be sent and it's small anyway. + */ + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + /* If we're exiting because there's something wrong with the key, it + * means we should initiate a new handshake. + */ + wg_packet_send_queued_handshake_initiation(peer, false); +} diff --git a/net/wireguard/socket.c b/net/wireguard/socket.c new file mode 100644 index 0000000000000..c46256d0d81c1 --- /dev/null +++ b/net/wireguard/socket.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include + +static int send4(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ + struct flowi4 fl = { + .saddr = endpoint->src4.s_addr, + .daddr = endpoint->addr4.sin_addr.s_addr, + .fl4_dport = endpoint->addr4.sin_port, + .flowi4_mark = wg->fwmark, + .flowi4_proto = IPPROTO_UDP + }; + struct rtable *rt = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock4); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl4_sport = inet_sk(sock)->inet_sport; + + if (cache) + rt = dst_cache_get_ip4(cache, &fl.saddr); + + if (!rt) { + security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); + if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + } + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && + PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && + rt->dst.dev->ifindex != endpoint->src_if4)))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + if (!IS_ERR(rt)) + ip_rt_put(rt); + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + } + if (unlikely(IS_ERR(rt))) { + ret = PTR_ERR(rt); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(rt->dst.dev == skb->dev)) { + ip_rt_put(rt); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +} + +static int send6(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct flowi6 fl = { + .saddr = endpoint->src6, + .daddr = endpoint->addr6.sin6_addr, + .fl6_dport = endpoint->addr6.sin6_port, + .flowi6_mark = wg->fwmark, + .flowi6_oif = endpoint->addr6.sin6_scope_id, + .flowi6_proto = IPPROTO_UDP + /* TODO: addr->sin6_flowinfo */ + }; + struct dst_entry *dst = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock6); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl6_sport = inet_sk(sock)->inet_sport; + + if (cache) + dst = dst_cache_get_ip6(cache, &fl.saddr); + + if (!dst) { + security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); + if (unlikely(!ipv6_addr_any(&fl.saddr) && + !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { + endpoint->src6 = fl.saddr = in6addr_any; + if (cache) + dst_cache_reset(cache); + } + dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, + NULL); + if (unlikely(IS_ERR(dst))) { + ret = PTR_ERR(dst); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(dst->dev == skb->dev)) { + dst_release(dst); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip6(cache, dst, &fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +#else + return -EAFNOSUPPORT; +#endif +} + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +{ + size_t skb_len = skb->len; + int ret = -EAFNOSUPPORT; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + ret = send4(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else if (peer->endpoint.addr.sa_family == AF_INET6) + ret = send6(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else + dev_kfree_skb(skb); + if (likely(!ret)) + peer->tx_bytes += skb_len; + read_unlock_bh(&peer->endpoint_lock); + + return ret; +} + +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, + size_t len, u8 ds) +{ + struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + + if (unlikely(!skb)) + return -ENOMEM; + + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + return wg_socket_send_skb_to_peer(peer, skb, ds); +} + +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, void *buffer, + size_t len) +{ + int ret = 0; + struct sk_buff *skb; + struct endpoint endpoint; + + if (unlikely(!in_skb)) + return -EINVAL; + ret = wg_socket_endpoint_from_skb(&endpoint, in_skb); + if (unlikely(ret < 0)) + return ret; + + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + if (unlikely(!skb)) + return -ENOMEM; + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + + if (endpoint.addr.sa_family == AF_INET) + ret = send4(wg, skb, &endpoint, 0, NULL); + else if (endpoint.addr.sa_family == AF_INET6) + ret = send6(wg, skb, &endpoint, 0, NULL); + /* No other possibilities if the endpoint is valid, which it is, + * as we checked above. + */ + + return ret; +} + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb) +{ + memset(endpoint, 0, sizeof(*endpoint)); + if (skb->protocol == htons(ETH_P_IP)) { + endpoint->addr4.sin_family = AF_INET; + endpoint->addr4.sin_port = udp_hdr(skb)->source; + endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; + endpoint->src4.s_addr = ip_hdr(skb)->daddr; + endpoint->src_if4 = skb->skb_iif; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + endpoint->addr6.sin6_family = AF_INET6; + endpoint->addr6.sin6_port = udp_hdr(skb)->source; + endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; + endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( + &ipv6_hdr(skb)->saddr, skb->skb_iif); + endpoint->src6 = ipv6_hdr(skb)->daddr; + } else { + return -EINVAL; + } + return 0; +} + +static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) +{ + return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && + a->addr4.sin_port == b->addr4.sin_port && + a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && + a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || + (a->addr.sa_family == AF_INET6 && + b->addr.sa_family == AF_INET6 && + a->addr6.sin6_port == b->addr6.sin6_port && + ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) && + a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && + ipv6_addr_equal(&a->src6, &b->src6)) || + unlikely(!a->addr.sa_family && !b->addr.sa_family); +} + +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (endpoint_eq(endpoint, &peer->endpoint)) + return; + write_lock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) { + peer->endpoint.addr4 = endpoint->addr4; + peer->endpoint.src4 = endpoint->src4; + peer->endpoint.src_if4 = endpoint->src_if4; + } else if (endpoint->addr.sa_family == AF_INET6) { + peer->endpoint.addr6 = endpoint->addr6; + peer->endpoint.src6 = endpoint->src6; + } else { + goto out; + } + dst_cache_reset(&peer->endpoint_cache); +out: + write_unlock_bh(&peer->endpoint_lock); +} + +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb) +{ + struct endpoint endpoint; + + if (!wg_socket_endpoint_from_skb(&endpoint, skb)) + wg_socket_set_peer_endpoint(peer, &endpoint); +} + +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +{ + write_lock_bh(&peer->endpoint_lock); + memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); + dst_cache_reset(&peer->endpoint_cache); + write_unlock_bh(&peer->endpoint_lock); +} + +static int wg_receive(struct sock *sk, struct sk_buff *skb) +{ + struct wg_device *wg; + + if (unlikely(!sk)) + goto err; + wg = sk->sk_user_data; + if (unlikely(!wg)) + goto err; + wg_packet_receive(wg, skb); + return 0; + +err: + kfree_skb(skb); + return 0; +} + +static void sock_free(struct sock *sock) +{ + if (unlikely(!sock)) + return; + sk_clear_memalloc(sock); + udp_tunnel_sock_release(sock->sk_socket); +} + +static void set_sock_opts(struct socket *sock) +{ + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_sndbuf = INT_MAX; + sk_set_memalloc(sock->sk); +} + +int wg_socket_init(struct wg_device *wg, u16 port) +{ + int ret; + struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = wg, + .encap_type = 1, + .encap_rcv = wg_receive + }; + struct socket *new4 = NULL, *new6 = NULL; + struct udp_port_cfg port4 = { + .family = AF_INET, + .local_ip.s_addr = htonl(INADDR_ANY), + .local_udp_port = htons(port), + .use_udp_checksums = true + }; +#if IS_ENABLED(CONFIG_IPV6) + int retries = 0; + struct udp_port_cfg port6 = { + .family = AF_INET6, + .local_ip6 = IN6ADDR_ANY_INIT, + .use_udp6_tx_checksums = true, + .use_udp6_rx_checksums = true, + .ipv6_v6only = true + }; +#endif + +#if IS_ENABLED(CONFIG_IPV6) +retry: +#endif + + ret = udp_sock_create(wg->creating_net, &port4, &new4); + if (ret < 0) { + pr_err("%s: Could not create IPv4 socket\n", wg->dev->name); + return ret; + } + set_sock_opts(new4); + setup_udp_tunnel_sock(wg->creating_net, new4, &cfg); + +#if IS_ENABLED(CONFIG_IPV6) + if (ipv6_mod_enabled()) { + port6.local_udp_port = inet_sk(new4->sk)->inet_sport; + ret = udp_sock_create(wg->creating_net, &port6, &new6); + if (ret < 0) { + udp_tunnel_sock_release(new4); + if (ret == -EADDRINUSE && !port && retries++ < 100) + goto retry; + pr_err("%s: Could not create IPv6 socket\n", + wg->dev->name); + return ret; + } + set_sock_opts(new6); + setup_udp_tunnel_sock(wg->creating_net, new6, &cfg); + } +#endif + + wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL); + return 0; +} + +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6) +{ + struct sock *old4, *old6; + + mutex_lock(&wg->socket_update_lock); + old4 = rcu_dereference_protected(wg->sock4, + lockdep_is_held(&wg->socket_update_lock)); + old6 = rcu_dereference_protected(wg->sock6, + lockdep_is_held(&wg->socket_update_lock)); + rcu_assign_pointer(wg->sock4, new4); + rcu_assign_pointer(wg->sock6, new6); + if (new4) + wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); + mutex_unlock(&wg->socket_update_lock); + synchronize_rcu(); + synchronize_net(); + sock_free(old4); + sock_free(old6); +} diff --git a/net/wireguard/socket.h b/net/wireguard/socket.h new file mode 100644 index 0000000000000..bab5848efbcdf --- /dev/null +++ b/net/wireguard/socket.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SOCKET_H +#define _WG_SOCKET_H + +#include +#include +#include +#include + +int wg_socket_init(struct wg_device *wg, u16 port); +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6); +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, + size_t len, u8 ds); +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, + u8 ds); +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, + void *out_buffer, size_t len); + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb); +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb); +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); + +#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) +#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ + struct endpoint __endpoint; \ + wg_socket_endpoint_from_skb(&__endpoint, skb); \ + net_dbg_ratelimited(fmt, dev, &__endpoint.addr, \ + ##__VA_ARGS__); \ + } while (0) +#else +#define net_dbg_skb_ratelimited(fmt, skb, ...) +#endif + +#endif /* _WG_SOCKET_H */ diff --git a/net/wireguard/timers.c b/net/wireguard/timers.c new file mode 100644 index 0000000000000..d54d32ac9bc41 --- /dev/null +++ b/net/wireguard/timers.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "queueing.h" +#include "socket.h" + +/* + * - Timer for retransmitting the handshake if we don't hear back after + * `REKEY_TIMEOUT + jitter` ms. + * + * - Timer for sending empty packet if we have received a packet but after have + * not sent one for `KEEPALIVE_TIMEOUT` ms. + * + * - Timer for initiating new handshake if we have sent a packet but after have + * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + * jitter` ms. + * + * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms + * if no new keys have been received. + * + * - Timer for, if enabled, sending an empty authenticated packet every user- + * specified seconds. + */ + +static inline void mod_peer_timer(struct wg_peer *peer, + struct timer_list *timer, + unsigned long expires) +{ + rcu_read_lock_bh(); + if (likely(netif_running(peer->device->dev) && + !READ_ONCE(peer->is_dead))) + mod_timer(timer, expires); + rcu_read_unlock_bh(); +} + +static void wg_expired_retransmit_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_retransmit_handshake); + + if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); + + del_timer(&peer->timer_send_keepalive); + /* We drop all packets without a keypair and don't try again, + * if we try unsuccessfully for too long to make a handshake. + */ + wg_packet_purge_staged_packets(peer); + + /* We set a timer for destroying any residue that might be left + * of a partial exchange. + */ + if (!timer_pending(&peer->timer_zero_key_material)) + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); + } else { + ++peer->timer_handshake_attempts; + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REKEY_TIMEOUT, + peer->timer_handshake_attempts + 1); + + /* We clear the endpoint address src address, in case this is + * the cause of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + + wg_packet_send_queued_handshake_initiation(peer, true); + } +} + +static void wg_expired_send_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); + + wg_packet_send_keepalive(peer); + if (peer->timer_need_another_keepalive) { + peer->timer_need_another_keepalive = false; + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + } +} + +static void wg_expired_new_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); + + pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + /* We clear the endpoint address src address, in case this is the cause + * of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static void wg_expired_zero_key_material(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); + + rcu_read_lock_bh(); + if (!READ_ONCE(peer->is_dead)) { + wg_peer_get(peer); + if (!queue_work(peer->device->handshake_send_wq, + &peer->clear_peer_work)) + /* If the work was already on the queue, we want to drop + * the extra reference. + */ + wg_peer_put(peer); + } + rcu_read_unlock_bh(); +} + +static void wg_queued_expired_zero_key_material(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + clear_peer_work); + + pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REJECT_AFTER_TIME * 3); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_peer_put(peer); +} + +static void wg_expired_send_persistent_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_persistent_keepalive); + + if (likely(peer->persistent_keepalive_interval)) + wg_packet_send_keepalive(peer); +} + +/* Should be called after an authenticated data packet is sent. */ +void wg_timers_data_sent(struct wg_peer *peer) +{ + if (!timer_pending(&peer->timer_new_handshake)) + mod_peer_timer(peer, &peer->timer_new_handshake, + jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after an authenticated data packet is received. */ +void wg_timers_data_received(struct wg_peer *peer) +{ + if (likely(netif_running(peer->device->dev))) { + if (!timer_pending(&peer->timer_send_keepalive)) + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + else + peer->timer_need_another_keepalive = true; + } +} + +/* Should be called after any type of authenticated packet is sent, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) +{ + del_timer(&peer->timer_send_keepalive); +} + +/* Should be called after any type of authenticated packet is received, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) +{ + del_timer(&peer->timer_new_handshake); +} + +/* Should be called after a handshake initiation message is sent. */ +void wg_timers_handshake_initiated(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_retransmit_handshake, + jiffies + REKEY_TIMEOUT * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after a handshake response message is received and processed + * or when getting key confirmation via the first data message. + */ +void wg_timers_handshake_complete(struct wg_peer *peer) +{ + del_timer(&peer->timer_retransmit_handshake); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + ktime_get_real_ts64(&peer->walltime_last_handshake); +} + +/* Should be called after an ephemeral key is created, which is before sending a + * handshake response or after receiving a handshake response. + */ +void wg_timers_session_derived(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); +} + +/* Should be called before a packet with authentication, whether + * keepalive, data, or handshakem is sent, or after one is received. + */ +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer) +{ + if (peer->persistent_keepalive_interval) + mod_peer_timer(peer, &peer->timer_persistent_keepalive, + jiffies + peer->persistent_keepalive_interval * HZ); +} + +void wg_timers_init(struct wg_peer *peer) +{ + timer_setup(&peer->timer_retransmit_handshake, + wg_expired_retransmit_handshake, 0); + timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0); + timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0); + timer_setup(&peer->timer_zero_key_material, + wg_expired_zero_key_material, 0); + timer_setup(&peer->timer_persistent_keepalive, + wg_expired_send_persistent_keepalive, 0); + INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + peer->timer_need_another_keepalive = false; +} + +void wg_timers_stop(struct wg_peer *peer) +{ + del_timer_sync(&peer->timer_retransmit_handshake); + del_timer_sync(&peer->timer_send_keepalive); + del_timer_sync(&peer->timer_new_handshake); + del_timer_sync(&peer->timer_zero_key_material); + del_timer_sync(&peer->timer_persistent_keepalive); + flush_work(&peer->clear_peer_work); +} diff --git a/net/wireguard/timers.h b/net/wireguard/timers.h new file mode 100644 index 0000000000000..f0653dcb13266 --- /dev/null +++ b/net/wireguard/timers.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_TIMERS_H +#define _WG_TIMERS_H + +#include + +struct wg_peer; + +void wg_timers_init(struct wg_peer *peer); +void wg_timers_stop(struct wg_peer *peer); +void wg_timers_data_sent(struct wg_peer *peer); +void wg_timers_data_received(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer); +void wg_timers_handshake_initiated(struct wg_peer *peer); +void wg_timers_handshake_complete(struct wg_peer *peer); +void wg_timers_session_derived(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer); + +static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds, + u64 expiration_seconds) +{ + return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC) + <= (s64)ktime_get_coarse_boottime_ns(); +} + +#endif /* _WG_TIMERS_H */ diff --git a/net/wireguard/uapi/wireguard.h b/net/wireguard/uapi/wireguard.h new file mode 100644 index 0000000000000..ae88be14c9478 --- /dev/null +++ b/net/wireguard/uapi/wireguard.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Documentation + * ============= + * + * The below enums and macros are for interfacing with WireGuard, using generic + * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two + * methods: get and set. Note that while they share many common attributes, + * these two functions actually accept a slightly different set of inputs and + * outputs. + * + * WG_CMD_GET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain + * one but not both of: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * + * The kernel will then return several messages (NLM_F_MULTI) containing the + * following tree of nested items: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_LISTEN_PORT: NLA_U16 + * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16 + * WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec + * WGPEER_A_RX_BYTES: NLA_U64 + * WGPEER_A_TX_BYTES: NLA_U64 + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32 + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that all of the allowed IPs of a single peer will not + * fit within a single netlink message. In that case, the same peer will + * be written in the following message, except it will only contain + * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several + * times in a row for the same peer. It is then up to the receiver to + * coalesce adjacent peers. Likewise, it is possible that all peers will + * not fit within a single message. So, subsequent peers will be sent + * in following messages, except those will only contain WGDEVICE_A_IFNAME + * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these + * messages to form the complete list of peers. + * + * Since this is an NLA_F_DUMP command, the final message will always be + * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message + * contains an integer error code. It is either zero or a negative error + * code corresponding to the errno. + * + * WG_CMD_SET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST. The command should contain the + * following tree of nested items, containing one but not both of + * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current + * peers should be removed prior to adding the list below. + * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove + * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly + * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN + * WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the + * specified peer should not exist at the end of the + * operation, rather than added/updated and/or + * WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed + * IPs of this peer should be removed prior to adding + * the list below and/or WGPEER_F_UPDATE_ONLY if the + * peer should only be set if it already exists. + * WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove + * WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at + * all by most users of this API, as the + * most recent protocol will be used when + * this is unset. Otherwise, must be set + * to 1. + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that the amount of configuration data exceeds that of + * the maximum message length accepted by the kernel. In that case, several + * messages should be sent one after another, with each successive one + * filling in information not contained in the prior. Note that if + * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably + * should not be specified in fragments that come after, so that the list + * of peers is only cleared the first time but appended after. Likewise for + * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message + * of a peer, it likely should not be specified in subsequent fragments. + * + * If an error occurs, NLMSG_ERROR will reply containing an errno. + */ + +#ifndef _WG_UAPI_WIREGUARD_H +#define _WG_UAPI_WIREGUARD_H + +#define WG_GENL_NAME "wireguard" +#define WG_GENL_VERSION 1 + +#define WG_KEY_LEN 32 + +enum wg_cmd { + WG_CMD_GET_DEVICE, + WG_CMD_SET_DEVICE, + __WG_CMD_MAX +}; +#define WG_CMD_MAX (__WG_CMD_MAX - 1) + +enum wgdevice_flag { + WGDEVICE_F_REPLACE_PEERS = 1U << 0, + __WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS +}; +enum wgdevice_attribute { + WGDEVICE_A_UNSPEC, + WGDEVICE_A_IFINDEX, + WGDEVICE_A_IFNAME, + WGDEVICE_A_PRIVATE_KEY, + WGDEVICE_A_PUBLIC_KEY, + WGDEVICE_A_FLAGS, + WGDEVICE_A_LISTEN_PORT, + WGDEVICE_A_FWMARK, + WGDEVICE_A_PEERS, + __WGDEVICE_A_LAST +}; +#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) + +enum wgpeer_flag { + WGPEER_F_REMOVE_ME = 1U << 0, + WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1, + WGPEER_F_UPDATE_ONLY = 1U << 2, + __WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS | + WGPEER_F_UPDATE_ONLY +}; +enum wgpeer_attribute { + WGPEER_A_UNSPEC, + WGPEER_A_PUBLIC_KEY, + WGPEER_A_PRESHARED_KEY, + WGPEER_A_FLAGS, + WGPEER_A_ENDPOINT, + WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + WGPEER_A_LAST_HANDSHAKE_TIME, + WGPEER_A_RX_BYTES, + WGPEER_A_TX_BYTES, + WGPEER_A_ALLOWEDIPS, + WGPEER_A_PROTOCOL_VERSION, + __WGPEER_A_LAST +}; +#define WGPEER_A_MAX (__WGPEER_A_LAST - 1) + +enum wgallowedip_attribute { + WGALLOWEDIP_A_UNSPEC, + WGALLOWEDIP_A_FAMILY, + WGALLOWEDIP_A_IPADDR, + WGALLOWEDIP_A_CIDR_MASK, + __WGALLOWEDIP_A_LAST +}; +#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) + +#endif /* _WG_UAPI_WIREGUARD_H */ diff --git a/net/wireguard/version.h b/net/wireguard/version.h new file mode 100644 index 0000000000000..ddba10344448b --- /dev/null +++ b/net/wireguard/version.h @@ -0,0 +1 @@ +#define WIREGUARD_VERSION "0.0.20191219" From 7abae39feeaf4fe86ca000ffe3cf87e773a5b405 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 9 Oct 2019 16:02:13 -0500 Subject: [PATCH 004/135] Add AM335x CM3 Power Managment Firmware http://git.ti.com/gitweb/?p=processor-firmware/ti-amx3-cm3-pm-firmware.git;a=commit;h=6a849767df85ce9399494f53fb5c753665396653 Signed-off-by: Robert Nelson --- firmware/am335x-bone-scale-data.bin | Bin 0 -> 73 bytes firmware/am335x-evm-scale-data.bin | Bin 0 -> 17 bytes firmware/am335x-pm-firmware.bin | Bin 0 -> 11320 bytes firmware/am335x-pm-firmware.elf | Bin 0 -> 217168 bytes firmware/am43x-evm-scale-data.bin | Bin 0 -> 41 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 firmware/am335x-bone-scale-data.bin create mode 100644 firmware/am335x-evm-scale-data.bin create mode 100755 firmware/am335x-pm-firmware.bin create mode 100755 firmware/am335x-pm-firmware.elf create mode 100644 firmware/am43x-evm-scale-data.bin diff --git a/firmware/am335x-bone-scale-data.bin b/firmware/am335x-bone-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1ce3c1c596d7a7f400b0cc89bda5a41eed2780c5 GIT binary patch literal 73 pcmd-HXHZUIU{c}EWl|AfLZWk+R0P|Ad@#)bSHb~R0-{lr003gr3L5|b literal 0 HcmV?d00001 diff --git a/firmware/am335x-evm-scale-data.bin b/firmware/am335x-evm-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a222389d233fa8f1c76ef35470b57284999c8df5 GIT binary patch literal 17 Ucmd-HXJAiZVA55UX8=>$024d{B>(^b literal 0 HcmV?d00001 diff --git a/firmware/am335x-pm-firmware.bin b/firmware/am335x-pm-firmware.bin new file mode 100755 index 0000000000000000000000000000000000000000..1fc92e2bc40e9aaec0b3f60e0c2083ff40f22c3e GIT binary patch literal 11320 zcmbt)3w#t+miMi$>ZFtIBps4K9z9S=hc(b4HV-v|k0xmzme+IY+BqTHTJzzGl2v`9;05ky40j~kwfFBSl)8rgej!-)I zVO^=h@oDbbGJDz?CsS_f(z$qN3(J&i%b52!&f-*^%Sd@}#PWP)j@)vXWC|hkyw1Q3 zt{i#g)RdVCpTt-4{o8pR&@3~1rk4{TNAMCKV+Lle%y*^AIkM7Wt>TsV_`J%i-Fdeh zA2_DZsLfrRSAKjzKP9*F_<(2nb#73teC_nwspaYN)JnZGy_Cxo@KrTsSdmwm|<&Fq8B zrJ;NWE4<`C({}^fPz)fev%`dr2D&k&7%)m4ws&14l|^fmwZVytD^Fmr4gEglw>=Rc3A!GBX3Qd2^X3r&;aDtM&3LrgSxixN||jr8;}?@{>TwtRS{Qw{z~ zwR=ZFqx2rt;Wcze(mRmOgY4rY-RBGrN>v{vo~WY|yTTz#n}w>>L6EyUT_%pt!)sx1q38W+b&xpHDvMZ3%Lvl6S@;5E3CYYkqj zn~9-r{gU%J#gZEQy?VX$|Iq$)8Yd!h6LoFf z=j;x?t+wpz0rQia-b!(qE3@QVL!!9Yl3Z?aV=fQJ{VYpzTJ+6a6ZuVWCc;NQQJ)+z zH;U3_dFH*1Gq6n9{TTXkH^8|f^PT9kzEqa5e7&?ox=Py9c1T*6BlNI>hhXU@^)2aP zX>y1&Cim-w=>@sc1wXs`f`5UuL=g@1r58dR<9^Z_u$neojt1^XVHeIdoIYn!RMyK0 zTyONxz)eOfhZFIf&J|tp7vuY%ysO7+nJ9fb_uDA_bENJ@dJXUZ_ZYX8dj_c{?w8yu z*x1Vj@f>2JlnP%fyAnly_vdleK~D_}eA@77yd&^sgE0`#Ta|eJ;}C1*0|sN(pl4&8 z@0|ckIqIz#WLtg{V2KHXp1YZKlk{}RVt+fBeq|vUShvnov{SgObpc zsMQwl<68_irhMJNx0tne)*!3Xqu&=b=vuNAPXZs{cpdUP8fIaHsKKT&p=DV+SsIfK zM`gd)&;=?}sv7L|)$MBVpU2S0NmxM&LVhZ_d2r+q*TQT!4~7ou_8;umB@lIMFmNa# zpkB+}rp;rwq9pmPm?9J4(z?z0r zA;ydW!N>}@-N!B*X{fC)l6oTCYl@^(lu|s<@H>shK=b#e4YKtPWes?jD0V}U^y}Uq zhq&FX0lVq7fSn~2G<$h)unS$DBFWUh)0EJ^*o3zH>+9nj1Ai!&$Oqg3lXsnTkHkIq z3M__&%lO@fzNk%c+&ncnJo=`zRkOkjE1G(n(f@NBtcs2&%Q(<})cbiH3;f0N2}(*E z@+3%`znJp=m&{7-THR})@zcxgXEPm2TmyN5d;_jl8dJn`TGLOu*7SnD8#GS0C(;}L z&`^%v=J%?>*GKvNLQ9rjcXD^hUWxleCvB6seK~P2_Um+c=l}FD3mo^v1=IyMwHxuI z-Z(J&Za}>`=wB+8Blqv4^PGb7k0C?b6s!FeTu_sZCjcpp&+W6kR zt2Yy6jUKCF;7b*OFYJ4{KbfnA&gN0>D#7{n*|>5w_{1o*_m)dV5_x|K*2H62Qxt&4 z;X?nBfIVfhk89c9uG`;|&1_tg@sE5!y&j|Ida_m;?b58}S}aaI#+gI;*dw&qw6Uu5 zVP+(}()-=gS51(%*Rz6~2VZ`JWcK$)d`*6~^$Obf-F1li{^r3QhaT@Gt2+F(y)VS| zpV%M&<%zrF`c7oWKYH?A^bg6TQjgdvWpO`xHc_O0Y(q+LJhn-nd?H-qapz;mMIm3+HLQaNAk*e_g2e;d&amuV^E;UD|r-{a^^eXj2W9im$F^byxN(bxQ-X{EPSx7KuNnDc}{B(qG zL!8tvV9#RSF^r4k$`sfBlBECgmu5MW%5xQ-6K3<0!3mk1?Ik66hW&=}gbHZYHI^Ll zIzobN=oI$IcbS)Bu@A66*=Hh!z3Ui2I{!NuI+GDA*oRmbxI{xY7kFynix&kvW_P4|aFlR6njGxnY6lOQEBW{#-2n0G zDjlgMA<>#dPULBtH;aL8Mh|?iE zUcS{UV%!#vmp|gASS}Yl|DsY9my7zl&o%(7RI8YlYsYq1Vuv4_dWG%Li?{Mk@eK(r5*sZP92mK)XSs z%>ZqqMwQKJ=s zcB4kS8Z@TSt_E$cMze!&?rfIY}p#4ds%>gY#qs;~F6OA?( zG^0kF2ikvVw0WR$8tod;J{Uur!)nkOZbTn{9k!!yZwphsRQlq3)VI{vk^H{*sBeEg zUcTo&=H+o-%a+sRQ~Hq;G#4$HI}>OpV>AJL%Mel0sU&hrjPPEj$4-wCy6cj25}gH` zN(?@nGbS~aFkfw%9w(S=jA-5^XzxhBQ=L0!F!sbhFvju(SGIeld%ycrclPR)tEqkw z<)qJ~=`&5xm*7e$Ppn9+Y+n6`)!Fxuep(|E#`6*VB)wSar)zbXOJ}#= zS@VklmT|x#p66IWY3+2+VA z#W|Smlu6U8jt^xs@+?g}FR&es%yB(>7wdO$+3WfK2L9HNUA$AKRV3m^^cU#ISgD*T zGlgd{dLrme=i*p`JV~x}?2YGqbYA~d-X|Mnv%K7425ftp`8b1(dNAIWk9u-Nry9Iv)ZntnTp{zByE&yQ%#;II1#rH^ z+%Y>+Fs`>?ZFwGkDDJ$@bsbK+dMS3+ZFvl}*;e=qP~$x#J&nf(o@QA0hD5o@#XEA| zJT|ZddH)ctDn+VZqW5x@<*LCHmCG$a&Zu&Vit}PB>iyLyhuTVT-d&|y>rcfSPEK{@ z)Z5-1>ys{RM#&pPpY}23$3rycGPN-`Ejs4T`)LN!FHsw5&l*!X?QY>H@*>Xn9a`J-TzOZt zeaE}h^0UKaZNbS{`?sL|nP~sV?UD8`)!Kh(EYki@#;E!FXvt-23u)|GZyT$U^BTBh0M z8jgAEPhn9`hef}QTJ%nsM#+?@MGG~HLZcCH4Qig6J(d8QczCL~J2Em_!+(Z7|7XbG z_iY;tou%?8(Q)~Pw!+dFGrw2diBT+Wz+B57r9R}x?b)x@|NGHMZ*f}vM@9=I4!zbf zYTbhQ&+|5&PY*pbM~WS!u@8GZ&Dq+{IB6_cKE?{1zPu{yVw-j^B;fAr6vo+nUvrrq z<)4k+FXv#)a3`2j9F|f4%R}Rw!okOk?MzuaL^myJ@P*;X&V6V&wsS{rTJ(M@-Gx#$ z*g8BGbYtF}2vO|t;4tkn-$!gv>oCH$EJR!}zZ}I_COac@$c)>Hrjp#~EXt+3jZs0O zx%Ap7&6`(7Y5xCklqvgw1tZvRQOj(s4`abvjE+`#)U36Cb9bCJlm6%fpVhjh~ zI?WC>IO!;hJL9DOC4W2E87JL?@{!eWkj;(kHMIiWVbTg|{T3~9-yFNwB)%ns>_LhM zs71!oh`biZB9^5%7#ixdI2ifj&I%VFXnZKsy_U0Lq!;?v;CZ#*x}MqQ_^ma}c8z}x zp2hxycrNfCMCncbFYsLG{{j@Jc87R<)lA%amiZ1sYOUXTKeOHC{{rbMt(3~DeZ}1r z<*H%Q|DQgJIKyG8<^TF9LK_HET_5@=O8hEJHU82^(P3Yh>OJnGsPIgfYCqW$v=%N_;r{M#A6hFTb{xS9#C;Ei?DCS3&SKOKB$`!NSm~&xj%`0Jwm=A`@y4Em7 z#RtM<-TpAO=_g@o&9h-@$DT0N{7jf4*d1ZAx-m>Xcrr}&Z3|QFTQ$3kRfY2Ni1doP z6DrM-zDLig!P>A*K3Fm^Zn4}Iv(Jn^rMYni`?3u<9;SW3h-emjgt8YiKn;F0Bw+Oa zU`U7OBSSPN9>P@|Qy$bJTbdag-l2PXLnoy@TG`_oUrdJN9|A=UwhS?4NjMp62yaWr z9rA-=+6_y)m@*c^T9p<%tr)ryzl;s^QS6NMr;BQR4R^@&ZGh?`-xhZ+xun)Lm!$gy ziE8kU=(YEABf>?p@8)4aN@^FS;%+K^9{13=?ZsM)r&kU3cy-$DlF>`+B7reNjiBqtvQ0t=;LiPa&W?TZ&cVlsMlq1XS?C8>G;ZG*XqUXNSfOj zx?$DAki$fNr243CtzFSJkv*}x-JsDPM7u^)N!*091LhKXVJl)z>PzbTDV0h`s=N26C+TGbETMQ&m3B>AZly?=kr@|u(ivt}iEoV>_v zJ^q*#$ucW)_~ycn8+~s{x05Bp?V)+1zW)Q3Dj%<*{fS$@ZM~+!M3&TDsYo3H@v}P-LE>C^f2Mrwa6}J{r`S1sKThIg56z{>IXrQ~T*5gV zr|=b+t+izdm3ri`4!POPfG>W+IiKV+uVTyq(0Jc4ov~Gbxzp%+qAe+4PEPr4@|P*h zuvqx16>aM99v@g(cYL5>8GS`w7BZWJHZ%MZ!aV`S9b^m6TbS3z3APxbcL3YZ#Ulb0 zkTc*$Ad-vBK@aM77cf=~*ntzkt3WGYaB&4WGS(orER;(-7@}~Km+m$qCq42R$rpE; zqy)LrV)W_p^`^Ls&X{LZjHqcYOVD53jkQS}#+;{h2lpndxSeU|?bd*fwGDX`M*{8x zp+_GnkJfbg4&|?vMQ;Bh^>Z#xTp_>PM|_-1hjiz7dY}ArEluC+(2|5WNs#2q8rOe? znCEn;?aZ4YcI@7&`^xXDcrV0y3KTAWR^t!bSYtxt6!)~Dofi(`dxL)Ow4rA%2z!5e zxUN1uqIyaAu6nI^yN8VOOVn7f77-4fAuu0|R{j z$D3|7FP+8$PkYoEjsZM#mke&()RG;)fCU;ns?9O*q(`-4WglcE*hL>HDc5^SsyxLu zPiv^re3NToMYbp2lj1pC&t^J(&r9VO`1?J(R`w z+!>#8f}1b+jE(qD28B<#DiT2WOvscA*+M*i*9%W?Xb$n2e?w+ z7V`fi)}Qs3RTj>dA)W8z`kxe%rs;Zw8(uC+_2~+R{CP6u>h1{TnGGbnAmB@sCk;Ao z4*QAcE3{RMBSaeWJ2B4wr}K?uk7CZg1ndMv%y|SJsKHym zL1W@R(8|9-(`mGu#%U!RH$Jp(t@!Y|O?5RJzc1SBr`L-c?-fg{=8Dx-qExf#fk#$v zT35uLM>|`ASAjO*2yo|gdvX0;)9dDo@CO4H-FhotOuS>^t+&v-;uZ;aT1<2(uNybLU-i?RCZ1ufAu^+I9EdC)cdMf5QXc-&p(Lrn-kVKm17jmLHIO!S^?B*noRY zeD7k3$R_~sAd3g$02M#Hm>$pp_}3zhwwS|MV?9nEU=T3&fIlPM1AGRo1w`<;@fPAR zCh>--_9H!9G)?+5ahJtBAc%k&AOi#-%4pJ~JeuB#dTN2^0Zs0|5!B?Og3aJB2W}ge z`)0Wy%D(&MxM-_@JMG)%bsob>m&2 zwfpbwtDae>oqq=Y0{lIFB^I4_<^zR5MDB0vt5($W67V{p8T>as+~8jeZ1_9nMDXVW zg#Zm%@(=Zi08oF>{NVxWA0rT(SH#C=z()8UWX78C-V2})*bV^w!O&k?89mTXY&Sst zLqGIqtUl5mtB2~OX+m{}LM$!0GCSPd3X!*EYRL^(ur{iAA0v& z6M!!9dl8L>oXA^ydhn$6>$~)!h|9!-F%O%fy&Y^a-rvPfZAb^m{<2$cXLSj$eO&YT z#q_t<-m(AUb=DgvZ;rnQMOXZe;2SgjmZmbcIU};>{WaR_;A;im0N<7<-<33uJqw;j z;I&DNeHrDulE$&ez(c=9X^iEfVQ}|fp%FFm`-mEc-(^Z9AA8nD_5Bh0mPYkmN#oe^ zDBl+7+d~eWpm6qtbWCs71lroD>}tq1L}khE#Ny$YNHUb(9!>Y3Ao_^5H=Rd%J2!Hg zcqskkgfw_A(+!u?o@kopOYBGb39%nc(dHBVKOX+8N&9Wsc;7$Wv-#c_%N6hc4_oH7 AF8}}l literal 0 HcmV?d00001 diff --git a/firmware/am335x-pm-firmware.elf b/firmware/am335x-pm-firmware.elf new file mode 100755 index 0000000000000000000000000000000000000000..843fc22ce3484251fc9700f013e13fac20896d67 GIT binary patch literal 217168 zcmeFZ3wTw<)jvFQ_TK00b4hY?A$Q>90)!-BxEjDq4Dk>M$W=u}8U*x!h!GIMS`BC` zDk_LtglMU^YU>3M6cvP4Yps_ysc5ZQg;s%zwn5M&-17Z?v-dfm*7xcA{-5vtf8X;= zVCJk@v(}n5zq#z0*>lbfXOE6bDMkFxBT7U(H4w~-F@_MPum*~Vu#lG_Qr-Nx6l6-B zU@0Tg&G#T>t2`Uz4g~MX`C%}^wiC#Qxo_mBB0m-R4E#$ukw5;^1v4DxH)HCDzsb@m zjetf#BcKt`2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)|1T+E~0gZr0KqH_L z&P&-3VVH_>+a`iqH??Y=jFCu0U9Va1X-c2-^@^5X@2OvAj^8IsDM?=MQg{ zMtXh2h>G;xHKO&5=RI{s&03KiYZxK6zg|ZuDd4T*T#7?l@WyOBEO&}s76!mi7U zUDz$9)c>Aul(GLt!-KFoCMvSG%Lw!swo^_BTNI5du1k&O#acHNjWJq%zQR$b)EA!f z^1&TemxhATg|#mqw6Y3Dy?n5-$Eo$;78wma8oJeH#JY{LV!=>-{=_AzOVVSZ=Dgbc zk&XK*#JKv^!l;obac!(iZR^(T5wr4JTQW!bV@7Ygd9_Fz`AW9jf0H>QmcLI8;5@;s>09`+leK4MXjAi_Hc(+Mg2Mp zc*jxAzV2i!znM4r8+qZ)qV>;5MX_l8@Mt6|2a49mjt(sEXsP>pd3x%owAh;J+~&@< zHJze$Ub}4F(;-{G?C2g5txJv$LVW$v&b9|T1*oj{sxMcic}E(cx3!B%cv~bYT9+Q( zwO4H43M?iS1C`C4ZNKZR*nSK=(knZww%gkr`Zmyo(EZWTZTsvhQscH#=7vtv^Ag;% zw~FoXxtP``x*ku5$6qUtlagW{v$UuQJ-%_UFza(SM@;8SEz!gj8EpZZ@N0 zQRRjn$p4@-qg|$ml8%=8yh>oxzS)PoMH^;qNRFNs{ZY2u^+dHFE%G~8uB~NPUi&sX z+P)Pe?0Hml)CV!oY;i+NMv+= za&3M+#_|#GuX3U_`POQ=iny6I0bumg&L1BPFN;J^=!@->q%E@H_jPE?6$rA<8Smux z`A6$mQEWwTi}s4HT>n&bW}dlAlq`U!^E#i6E{tZj%V6?;%j{875dE@6obqMMu;|#< zh&?p=K)V$3r|S+Bg|5!udf@yN@#VfHAMeX=?G$^YDfhVlY?%ovPeu^;kbS@NFD~~# z`5c!hq4;v;X- zp~hL_!23(y_f;ME$CBUypHbB6Gv00&>x=_-Fy~O?mEPtT4v3Vk+eaP}i(fw=0{%md zQ$^9d=_p5`so+i&Klr;aUsJSYjO7k@vYwa@8`mZI% z+OYb}IV9#=X!o<1c-H2$Hu{YN((oXC+mb%$A)Rfjj+<*IZlI@e-40i`>yqce6|G~I zR4f{|p|kD!<7nd~%pfK0EzCLd(9x&lT2Xf9q4uXdYaZI~@soQ)+kvP22RaAKGpf+r zXB?lO-_bI(~DfMXcAfJvRH#9=S%8R<(MU5CvibsdemAl=)cMr>*IGz+VBL^sIDR zqvw)Zw6<|qXWJ9Uj5X%k9Lw|0it4+f@;y&woxZAi{Pt*g`>5p=ble zWuG=I0zc9>O8$~hYop^)pTIG+#t^gSj1BP`zH`*6;Xu_2lRicJ&u`zea1nd%qpIg_ zh~Dt^BTrSJNA`_g46DP@&-z9uf9hE)2Y6P6_juNN@EL-)&+lCAdv522-pxC^`Np;e zMCm&(cEmkJ3`sbzRhO;T+^t0B#PM%1o> zPTq~x)rdP<>{|awSoJKc-m<-_-7Gu&0iKaBitSvBy@KacMJXseyLKSZ=6@CJ&B<6R zR2&x1Ln>l#lLu}+>*rtXPPMrLJ8)~&y03PJ1y;VY=&Wkl+Fgp)?w-lpM^%fGE!(?S zRHZgzD^Xvys!Dt{t;>cjb)3o`aP2jH*hlZjV$njA+~~cKt_ke-pbrSpRWcaUB`{ zBA)gn()Q*sp5U)^h9jc2ZtIM%*ZgX|W;C;e&Nk}>+KWM|vu!afkay&HXZ>hzE`r?) zmE9uPy&ZP#&Bfr2Q@mpEZdAMy@S=)W0^VZ9>kZy%iq{*wD;2L4y#9(;3f@e`D+8}Y z@yfuvNb$!x_+;Eh+jKHz02ULWvAC|+Oi0*cocywer$6!3)NodVtf#j61C>*K_y z0=(Xe*AKkU6|Wz7-4(Atcz;v8{@`^{yaC|7r+5Rv3o70~@cy891HqGuHwe5}j-$>U zoozK8%h1OE>Zm~5PU>K}%>DUGY+JUqlfLgIw(V<){FN_>?FRO1xv})v2iDP@9EQ90FVuTEY4-px$T<_I}~DB01J4 zwllkV+@g-!-r%PSKrVnP_wCwD~7hJx2*88(wIr!_02h;+3-+HoZPODOMXZS`G8=D(rqe#h+Pb@~knlmR1!G$39prkI~X&Rlkpg zkrwHZ&rES!Rd<RVVgW$=un-{w2nQ_KL_wG0q|mj%Y9NcZlJ!?3idZM4TT0f7L#( z@W;|(qpI%m$!6}?-;doF3&z5+$yH&#A%U5nk}1;-+bRu>q)?Cqj8 z#T<>zEGFdOv0X>@P8P!c{5g z*NZW?Gy#Xo&8OC#ik+?%jqi2WUWd}kE@&x1iI*JRwQR@1+Xd#m!;x6uI-@G@nH>km zBHePBvr6AiE6UHwouZ(#Eu~Wylpq!Cl&ziEFQ%g0-yM@Ets48?DV;@&KJeM^bgRo- zRQAmAz0t3)M$VIm-`^`*?>)>um#z9-cel@d*1{2Vai=W#v?YC%z-ZvpIK3Y8-f`|a zFF78BB~kX5mTmhgRh?3R$(=_Y+{>{b%06!SJ)TRfBcEBvJEb~X*ot%n`}<9*?uB)Q z->Lg;FS6#J9-+4-@5JlB81>Ia{omf;)PKCH|5L}E`oDLa^}qADQ~%qJi`F_-?^lmw z94)$NL%grFAg75e?U-|Y^RsP%Rlc3&`gS^eJ9aeb8(!`2*aLrdcD#%DUx(jHcy$@(tnd1jUzh(K zza}cb>W;+G`dddNmI066a6NjVgS{ln_2?|+QTs6mtwRc_e#iar$pBK7+nk=UuHzH< z^NYhRd%x|2y?J=-J-1&zsb*OAvG7Zko6(CSb1>F&kFgDnggGrf77TIu|l=_Voe! z+0f?IBPx*p;ql93dFV58r)aJ0h@t#P4kskdhpr255Un!~^Q5J-?SUiCll!S7@h7)) z(qgqR_fF(?wyisIysaMNW@kIs4p$xFQ)VvK1`Sm~_?CkeR~(nE=*#re8AD;5R;;Qn za7R%A&o+*kQI4g@k8#{QdW_@$=3}DuK7?UM@w`PT{V+cqZ)-sBSO-MIs{U7R^Qw{b z*JkQ%I?DL|qwlJ-9gd$c&XewQZU0c&UZiYa0J|x$>r*zPu(>c{vksV2??t&LaZ)Z^4p@~K^1 z^)b!{$uBs;$CI0UQD|>=o~YPks}nZI4%;cIF1a=@mOL`NIT@#{DFgFT%zB($#)3%6 zqV#&vdJYgXvBs#vr$_nG*_O6dcz4&ZeTjFK*j>YOP{WxG4~YTJv!=o18761QqHpn> z^XB-uCgsK+5?66Wz#0XIJNjxJi?uA*!LTr2t%H%S+&r?*IIwI%cKr-lgq~jBG9B?L zEk(0MS^t)zS)y!E%XGw*Ee|0c*76W?&uIBO;gYqjlraab!m0smG|ERFj_Gr1zwuX+f*hAF^6CTH&kNX!!n{sUI#&fw0 z;pGlK2ZC75;u+C;7e+v5+gpcC^!{rPdk|l9m}6oAj^ae?Rcd9+kum25p3~c#NjIvz zA1S%G4)qs+)7iH6uxK6Ik&HRSD9gYZ@>Lyt8V-M6v>tEAT$LW*t=QYJemQ<{FW1hP zf1YQlkK+uP?*>>Fv8~)Z_Uq31tE)4b%_vLQe$-N*_Z&69qJL)|F{4QvBGJli%x%Is zG){Xl*CO8D*|uxDM?GD-?BP5~n@&0>k>bP|uE=Yf38Gfb8x{UtOiRF^b1>73*-$)i(7rqm9t?_Xb5Uxc-A`?e&tGi{uh zUcVpxjB@}nF;LyxVc%P873>N)3_ zWx-~nj6DZC{Q4_f#U&`W&aTg`2Ug)jaAj&llBf6OK61~u;yf2vhA}*wQfp{-G{d!Kt8ogU6qTBRfxGr#w!+M9(S&pnu@ zEPcBuH0q2|I19jPpBOMNOG=CjwT^k--p26}JtB+vt_p9(BDYng*E>CY4EB`qcV2g2 zwF_hNWS#PVtHZr+TxSJz=!a2ew(#B6@K&K_a{9qKot#(nt-J_u&xncgq0L3H3-V8G z9v6Fg|I5a>_5tI>{+DIn*r-_l;idIVkDAptPS@WVdn?vIcKYzr;r&%!;6&cIJ7$%t zJ(ALjyfq8Ykk2e5y}oYtt+V~JbB70G>D5PDCbgSqUYP${3)^l6{45&Px;Zl@`c^d$ zfFGwfKN~%lo|xyh4~$s*UlFOXmuK<$Bxf(3tt>P%=f+v7>Zv7XSD)a9eX|0wsjy%8 zna>lbhg`$2V?T)vjrAK|_?h$!B_})rGKu_fRg5VC=?n8;ZfQl|rcc4T9s7643uEC; zp8db}?8M%!(TKz%tv-2Ht0fES(=K0)-XO}3w(z~9Q;&eFL)o=4>NU^C`&H@dZParY zo_4v?0OgcnQeIqy6(shBmuJUzY;^3fKgIXqYp@fiKO9TWK0I*JSi(LWyYLGzS{p|A zM_EW=9;z3m2bz5+?enRh-Af1?f&G0>4zL++*(UM33rTm*3V>xeaquRPL@ zdQ4+wS>w9)W#KdG&KjB9=xa=A+_*@T)-*RoYrixu|G2$<9m0w`qH`+DA!83+J74aS z!^SlyE&IYeyz{Ac(QxUabG!YrUDV&T@zMOpar)jr`o+fm`A@Wq^+h$zPivPrWi}fR zx5z=zpSK{#*maRFWv3i!HV2pCCPS-{@^v%(na3wiqz*o)_kGlzcd;marA0)xwTH%K z)@6>&Zp<8&)VQtP9@k}b7krxIdLYiI%10coC}_XwwCeVUmpl4#i=l)?QPj9s6j+5< z@4aQ?{Jm>8E=FGouisVkFv5C-O$b{MN)XBsokDL&7|(0UFK?)K#q`BJA_a(a-*;HI;@W;ltN5TQ+hmr83VFYzWkxiL-j!e+4XE0IExPR_gt8h-XP9ABF|5zW^ z35=A^wkfzHU~i*s@;G`r+a`R2CnEQOSNjc~NAb=`@T#x8a>2|Qk%cqo&7XDU+(^Zu z9*ZJZUK$xbWNt6 zs!Z?&#^N}P%NhhG`Bw!x#BqvXBjl?*j2C>CBbYX1*^dpJ8zBfAp9n0B^93u+@>oHZ z&B0CoIFyl<7|O&#{ki!GJ=f0n$!FkPDjz{4!xw~52!0~KR-~(A(av)|RosY}^Vj#; zLzc`&Bt)agQlC{K6QAEF&pKov(EkzVoGa%0AAft+M_*+;JLCK{U!7WXdgj%>OOU1i zGy)m{jetf#BcKt`2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)|1T+E~0gZr0 zKqH_L&4rwdV#>6H6j9=&n?-pX)3PjKQn z&S!&b@2{{o-nI8#o)nW^xy7)z5^}1-uBm?i#(2(2ycw?UH0UmIbxqeqqZAREq4_=jq+$G`JaKlQ6Bv`PXG8M8ZsyN_6fStr9D(~|7iatmjql9DnAu?rZ_@#uDt9A zmt1|RtMT2B?QLtn$f&9It{@QG{Pvm1n!R)Ks&dP*U0n&XhL(p9{|Al(CO zDGbwB?-jy30vbxjGvp7dTNWno3wg^ zW9cE%jbDOq2gab6dzN&74h6O#_AKoI@o?Y^mT)8K)If~#H<3;gDW5|q%m@XucF88o zN=-J~@sYL$qVi^9AoE}$ecsH=kS2+)ZjVP-&R9W_74mHGy8H{Zmn$glNUl(l8BdY@ zXUUG5sYL#w5YD2DCoCPuP5J(A;cXf=^vkQ*oS#)Hql{7%S`3nzs9 zU>n$7QxJ7yW=KrSYAycb1X6oSd<)Bz@p`NQCLB>dnQ4Kvyp7N9x_F= zNf!U)YGrS&N)&$uuKXEWDh8^Y`Avmhh|9)dU+jn3V?~BG1XZ7nv1E$Y--*zZck+qLG#+ z7*UPailk~h6Sf?wsgPPtsXzTDF2(1NlIoL?x&%_|DfRZgxKwpq>O4vnvBVjWdW}*$ zRcV*UC2pXEh_El-1QDmNUIsr^Hzand8!``-K{u>Ho&s?rHB~ocS7)`|Zun~4_WR0q zVI#8T?k2=ezjz-Oy6r)p+|5S)0#unAIe%xHB|}aXNd;Bpr_aTUEC{Z6gd`IW*0>+ zqClo1N24S&OOfwVAX|}~Cd?c~4y0@@$$qmDF?iEOlmg;8YB=kWG_*U{BfC%u{+$cj z<19iv&RWCcJV`v#I|{NMkM$5N8_wwS*j6XYc@L4NVjA>4(yoP=c?D$$a7cP=-+4$_ zEr`NKx97m}E}S_}7~Nwag?H}NR|q3Q+Vt_UY7|h!_Wg`>As>xlVKe)82!2LVz%@3J zV*eFHfNWvi&ytNE9@N42E7Z~GsfzG%Ha2>ZZ3_E5NTz~xb#_3f+$bFbj(5g1VU#&u z&6zn@809B<)rYk0OMA$GLY=Buax{j zbUMSgi}YYRlkL>ymvLiQk-Y^>Wo9h|kD@Ab@F|;Cf(p$-pCg;k*(49D)=a_%@d1eB zFo-=MIA5m7ah?S?h@@ih4p6ED1kELtgYY3Fv8SaWBo9W^6`>EZp(n?R8;!R_0o(oN3jzTQW-H1KrOvHw{ z3bAP>+i=Ki1no74A@-RSh;8!;#D4QR!~ydb#6j}`#33^Sagte#7%M@yvWsj+nYMaDbfFW%zXcI}cbechQh`?xWq}psQkm0CNRBfOO3b232T=!^-w3|* zIJ+8HB`b6JG*^j?6k1#g0-{)vje~@;cmY_!pCI~%#qb^AuuIQ?9Ln+7g(7t|cw9ue zX_53BR7LhATWt!P-cA(Tlf}}{@B@fwn#~t?IQ%e}w9ffAZv83nzG0n{q&#Ik*pumU z6Q4h67;gP&do9?iN}q#JAe~cyd{w2tf))G{ky|BM0Y5nRw?SeM0*hP#-fzI88zbYq z8$|lM;BgqIzmF&hAeONE^Dc)h@lv!BiRV|xMIUZ_$ZPr^Kn-UWd=Ry z{VdbGHt@u6k(&Ww*IGn^S8 zPu7skNAO%`@Ro=nFmH+ybU1g6bVXn0(TqY7flt}=XT)A82T6qClJWR=l54}F3b1xK}h-2@QDopVhi|cTHor`D+3i6=~UW0gob+x#LiznNullcVAbfID+M01syS7~Pcf_caw$-S}vKFVb= z8Gy%uJHR^w2~0e1BX&Uvy$RZUl%ic}vI$as@hO|Qk#bCSr46pol76P-hz-~1P6^$a z`+Jt~Y`lcz4V3c@p#>uYtEQlNCTJ8{>&2j4$y~)rSxC}iW|$p-K|GC=Dc**p+Y8?V zm%Z>!*k1&JRrth;g1yJ%^d5tgybOPVObR&vR2a*LaW~-)8r`|v${nHwR%2w@NsPzyIw&WEFw6EY zHm(D&&?^GT5N?97>N=Q%S5EO#bY;B=Wg_br$btrPV$gosBd((AH_Ze@pUKUe$ob$2UMuyj6WTQd4 z5P}6z%w%K|PPTr=KFZn4X;D=sg|mAW8=Jx7%la1t7(CuAc4r2|E3QPQDMCoNO*S4r zvdL!!C1Al( zyF1C@1Hkp8SDa0KKZfnuES-;X6;mlhbmWC3S$WRHvgu}Y$|TkIE+jbt%p~@%3-3Z| zJ5tHYmwHYB#>!1Mq#$n^CI5=N@KTnok}1<;LS91Ky)n_JUpZlF6e9OX3-`$h#KJgp_CpqmTMLFf}8yN`&}y1dAPapD|+ zrNlX5kB43%dN9Qo!$dY*n*!cx30^*C#A|WRGRMs+SOnrSh)i)rETgp;-WSk)YA*b? zW?m_JcHCndYl@(GJ7fykaf{USX^F?dDCYD@0_SFCPv*XeL@!1W&PvyVQ^NU8j#m@Q zbuilvJlPZhv$q@$A{j%~Ih z_M5~hVBUc^XeJ{LnL7|CnU^9?Hv1w@FcJW&i^$5ROA#A?q=R9SGA+q#NI_GhwVs+IN7JW*@BOJ zQ>;O@3!Fd60j?RWDEjqm0{4yn=N=Xs?U0Z;l*2{?Xx@O>JRRyXk0drB?;&p}3b+zU zFZx53@O=c<*SKkUWsZZo|0oD9=KUTNcqxeh zh^t71L0ktSkPG5w5dTc*TM@AC0*fnWy3k()N?oZ8mo-*c6=2;Dsa`Itp9l<=!u|wQ^9j_VOkg$ zQFBy1*fmPCE82p`f}@FT0P72Dwrl1OC(5_xf%gnd|1HkL!Qidn5f5vHtI!$G1n*u~ zAsy!F3o4QHfU6*kIdtOUhmm&NKOa)o6RsKd4&{E42tEs5!u=w+|08P7*pkrXsiw8V z)#TxW(kvFi*H6~OQ_Z*x40-vR3C%aCxzp9;X^qk>5y6j6*2I7|7EyEFu7oC^4c0za z^H5x~H%@U**6a<*~697)aft|s@-s-6Qx@P(5# z2SD?;)ZD%`p?M=UUvV|(#We?t;2%!b90<+5)O@uW?5G;*r*N>o=W0F?*Bm7LAA+Z5 zzg-~I?6(Jmn*BZpp(eerL2%OhC1swxcTlK}WrZ0Voe&ZR2dy;z3fhBZk~%K1eu0$CqcT?*vgszfIzquB0-g7*N0z7Gq|Cir$>_CyvBiM|R z;~NBjF;anX#3hwy0+ta0l>Wy-`t~DYP#UK2V&YiM=YXZv%QS^hjXeM|{)DnTI zlORR~3f?Ij4s%9igK`8`P6zF(2wlXS@U2Kl{6hke`GEG48fd`rmh zO*-4x$-+xX=lkv=T}C?ME5bwEEGJ#;yMi|RkS_5Zpl)B%<-%w~My1*BQ6T2q%8dTZ z5Wa3yP_M&H-)ZCzB;WQ8q)|0xhJ2&RA54DOS5N+_q%(Zw~X{1XWr%xwc?l^r0=?cf`Gf58=#=S6Vo7G%Q`Bu}>;mi=etLQ`xX;T%;-ZYaoXN9YA8$Q&i)yP0$@XrI<^2gEm8Os)g#9Un z)EJO_Ee8itoJvzj5yIvJSuRk;?M(=&G=&r)?1xbwd9D)Tn=QwZN>fM?!u|_|#wwwH z2_coHkRpUVi3pBVLgyufRGLDH5cU*Wx>gDCm6B7ON>fM?!tP5;4?lr?$;jV;2i3!; zV$cM4;L#%=rFw7y5&s2AZ0^_Jjk;+ZUQAyOjQpjso#ss%Mgsr?gbVcyP=jU;~x!BkXXDW$d`k;aCh zcm!tTP17Nh&UdZM%lH!rk;b0FK2b@Do4F#&mDkWwx20HxlespL7ZAvdyDL?BlI7Oqj@ zQ5D=p*EcaEmmQ0_KnfT4xa3N|4Xh3r$cyvy?*S(X<>arUGz%1&R2&RN^eiD?=t?D- z7iV;%hMcJ62Pk>b{XrJmbr6?tMkUrLS?ZQUgcE=F!_4b8Uge#p^3Dex;oRSy%TL+; z3YAx-^5!#-Q*QyMvSmmWGD=(*>i!#6ykL=1i9n@@Yar*vcTo=VykTCjD0+}&DoZM$ z$u|*-MNtaM^QQ4GMda6riZ(*Z=O_eGS)WoUS_COy+OfD4F5h$Zd>pdw65w zgnj#u$d}!UVM5q#q`Q+g?F(oxLfW=hlV3nOWPd@rkaXDo9qA&{8FoL)7n9Dmdy(!z zI^TYhbT85o`zFelkS?~@k?u{p#I`9@M!MWyOny1(3i}rF`;Z%WKbH02y&qonn}^I*DBE|6lD(*3lU5)6dlsV|1F3S ze;xoii~lN!$S~voV)4(Tr}GiKj3nS!r~JMuq-?I4J*hYyfs;4&MEj2bDL!YJ;=U6l ze~z)llE;Cy5VHW!Xw=pJ(8*W--5dVq2uPZz!j5OiW?KH=x#911Z}_*MNS|}VpFXG? z{$8~-^^E74l8;MM&jgi+rK!)k;jfZdn);j@{wm4wz@@2YvPxoU>f_SXbAhAV)%0;` z>bcNK=9@k?w&x-znTM?xkn&7%baPA}m!_VnPBP2%acS!Lfg_x0`nWXp)H}&8rcYfj zndT%jOdpq~o=co$y6JOn_^TwAratF}ze-|hs;(k>W;?=In%cPG|1Wsv=TVI3D)RBd zmafiIG9uvK@aI`vkdH&pT!I*KZuqkaL(UC2|PVV*RzwniFbOTBQg)C&hoy>PJ9 z3kQq)L(@_(94z(1!BQ_AEcL>{QZF2=9LNML^}@kYFB~lO!ogB694z(1!BQ_AQk@qL zY0jF(R2S-4x|z0{KBxp4u_ z0ZUz|v($w;OI@h5)P*`rU8qatL*NE1lR2xi^{w!67OS=?ma|PU)nrbeylF$w5|*0G zEj5{2YBIOfWNxX++)|Tysxz6J5zHsbt}o12mzFw`u+))+>z$&c^#jd$q{A4p{}r;s`>P$8Ho^H@V^m6%TTaBJ(cr%4zgo5U0{;6NqXMIdFrT zm?d)_b$Lk3GufXwI?`WYmBu}RyrW3UrpNJYcB(XvRk|lpr3uVA&LrO_%89H}eh!i~ ztl=ak$1(XHbI)ZHOh+>FO;}bHeiy_TI=KhLWID;IdOC=gPB@u$2)w!A{plp$Vel4% z_o?EImlYz*gMm!eaaZhW)N(vd$1*v>Z$d_90EEIM8N{97WMzOLCgrFHF@SV7SPHMc zAn4O5C(j66JzHdQm$-(S=YV(w1dsI`&8$nnS`XG@mj$0C3^Qg25u9QvWW+34*vHM{~r@l)uq-F|6NQf{F?~)K_(Uc_p9c> zEwap4VafpD9Y`>9IFP(SNKQd5 z9Z23HB)2lDAo&9^xSvS{$sd_~m5^KlQwoxKAQU7^K`2O;fgmK+C?gKZ3h)TYc_;Di z1&{TgrFaUG2VB-tS1bX^gUC>jYy_d|??^cH-vXwp|4SfL{a*(`OR4Gq&i8j_>PI9W-&g%BWL*4dchcwI3ez;rqxWz8lR79bW=6XmaOt3R%3i4UL!0JHH#@2&tGrfMw5iO3z;`QQdsGlU*iLS4b=8l&o2 zlGAk%8{7iC&K#~%)ktM>F5io?v)Vy?31Suuv!h^rpJ@W;W8h^5Kyd4unGYfzL{>2f zZi`pD;tRmztNh;JJrU>KAhN1mIj%QomU6YO+|Coc%+cTtWC`OyoXHZ-12GCjha-hI zyIE6QRubw(W4NoGb18V|LX0go3>(v84y~)@0t(QxCrU2*2&_MV#R^woFxt*>irYhs(N38VD-j9iY8FKqb_T@%i_})M(N5zmxbzi!BVvf zfKZhQgJ5Nv93i%CW(IhyOinitwBVE>^7nBH_>GK!-B-EE7otupyrSIPK{tCMTe;a6 z1l?>q;pRY>6@a2s`K+N}v5>AV3vQkP7TwIM0YUGEI(hW2_#E))oybqgfIA<`abN&M z3@E<(bKLnzx$_a-IS<*&ov9$`&QIKefZ7a~b)UxavKgufH zi)>Ye2SBh2BO&EB{KGD5ipxS39s^5t;wM0;R^AALRaoi>sVY1R9;=YG6$GpBK*DIw zt6+Ts7Rwlbi+t?WvfJ|z=-C5q_TtfyJ&3b#FP=8qgB8i|4tj3pE~tcGaQqD7y+yC9 zAmv&1F%rFg!!6;Rv`~5tEHc>9Qr?JS@Q*0vohJtW$WmTrV({0NegQcKv%K^{d@vYg zrRU=T%V6{_Jp~*~s9_E#-?$248k*FBHmP4==qQDh%EMrT3!E7|hXRvEZ<8O!qy-VZoYy+jG@voEw!=)S101Wn! z()bJDKuKx*+a+d6DUUH3tnsBur~!jjSL%a#2BWl;k0A!5y!1kRFnIfw`r#FWSyuWl za2U+KrHyD!22WGzLU0)Dvr758mcf@(%D2A^zO2#}$Ye13l>P;V7`y|;LCk%o_zd%h zd(z~=8bY1)A)jB>;X&tj&wHGodI)|0Pd%v3?H|ygj5NmoZ*OjW&gRzra)?)LV_B}5 z7Dm_pPqwi>_e@f4W5?0u@BpfQcN^w8_Z<45 z4!Qj5kjp#%W%Pvr54pUFpRow=kjtC+8H)f9xx9&=u?X;x%bWNaivX5t!khRRivSO~ zyceosWWE{TA(!_eCz)pkc*x~V{ES6_hg{yo&sYR_$mLD^j75NlT;6&|w~HCzA(wZW zlguy!+@gCgagym~fQMY(=_-jst^g0YyosN&2uz~xiJ!3u+TMmwP*{kETwdpAEJ8fw z^144`k)#f}yjQ3%jPj7nJC`7)@Q}-UC7UpXhg{zIgirXfdlMn=!m+Rx;vtv!8g>K` z;)2k-h-@=7ob=*z5!)dia(S;M9S~uYp16lxsm{wpR>Qah*=f$pM5fc7mx+pWUM7+h z8GOfr1s0*~qAng@!(J>hdSX<{C6ZqrbNDwZe&$(}ua@#XWMpxYeE!OnuOVB#Y;3B+X+i-ayBRyk2T6e1O9k17cC;1;1k6!XxIyh(9hi71}n@?vq`08u>El`aCjeCA?c;UuX^q5m2VD@xcP4Pfzd zto#RG-SsA^TID?l%VcB^gI~+z#(`yq%j1EMDgapKx;*~MgW~lT#SxeHTAT+wdb+$1 zfET^4HJ?6MZ0P<+>AEq7TJqI8$bqG_z| z%em;3w)_O_230A39itDY=v=I?*Rylw$`7F5n{*i`>b$|o;hYX{KZo8jxb{!C1R(yU z%i<9cOF;v+yQ~bCg*qK~S-o5qDqzAaZ4Gru>w4gYA(yoftX*`|VVMgW z8zZ9EP2lppz!AdyY7RiY`vqS09K9P7&a68HIjT?PH-V@6lzNyxoRGMl5^SUVC%{u} zG+>{q5}yc`QG<8;1G#~dgB`|JGPsSC>L*|FbLj?Ky)9N(Ln z0|v+UB<8@u@%@XHGk7pm7y=oC|9}sMK{}Nc)K=y}0Qvv93!$^%bo@f-)}B}^ zJPt3vc_GwtFN9u0(IV$U=))BK0HT3@eM`w6squTrqO-=|4TgGxuz+Ekpw{u$V7N4Zdl5X+o$m}iNS0ip1`hYtfy#SKaQ4_qXgwum@K=Z9> zNu6=bg-uTyuBLII6v#1fI?$qMtd#iuOYAO{4RuLMti6uiJ?(%jowSI+%~Tq5nc~c|vdv1$#pfvl4C)H&JdB z^+H@9$VUc32em@Z({5#c!OC1hkxVWNMikVk<$rOvB?GOsssEhC0umv5c|I`3)-WHm4^ z|7?{vL*>;mkM&YFyX%oshE0)m4-4)#PRY)R4_{*t9ah8FP&e3fX#6gOZlWvi@R{kn zhn;sbq8!?O(#hhTJu_nqvmQj$^#K^S(q3blqFt|II-SdOmG9oBd>fO!JGxsTz|lPp z0ROvDt-7VYO(~q)l!P~an=-o>#`^c)rX=$5+mtp%)oscObVO6kz+=zFKZK?r{(QIW z{;Z1I42C*nm19PrW0*W-mFHlZa1L1wb;v5m^C4^=X5Ivq7ypuroT!rVFS$rQjD(4` zwEB{ZoUD@ZFS*DI99{QIF7iSr>3+#YUgRX*FS*Dmj;{M97dh2Qx?ggUKX8QIFS*Ei zC+U94MNV^)?w4HTB~H@)l8c!4PalV|jtu_3q^HVZ@R!5`qw%=)jCUCPsqp}R-^?@3 zVemJ^1N_BF&-D(2KiVDO?@oGdb{Ogxz_Y)F0k6rq1oWDtH5BrY=OgoxU{isT%WT+n5h1^V5tv}Sl zR6DB+cYIm#bapXPb|IubKyD?Za5^AlzYK1WWX9J>7^(XaQ-*(;ta^0>t&M~n-sLn} zi>Gr&P-~*HMIO4JkVCuBRu`J;LQDM1y^09b5`l3Myt2ioY~u1YPeGwF@=#*_tQq(* zwp!*~j$~JL&@qZ?4>E~`MI96WLZaKHFhqp9<2;s+#fZ5bY6Zly@IIV4$niZuE`k*P zqqrl;g?;cTl2QbFYVD-n5LF;4ium$YPF)RoXVtvGz=nY;Q>lX2`h>O#*=kKupGs(d z0L`p8Pr-W~I0}*nA&G-Kz?(`)?gT%TuslS9&`^?pMSKG)r*DeH3+ClsRq%ATV79^* zWUGQ_rm_{DgK}1!m(7;g363iGB}l4*hpK}2g0BibLV^WTl7B^fb1h@d5%gmu@sl2C z!XED-$=(KJanL7<_)=D0@nJ&fR0@4^l2CsL@eQk-w=W^YtFS#HkhI+?usGUa%)z%~6ZT@PAnQ1T5&LfCQ z*b8_k7FmxnEB#$&?LgG4oau}`iS%@&tu$_3lxnYPrqjKc0Ui1Gn9kr&wyCr@1t6QE z0;^B=d3qEAe4ci}vh-W)88zQMhKhO882?}SWtUe`@OA{7QA&K3>3pY_b(d+*zOKFm z&#R{)<2!(eL23BiBNW0S3w)?2zVbuS-Eq-;@RX<&0fHq@>7%47C=~+fLiW%J$hzan zLddTE{~u)RX)@J;Z0Em+EZ{=Mo)?GgvG0Y9{ZUppkgZEVmW~c=icCBQ++J4gLiP?J z8;)t+913J*n6dGTWXA3-cK56A>N~fx@FV1S`JG!?L{Z`8cWz}dY17N^+{zxLZ7;ub zD|?a-dEX$v7wNE<_lsl+=?pKg@5tVyv%UPztt=&-@4bt38R>|Z-?^3Lq>H`$&aLc2 zy2Q)x+{(VB%Y~)Bb1VCCx#Zo-jQ-3JUVi6R4j^rMPa}UIY1_-&)N&B%kaslsgGqTE|ekJK_FTZmuhmg*9oE}O#;y68wbg|>~X{1XWr%xwc?l^r0=?cf`Gf58= zmio@EtlmPW`JG!ioEdoOPA6(eo5IJhn0eBF4;jmt(i^^?1-_5pHuDsHthmMGvRV8` zkUoSEV3hcRl1)^6{8E}{V5*u4GpO+ixZ)E$flaaMM7Q{d+bs@7Frh<&-;Vs&B=moN zRB|N>!2)PC9I(}Xxm=OXT8P_K;jU=?empZ}Kjz98DXm2MDUm=8^9D=*k7(_5^5i~5 zLG^q;lj$Iy?Nv(HznwzoD4|?b%MltE7aC6@l=K0GCb2XCHh|_YkvBj(xrG^*P$;RB z8WQ{D zT>MVD3#duA;>Vjfh@W^88cgcR2K-L`Z$SM3wHqRxB>Za&v5&VZ@s)sB1mXb_@4ZEd zZwAB(Af6&|@Gq4(k((^3b0UamNIdnJ5-$}Z2XQTkbfCxn<}Z9SXbwUZ;DRkk7VA`&yO@jM82 zQ~U{FSFrfcp3%%-LS|{ivq%`Oli=b}Lb1>NqjLSDa{Z@peF>}&f$L6kJv<%lnaQ>V zsCP*Jig1rFMbAdUmi-mvI>lUPmLiw1U9D{E$R>*O1i0wIiG6V=xLn0yHBW;2EU91S zpDJ;wUkCMRP|uS3=x;&2n8_*M2R@C`r_ik3j>JUvfSf!WI=1qs_>~?6DAR-QY17WJ zY2Hk#?X$>L&s@tUQDt#Slb->a&-!BD;yJR?vB_91rT>>B;Z0+e@e(_FU+3AAv)G#e zWvWbTrkj`vr`_Ub8gliq>~H@Fxh`gM%4@)X2rN_l7zB?*;uz4M06jSlm|1LUiPpAB zX4mUd;!L(ML^QMgut>nypx zTbgg>aEo$%B;eWzt^$-2``$|B`jm1#9&l|3*F184zgM}QRjz*^m&}OW-~!o;J$H|C z{fBZDUIv1Qxr;TD>jyG6B_K!_uObId0sRK@+CMZ=O1o%J0xqNPlp%gi8C|vv@hj~o zz&kPKcKg!=%f}t#-YKBVTZ;9ptcbsVRm4nvW_A?&13Ij@VC7GtNN9 zpKW>v8!;No>NQMw1*geS5&M0>P@xP}!Z2np8J5ik9WPgCDa(X5_F>XiFilz9OrDcp z0@FBi|4zbcjCP^9e<$TOfES_mRiu3dK+)q!ynx)(n0xI9!886zB+r31LGi^jC*v;1 zk2jHBnOti5H;n%Uz@4B>P+ajjU~Up#Z&Kb@QQqV_U@qnZ{n#Z9$hi?9K5F5kr`UBT zfG870GLE%|xAoVNKh`vjHIZD;39(gb`VPvygs{}|im7D^pIpVZKMq#T_t?q3TGDn~ z68lKtCc7%-e0VAI<%xH8NQR1wXgutgPm<&Q zcZS5XpZwydNO}Z637+!gPvj#;1M{tcya}%mX4V8>4ah5|L)*~#Qb69s(Rmj3%#GMz zl>!^P1I-xwO$-_>5#NUT|8KfDIE2=i`k#O@>3-iVzh_NN%r3*LzF43T)X zxe+_g9C9NT8y}lpSUkoryrgh^Y(jn!%nd$l@(ZsXbIq8Q#ZWmMg*m1Ubm&c9Yz%Ou z#6TG_4{UtXDnBpg7rK;@#ez`Af^k84W5$ddGj_~EWExeB!jxll$U)kei*xdi4XH4v z01C8mg%_c5Y~@9hVsodmQAHy>!s91-bD`D~QnAHTO<(3tU0Ixvb{CIuN|@+!82Tj8 zgYHkp5mfR`pZW3$`ZF5Ic+OtT^nMX zE;fxROL?wD0#>*;(Nil}wsDS|k8PaGy0BEBN$3xn1gGTt(Y*MS4=+UvF0*=qt<&Y9 z%vjQV2Lxkz2riyli&9c9g7T|~#a0%x%E}}LvjQTlVykPX_7oa1L(?6#-E{Hzn5b>2 zsj9SSOrEREBXSipKh!0v-N;bS2o`8FMrkufGXeEWqvrYBi-hi z$%4JPsy^oQ5Cz2_YcVQ6ChX)`r|;w35(8!A*cXNcQ;9B$PMJVzWFBuyxATNza>k$- zL`))%frw|w6vnNgD@O`;@>`OlJC_pB7(a#H2E90smb&_=dP?K!N+P0EBOnXy5C$18 z{Q!h$M&@?6fHo0h5f0mEwUY2LP=DRY6X?XB*etXoKOEwb&JY)b4H*@RnPWOS9GEfM z#IdG{<8nv8(8&k^Toi)FgcOCtb)N1yw&4PT&dZf-RPLx+q-_@j_MsqG`0SxGQtTlt zm6`Kn9HER=0v5DTKH+j^q)?P=uEx_ott?(RW&t#Gf*Iyww>N?6aZXGOg+$AM=a|N* zPF8DnaVRvdX@6L7u8N#@FAy$~y55cqg$j9v0}CAur3GyW}KIx>eBi9F|18>jLsjw&PIl(Ky?7! zb<}BeFrv}T=@@68prd%F$DKCg-dew%ZFJ5c`6$h?hIye_IkVg$MAhz2A{0%*ur2?? z)@ayloEUB+Q0oX47-{*A$f!hOoMDwh+cV0j7H6)t-lAp9Z1*r$>xSRyS)yZ&j3!?6 zmgF4O5?^;?@QUbN*ogCX+}#(TL8cu}hW`W(bBW!vyPUlo%5`*OG z{hzkC)MD{K$87pX83Q-kZ2(*7R>^)utIaitdEur$y!jjv6*T@{q`8-k-UAF>&PIh3 z2+|7mANmot6!R=pDVbwNh|Y~8ZH_aoBGVjy7{HyTxUJPu;Kbqz=B#cLqDu#Ean0g1 z(JV!>dWAQElijrkK1SVqO>r9(s0dXbOIlDNhG3do0O^WAZ;5VwRjNAC4_?`b#9IrAW@hN6N|of>xX zYVrBn+mS{L+(!n}ELwb^hp~WH&ZhTNPblC-GIaF%&&|*Gn97IXv)BO{QDG5xm zTOVFK)84?hk7Vfip>M`&2Rt)IW9Ei9Gfn2HZe7axa@kv2aL%t#u0JnSr>LTh)M0g3 z&|IB!e0!nJjd69(@$XUR#^vf9PNE9?evzwlctzWI|KK(;ETNNcLTIqCtq;|SJ^;t+ zT902SO5XY_9fM=&7h5{C09rQ0$uU)J-n1ZVbQ8v#0PBO8o-?vDP_$#oXSyXsmV{ZZ z(S^3DjfWFZ1wMsA-{E^DNt70$FmhblwqPlp$^%1X>m$?A~G-q%utJCXbpP!sORmw3?n*)fXDrf? zeP=KxIb#+`Au@EFDTS_`fKN@$Ns*chY%-11bGDZlA7cTqwNwGIxJmBVM<%p*Qw_7T zzGCAU%9xWC4ivY0ap)v+_+>yFYY{{=czPbd)4_*oVJaao0*dvao8&DLXQE zt#s4|H+Jq?`PS~D{+HOCP$|G8j*SoZ1oFLP~ zP7{nS{A#3dpX8XRgsOVdsD93KzDRwg?hFhB!QPcQRinwXbJp%T&a4CR4{R&loA3V( ztKY>-Tci0?)&hRgJP^$0LdmeXN5i6iuu%8SeZ1=t%k_4?bp`sbGDUcS!hB2THRF9FbFiG(j8t&p{W34^Rr4)k zxs+`j)XN7ci6IZe13s3Qn*?~0%RvNR#Mtyi4yje0X+B+~|AtQmsKVI&g| zafy2i0!GEI z8J$4L75V!EhGt~|uhz`$cOtk0LbhY)!8uu3)qMy+F?tXox8_p;L$fkOvY!M;+`BT6 zG1*R^2j^rh!26P85n|p(Mi)u4iBS=P?`01SGSMM;zJVKDcn6m4Wi%Qgx4mP5!B<>l zU_=eRAOqh|&1GN~yrjnx7@3c<+zCScuSbY`m4l2|A)ptG)=096(OLxQ1rx^wAN1k? zz{3bE0g+t%M56>D=B;OxLSVbr=P-~d=51h9hTwbshjQpdqTP&m^r)j^l>iqQDx+FM zWmJ#gdzr8xvR;Y6)-hg)FcJsoB{E*gAOYq;GqvT6DRxxMC?UzsjOIwPi4lbs^ETsG zF0~=ZLsLQ-L4=GbB#|3px$5yt9?N>IBsVf5Ys}lkhy~{k;}&3c2tyO28zgxhqaFmQ zfr)*B-^l0|#WO+s6!WfQbi3ja%f+I1B0vU=Sf=mYcxaIM4l96IuI>0`1JHa%EDYu1 zH@IlHL1n~u+D$i@N*My9u7H4qTnYTL*ARff_@SyA)J8_kgL3g3T!lyh#b|;gH!+$h z$sP5GjC40MienC=@X?5#j7yTybCTT5=;sK&x9`y4Z;;@zc1}}}-xj-mCX(|J9F2J$ zWkS7x+RUgx3u0oj!Jojq|EbLVt&c<9o)VSYT#HG+0wLyYMG11T@pe69Ep-U!A)|VP z0*orow0LvcAiI<&U20;)ZiPfb{LUCT8Er@K@mc*~JsPuA8q~z-S_Hu}vBKaPH7TBn z4F=DMe}+r=nRu7M??TjWl8pFyoUkx)*x(u6sdy&dZ}5!nRy-5O4W7|S#WTUbhKQ~( z`mExac+}t-nQ`3uJdzwo2$az|Nj5R!-@cTFF@Y2dWHbsv@H~5n78-a){Bln4OiVL) zM)MWV1do6QKciKOXQI*I88s=Mi5&*ds7LWk^cg&(1Bz$jE`w+E5yexBUNU$_{Mt;~ z$pl~8M|&6*Lc>hIkSG#$YhpB0@k}`Ob*|!R5qV+?L`W_Dq2gafg8C|jwj+@h#cg85 zBUga09Ax5%;5RcmD#<2BA4HI>_abrH;2Hf;@k~5#@Qi+{__Ijx1Y*gIp_5BTpEjMt z|1?6(+s@(tjY;yz-Oq|RHiSLeegg{YLts9}rh&9+a*f6>jS0yqOqzNrYa17B4pYqA z#?K%qS|quov1lYIFEgdLJ&Pno1#BKFfQO1n_-yDrVUmKN43K;j(N|28-DOXKjia*# zVt%&qW$$SwNf8*Mw9T{??XWHMy$_#0MD=1D<6iwBRe_C+dsh!?D}_nhDF7S%wB_O` zv#CPB8)?#5W^YHDLSwPoa$#Z@*mg2$qQb<4Sw}WAqBsOkMQm4qi5n2m8%DP){yrq`H+V)9P47=0;((i}jpi}I zQO9D&h~khMsGY9}zKPK}#WV5mg5S)DIx74SkX-fnHEZt%%7ouwVavy!n-yfkE+l`A z04%#ogN$S$qN`1k7IlM3Qp0Hp!NBMQ0xCOwhzWiP1w6%a&LlYpXxD%ZZNLW3Lv|8a zYW881Jc@{_0_;~1Jz$f4!!`&;TjaG&1KS>>*gck z(D4pOG#Wy4c2RDwa6hnK?{BfQbmT=V8bMQEFz%nwUww}>w1gVP&4kK6? z)k$(aBaSGrj1C(tBaR%fj3Rdq!0OH&Nsgsa@AZ!%$^BFT+6vvxFByCa5KRFjqZR>K zXb(oq;H_swg$I_A(S-G()^DWNpAm-jjLh(F86!59vjz-|o{?m8 zIC+}G$JeT)K_lq?u@q2euw zy;$fG1oT5u^HE2o)U%aQwW*TPLIgA=Hpm1mfK$+;P^&MBhHYkaMv~Vv`X++!y*oC@ z#M1_^b?ruz%`BBgs7Ly=l~ESK_YTGenYhW|HP>l?&Jy*Yp)z_HL2@y{c@JgNT&o}i zI#7UOv>8EInb>ad$~qm11OlYOXbu9r1vd{eAzv?t(&8N~)d{AA5j~9~y&Df7KK%T< z2MMq!BF?-D1k!SaWh2)RA%0p=xP*oaoi#d1#xs)-Q`w%HA1;O9FH%BKyZ=4AzA+=xRxBVno-I zshX)941rO%B%2uVP6|~qx(gw0ZK{>f5C6Qg+u zxxH0}I3_VUW+03Z)4{1C7c-?0AZtc+TA>C;HzDK}R|!m37y{vuajl@57#V8NK;E}q zXQ*1=VSslS6r&Q5;6K8l5Ar1B7&?NOqimB zmNuZkYo*X8M%;Zkv+xu^-XKE^jGoM;J`eB(LuK^4TxtRK7-rkt#ON{vr{om?xkEw8 zjOcfAs5E4w5wMf+>Kn?Eg9Ny}aIE7@fF%f{z%%~MH{aYpZIB?%n!`oQxkFIR?TEMn zJCmPQ=Dy)kd0zdZvcM`n45mkO85QLkamD3kK7)*Ajc#J}973)JJps%&5uBD0W%D2< zm!k5#(Qk&&N?!u)=Lk;Ta{ymM7zv{{=FK;z4H5u@%R_LAu`i6e5KNnxqT#~8FuD-| zPkH%wcbMc(IJfQp4(w|PPW^O5K8z65Pv8**hvzP+;$I-38b)UjOu0-wVF--4iO#K% zIUhox+!;THfKnJSFSLl!*AU3Tm}NN1bqb*23})m6bf6@+6ZvQ_%_4NE5Rvf%1e=G) zf%=+IgZqf*0sb@>MX^sna8-oxWCTYR34mpS+RVs2NZF7=(kSz0Mk|D21EW<4aV*k| z8wI_Y(N0NjV6+P%j$Kzf;w}VK#;8Zo{rEN4orq2%xa}kSAcE7r2LN&}2T?G3#8AJ6 zh$;yzBcrU%&m&nRlH7olT+;zo3#y4x6GHBY64P!N7#$wMK#bW4H8DCdgy9%4yzP{Y zHg=v`H95KDvV*$x(!X>HxcdZOJ;<1=ZQQFFWc*1!foR62c}-!1nzSre zv)salqn}1eM)R5&QK@n}@enXnz7e>^tC=>)Y$@UhJa2A>5`9mjyZx=ap{ z&MyHh*E9J29)crtqPWj=U$F>r;hQKgf#)2;u;so2!mkmWa*6s4g8R})hzrXFi_?^ZiJiRDcO? zYMoK%QuI@)X9K4u^Tlzapck8A#_75+bMUl#BB*r;aQU7NhJuI&gPb2AiCAlp4+Tgf zHW=h10g?#TiZUMyklKuq-hM8K(+0IM0RlPX+KJy?2oNlzE=dNqfA>(-KA=t;E-lpA zU)}`DaiKP!f@mHP;RVY8DIZ0E1Lfc#6J|=ulN_2LCq|GSO2>QnfM?A zCM%=670=pet$#l-DR zj!;9SQ}IG@u`rQNSX4M@b3mMrAadx3qzI%8;;c^u0TvT*XfXjon?+9U2GFcyqU z6jnx#Z3tZU8E;3B%4iD+><|PhBZ5Ge%Yn39&*Jwb1iLHX8J6L0Ly4@9Og~3peT->; zg8Cd|Wa^_aqRj|6ke2Hle)FIj!`7!n)<>p%1lGrxW>QL{>30udz8lWH9wVFh%Jn(Gx18UWWc5d=Y8te-(7j0KE)-(KMwOjUBH2{KJ;43t8)B=r@-teF`LXOo@mmBE-G#mML9HCg1O8EB=1uyWbgKlP{K1 zy5=+ZTPvb?Q{I0Dc)sYn53E9DZo8wNq??<~IKzI;=9yh({a^8Y)145tEpA$$wjKIg)k30C2$oM=0+gaq4 zzY*|e1Sq;JnmZ8pAm~NWz=+FVx2#npV%lIBRq^3YN4x-GAp&{o5cHae#1w=QggFSL zH6ZB4yfYClLzs_1+O-Ic2$<}_MVp=Pj1n&6ta&g??>&0>S9)il9{XVBeo8}n3hVVK9 z$Aud#z1Aa57cfWbKXHgl3>N@|Sf;-%r|~o}vnPc=G-(nY z$nW*_w{^F)_4cPz2JCLhWLvZ8miE?ct3hS5?d|E^Hhr+C&7^S08RroBvts@)yqN!e zFIC}x{?ID_BZpQM`#<+m#jDpB-|lB({zVD@#sq$EF5Z5!^aj5r<|kMA;}VNzmCUM3 z&h+tgttwGoa*e+(;a{5auS@usCH%sK-=CNTu97(w6;)ejd2xIpJdmhhoWCl6|4M&s z!OQ;Am|vIjmnQtLLdQ=4mquP7BAI(3W%S0siglmV0i&X@Z&)I6WjY& zhZ1P~mg0f^#RKt?5FrOYE`Egz3$>!7w>w?YT3LP7g32qZyL#JNyDGB1y}S*tskn(9s9d`+tDvj*ahAw z_>*GEiY5L{i3;?juDm{3Rad^Yj6Epf&m#Uv|4Ny<{EEHk(=vZm?IOP{waA~m8-_6! zt7qfA%lUi3nP;0O?Uw11hXy2LQpOf3qwaiVVCZuvZYJB`*<LbhKTMP?D!;xe zL7Bz!v-uU}`{O12jaqwsO-D3PUQlBDd+# z1`cf$w}3D#F@Idj&m{b@DLPUmdf`>W&Iz<;;k|Nfm>Gj=R7baW>r)! z@)JjdrB;5$ye+Eq@4qGW(9AvmR_A{|wn_+F@)tF(*;+Ect~xoZN-(MXMQhfqsVJ|U zStBs{;;Zugdv3`ty2k%yEag9TOUnNbFR|D^3aj@9YP|DM>e4y$Dv~9OR?kZMU9}DV z0gPp$(jQ2nHU7TT{8E2vVyoYl@=H_YmHzf4eo10=dF9G7|5@)yo&U(8)aqGveqqXA zmh$VM0*m~wpy+RTsk%k}qlb?9Kk^dQt84s&i86@4;^Z3tcVptwK{% zh!c?Z-LYE#K*B#z>o2dJRe4SFYJW$mb}`(2k<|m$j=_CWkr*- z{&QY!a+d$%Evsf#_&22dyJE01L!Svh_`(o{qS;^}72UH z6%Vi5yL$(Euyl9!_Q05DTl=#EeP(KN^3c`bKN*YpEvc!G&o7ye(f7w4i*xTFlU6=0 z9CpgM#ikFmrTbVzOGj%@dsn*O8V}Y$XN6xFTVO7+bnVjM6>p)zIm;XVAhv0E8&=o7 z?Y-StSVNmfr-wfwmV8%L@eRdWr5@9R^IB!M2V6S>t4#k0EXw4g{Uz%rADvlwkNixk zs9cK>&x35{%ckJgx7&mCfh-p2Y=1;d!NS$o9a6-|m<1WO%dWyhS2T*4{u+O3tYp!W zigzXJDwnNUQeE!vtM$j!miw7nzYx*E+J^FV$-44&=U>@sy2>wGR9}t~4CfygFIc=3 zm9sj3pw=H-TQ1AMU&@GgoljoqcP8Dxud^+k$z9{R+I!8gxH}dI0r-}hnkx>N;U~19 z=0FIAT0vKpe+m9%MqaFL(Ha{~-deGpp}1q~3Q_eYY2K>0-8|V&n1+b*xZ8`XL`6RS zw#)46-JkBa$}NR<_L}Yly5Mq%mPCZ(&>bPU*icwhAG+oId9mU}NnG2DZ`fLqoSB5l zO__L9L8=%V^NNaD<;mpCie$3Xzk0Vn5c9_*u>Z!sJee1VbxL46{v)p-3*gfhSQVuI z&C9Owdtz7t{k;i)LSkOQ;Pu5X6z}mT?DqHJkNzf0HqWXk@h{pUvy1oTae238Sj(fg zE;J6V=6LOWU0skB1kaR?3)DqC0desOyuA{se^OweaBXNAJQetfdtZI~gX>Ii8Qfq{ zhuAbnmz<$9s$e^6CW1jlbbz)}bR)1W&ZgMCeWL=`*n%?sL~ZHi{@8n?9Dctt%ml;3 zhAfpYUffuR{hhyWi+{<3rJd#ViP_W3D(kkP;Kr?s%hxT4C9r~Vny=>1t<@!Ut1IgU znD#qUY^Xn-WV5a3KEv$t=P%f^XfZbS_2r-N$D}5`L-zXlwfRkTi|XpIP@}))Qpld# zM()r`4IWs3^eO`^E-i<^n64oAvLt8*M()bR#t#027==gQzTO14BnWS#1?4sw{M>`(KU6rgF3w)tShD={>c@vyv4hmF3%uOJp>6*p%f;uAQui1^dbN_LCpoWW}ea!1W<<-`0D~FoB(;xTb6zKJw(ix={i`=Wu zzxPm9|D=Yo);6>jr9NJP@2ll%1?MddUtE*b$*QQW@<*9Oi zf6AY6Z`sz>i?+_btezX@F^TIZ`D2bu!ZW^D>`z-!#6AyMQJO`0_PlN7;h0Kc5gQTc zhoeT((dMHNzalmbhG155E{#J;vINnr62=D;Ny16o4q~(m8cWTcNNxVkvgygvC@mb{He1}YZUlb64^E5imJeC5eS!Yn`j z_{84gmB~G6%3}7S>x)laE%K{7FzJpplkY3uyA}B_rC2NcQ3VMM;q~)y*ODw*Tv;LR zn*XA8*lIJ^O}-}(_g0@z*5_T>jjCdxxZa1($TnGw=@O26r_NWXyG6*kpKz_9jHB+f zBBSW=Ir}LW8;+?ve;f{DQ@PgiS)V^GCc7;Erd1V5KT)u1;;iIs-0Z;scGSZDm1bc~ z)+9+M%|HUR!69+>mRkQ~u`T6D5RvfjKD7H%ed!583{N4Ygw!-jwnVpRc_0YN=VeWE>*1 zRw&MVHSq8=)mZ>JhJ4=bKMAA!j+$%y-b4osY~iY}UFKL#{C6I4X7Iv-RsOb^ zKR4lTP55(D_-#a%OLz0Z1}rmNKP4&r5?OYn*x!}%XVzAnD7k(O{wMig?t1>fMEc_t z-Oem3`m!|*Eh`&ZmNji!+0eMWWpg$B{_XNm$Lniv>Fyh7>2K|B=}%{kh2_J57IO5b zTiaVY+ckefb2anY(wa9eS=+F5D>1DL!J0vK>jRSF)gR<#I&VtTm+ZZB?WV-Cs;UH@ zf~2Yvd=64sRaKKH?{4cGScIoe+4O<(Zo=7)f$m+1cX#!+X3OzRWKl)Li+_$@L zUrl*?)swmCmNidj`6?%KLiZdr9|~ zw{@i3_GS#79^JNVdGEe-KkC_UkbyfZlhe&5jqBDfmtOI&R9l&KRXZZ&;ZOb|0uI$8Ox~u4&ob+26gtwLjg0#%1Wf6X!U@+m#J! zuX1YNg{Mu@Jg$%}`&zpO(!n5hW{AKj>ZrDNX8KyQ5Jf+VP4~5Ay7!-QUvL zmdUhqr(KoU*KQOAMS1O+N?~v52&A-m`G)3|>(+XF2GxQNn3cM_wXLJGCv1weZlDiU zrKvNvbAi@mdhw`=ilQy;NvGR0Ef}Xgd(c>`cN;b?Yq@qw^F}nI+O*nKrn-mwUyF7g zd{%H9(dHBpB$qcY->AUKwVRhTu51W8=yk%QX$phQw}R^3y&J0TQQ`adrGv>~r-K&N z&|JxxVFlXT*4^d|*Y4J?Oxk0S`&;*>TWpDzS9)VKsLJc?ZRyPPb{QWodxnQtwnc=p zo!#l)0U7Z?XuDf8*+6F9y?Fd*O5B&tw(RLcfqk7lUZw-nEiurO>D<$kZcku#;Q17k zre!HL$+jd{185MSMeUiYVDO3B)787HwaW+z&N$q0I3fs>RbjGPn!3Ng6I#HLrj>Cm zM#x!35ki|}R)LRl@5q?C*qvj4rJx+vGWT}=xN_|&`OIB zRa?yCCOaS;2l}?X9(Y7sFj4v7wxzTGMoyQW-fZXYgI;SlEW14H^#;JduqEqd+WK44 zJ;E!Kxwj{Mpc5);9{qYV(HKd7z&690G8KPP0Q7hyHKG^jZ+5EN8i|(zsZp}I@gED3&y>8>ROInt%S$VCc zty|V~U6V~WF1YqOP1dxmT-&^Hg9V)o%NkcVtlV5vz1ouA*|c(9r4d|PSGpCp6mm0l zHP+a~nx;)QwX%9yrA@9|wr1IeC2N#-!HVV}+1Na!JDlW}zI<(irqGWTw3$|+-}IPw zJ$(GN&}&#r;p(9HHAi+#GRB+Q3?YWx9aC(ZmqwInC&+}+i>C*!qX4QuJ@ zM5oP4*p=Rw?g~oPN5_KAc>C;}GOV5L2W%#G$7F12Y3=j&W`wWD8|cHL=b81Rg%*)b zfzd&;v%P&v=;;+w54sdo96T!qbDP(`zq38t;h`_cls-`1nrZ9oG;k|^V@_B5oi4?CaUemf0>j7)8N zJsP|Itf^bjvb=NPobrRsjy8bL?)p$iW}jlbT?3ijtpi=*)((T*-P+T-2NuFS@PmB% z+jDtD@ZI4vK`*lp6Wp>7WRN+Rzo)x1Aag>kX=$u!ZrRwp%;O^0pSEj6K=Mq=Ok|sG zT-{Q=V9kncmL^lwX5c)wTusfIY+Tl`%wiik*$e|GlSy5-dQ+24HZ(T}4Kd4>Vc|SB zsg=vXVv^XNhNEw9TCJ8(Z*GURS%f}Lm$dop)3qar6jrQ zwS=nzf<=U-TST-E3!;5k5cYu&m&^h+&?k>J_qSpLy&F>+oBq~r-m|ul2aPb#N1fZ@ z3O=H0^BNo0*(EJ3ti5-Cj}#}%uHo3QTv(~JvW0L8W=5=;jDq@cH>iB+Oa`_Kw^=Q+ zyk-=i3qiMFUw3Qe@zW7Y@OVFpn7a$Gaa3~vtF0n>I5L~ zP$VxfSZI3S<;-N;umrVW+hg{^(R!V#ATis0Yzw7YmgiQ7+Y;=vT0q7sh#j)LKjD;M z_FhBonzb!``}@(b?$*p+R3rl!R747Bef{ZuqEe>W+`qE4U7gvCXKpHkzS@ojIrd-) zv+R~c)x@d-6YbfX_#iVF**7WIW+d!nw2}yF2dgy9?XB5|xfrO5+|cCQA87Z7lb+l% zV@1N7q;|ltz<0J8p<78&DZ=Jj$psBVS0q15LJx+mc<4=QP+1?k+_A@MlNNevN`=W2y=Titmx_b$eBVi2e3Wi zO%&vUY3B9q>FsUjy(%R#kfByWZ?h8j_4Y(44NZ-WuxhYbHpSa4+zAW2JWN3qEt&l- zGVfWw@HI3*uHuPe@k$ncBCy=b{d35_vbo9RL3+=3@X8B+td7HB{w-6isnVtAv%+qnZR0*X13l4dkJj287bDwrn~!? z5iHy@DNA&{vYRPWt<7m1CpI&E8k!l!Nt2^O%hFOQzlK}VJkTqI`qeWd%kk%vN zsb>O42nDua5_ERKN!8oW7U{5R-}pk#u7iE88Brk-?u7*INQE8*td*YK3=gf1Me~h7 zm@m9^PoMRW>4EKsZ3ypf;GSxM3(ShoXee=FH0Xbz1It4DvVAHXdBC8KkiZ`&SQSO+ zrfY|dG_N6k7)jA~h@z6?_S}7}Ze^g8*cc{l zqqT0llZ8)4XvQn943=HZnY-n)+%C%2Wix5b@;JicqOEmQUN-WgV3D@kc5xzLXf9BK zJ`27Oi(OfxEbbQ13bq%IDd2R=U{?UgK9=kj<0~+0roF4%waXJe-jl=S)dT-BR6$UE z$jS8)J;!DU&yq6f8wb)oa3klG0B6fV4^|`H-XcQ{c?!dBe9$_M{miMcr@C|Pp-V! zd?ZIKGmjwG)Oh)$N3IzaADuTLZ(?jx>|%dPY-(&;e0pqVyaewI$X`Cf34?FOy!g$9 z_=D0>DKM+z^5J_2EQ^&aVa8vN8l-#{oZqczI#b(Y)LI z+v9h{_KyD6H^1>jp|=F@+DqN!+k%I_%(tvC?`UiVsJOI^M?rJjcoMK%=R*&Xqkzn6HnCz_w~;D=eGO@85HTZummJkvJ+ zif6uFC@^XKY8me)D(#)5v0l=6B;Xc9llHJ);tMb^XzGx5kp492V;wqSo|CEkgeOMe zDGu!Ri{(wH%s`)D>mytVSmo~WQ`U{3^K+|Oq?Z&t?%zSVQ3j+R1|4tc zfnN09yrceYq-oxhprwY%8?>*@ojNgP*yeY0@UbrvUT&lV}?W^MFZyw4(fk&qd(ZBJk@G zm<7A}^CECT1Rm?aZu_Yl8!;~YxP>;uoj1fMfX9Ccgy%c3TMl`s5B#!ap5-UJGXmck zfgb{_wu<~`9M~%> zI5SslS<5+}6PC6Uw1%*}4N~6HF)Q5sr-#X}Wj}TS+S3=Hy>tQExeL$=pq-reZeO%5 z(=R})ya27?0<&4X0`~!4Xz8vllmAJS$IoIkKjAZgw}3|dCOicb zgZjO+a4aqF?XmT;1_$&Npl^5Sow@YWpwk95X#RH@y4L?o&{>zl1^7zCZBGI)>nGi9 zKY8vfz>%t;vwp~yb8Zi(T$!ak2-@pH3)YQz+i2K#5kKsT<+~qBdosee%&-MK4LH~6 zM1-f_@@QLL1kcf29_b6)xf6p(KC1(=rY8WCb^x>ux%twzXz4e3o(2zXt@iC%z^6?f zvp@P^EdI{X&xmlB!A?_N%$J>H@=?Z87vMgm8;rbmSsN9!^DOAxELwS9M|o=h64-d8 zLfS@`#y?HLakBYxXoo?=Fl*l%bU(xP90TpjFrU;H9c#i_Y?Sy>2HT?WOBN=M^w$8h zpG(cYOZG-en|Lv7EojS4zc;b%TFy?;AkF~JM&LUm@JYa|NAo`hnEm6}NIL=Z^bKdf z62Bv&cjWVM))n<`X}GV6tKMn5Pl9I!c&y&+pzK&T^S=xl$I{aFie15;Li29Krs-;- z2l^kh`50(!4Ad41^Qt^j%0g^}Wlz9{`K+JqW%^lN9(xzzM*ViIfX?0g_f`O{$Be zRpK*+C{4?42Mt-tCO?NAI4Yk6)<}Nxa3@$XwtD|^$dwOy%CO-(E<6*RU9V{?>5qZV z_AKYU%?2neF4j-lFG16O5}uAX?%7ABF91BzfmZmfckQCj+EU=6)g;~! z#Khe-Y2d@l_pm5C zmoM58B76@M!MS`nH!MW>9yVsq<-=NQkBQFn{`F1Pzytr zm!Y4H&`%q&fvw1cNe(WpkA5I?#k?dEQvtJFt#7_VXVLT%nJXr?tqL$lLiy_*I{E3p zFqdf`Pq$N;bo!ALX1zNdm~{G;%w_W5<-nxV&*aj}xXF?$CL>1wlEN%M>%h!UACtLE z`6nEhbo!iJI!~ulm~{G}6lVEn9GG7hOpd7Vl^tmVF!&>GbDwpzEdmq2J70 zrv8sQFy(h`V3rO2&cN&%eK_X8Y|jB^lFQI*2 zV7;{c^r@N4wC_2G|EEeqto-zOnal9EQ&@5p<(!hK{Pb;^%kUp{U{+1vmP>!ofk~&Y z%cVc=z@(pXVD|Uv2>hVazE^BXjI@39jhV}|@2mr}eEP^-`l(>qGvh;lmP;QSjDL|i z_0v+dKKe&o{&^0}`sgol>9r0_I{haueWL@DPJfC^?{Z+$>0fc_CnE54XFlY?))1x$ zb$%ZRwiQL*xWE<}c~bu89Q|`oh1CAje`78q@5ddO?V~@(rN8FDq|?9S(i$#=Zz0x(&_IpmudfW2PU2VAD7-3ffJ5==RtmiDL2ZOAH6Pg zRT)90i7e*ylV zMflG-bn??Dr2K4e5qvWWlTN>o!lXAkFzNISnalL2&w)v&ugIkzbzsuzJ96nK9GGGUy~%k+N*{5}ek zP9KxP%zw&(Nv9vlTqgfR5tx5n&0KljV;E1uu6;Zj+(u!&jm#(3cb7BYei)&%e)_4* zW$HiYz@*bh<5l6Hsmt$r{79pw)dz5Ge7-R<}&Sj-hoM{ z|H`Gm>cCRJ1GD@x_??uWbo#UuCVh(olTN>uxlH{B9GGKklUovq`$X zh5MT#@BbcVeCQ{0%TK_sr7-KKk4#~<{}Bf!ojx*inf9M`VAAO)bLmsy$5NPd`ppz( z`IQb#I{j$oGUYcoFzNKGx%9&hOgjB+F8%2U{F<*x@!5vaXC?$F79!hu=dLk>*(BMwaZ(+*7fvkpx9^A1esIdCx&!g!6!)J^demk2&x!0Uvi@j>l`xc){foT&hpU zo$)FLeK`uR1=j@Ba0y_prz~eY;41>+)aEN>EE>N7`x z@NB^NOXuPr8;@3}0IR>a$Q$9+5e7GXPNc6Uztg^LfO)^5?HvF-waoZeSpS~^{ZV8Ap8p8so0-%Sr|>m zJxkc1ae&?OX9Avz##UPX6yRS$SPEYSxE19$SUUguM>q6M>)Q(Ww+Zv?h4r@q9)
{-gk(!2CiNA)h?&8o&KLor9gMYQH{}twkK3!|!{{%dM@!%~K^<%`S-1;X0{ss!u`Ys3jlSHVWS4H?& z08V4SuKb$;S6+s~aj|_n0pIG-`v7ls^!rx8-^P65yAf>93Bcvfcsv5wmCrXEnEc-d z{8jj`wxB-3F9Sa1=+CQwzwFc(L&v*jLtjuI^?d^1#~k~42?c{o>GJ?D$Nc1bn=HQ) za1x!p3p|AD0K4s53%KmkaQ)f=_%WwFT@n8G0G^8WPqgLzS%iKs;ICqQRUbbOcq{l- z-=6@SK!0kHpZ)nB;P-)lo-OZZfIpuI$LE)T-S)l_feWFFr;r(HBIT6p0fbIi1Dkm{1t%R^6CMP zLeTlP9_euX|JOIS1O7bbUpvs$k3PUj1hNsn1&Lq49$jhSy8++o=;tQ@pPm!8_e+4+ z!yd6c%>RAB%N_og0ndOubiMs&z>hon{2Ra@!}X>gsV@b1Jm{*Aive%KT+;De2l#U+U*}^QaJ!@by?|YL9tQjmXkWtC|6#x% zbL{O&z)R40E&mIEPlApnU_A4@Zvnm(nN{Ck0337L|0}?5eQyB%4Cb%OvjBZNi2V33 z@|gtKOHljS$9yjd_(7-rmjV7Xg39|Eq(7Dj=hr&{yY=yXR2S|fjP-m4%IA3A2$=q1 z)=2%NF1!Qn*ZzDM@TVR9`~=`Pu)opqd>HVrFc@lo@hX^izau}qM(VwR!R)g2{TMLs zAGN-J0lX7%y{-QZzz-rnR7&JK8g&-{=2e87hnau}puegg)quH2R(J*AgBYw0;Aj7K z0RA|LI)43t%MlcQFVbg%^7FjA0RPs`U*^9T@QnxyZF!$UI&4p#_b}kU#+d4Od>!!5 zP@l@^B5%l)v5nt!0WMJ-EQl@6!3Qw;e5Ul@W)|) z?zHr40k3h|a~thGAz6Xa5?l9|D}D~0K5I&8-Z^I9PWR8?{2^|pmREYpCCW_qvOYWzX7Lz-z43U z=MModggo$H8*7hmAwl%z5_V2~f>4L#ToDESmtzr^*1o~P#g0W16Wo%ARwybWdk*pWRu z$~hZKUZg^m_S`4)UX?lF$-b9`v!8-9nj%Ls4SV{LJ*r2J(BWYs_yV?HTg~r>@ui#j z4{eeZma1|B_TBIAXJSm5hE977yKSITvE)Jsg z)Oijh82&In)4k|d__jFgMKz=JVT za%2L=+@4u%YwTJP=mtrW<=uVE%nT>N4x0g*ye_|CimD$cS$p+ zlQ;)FTPZkt8a=RxMpe$)%{h!RID`{c6k314G~7<1PUlhbRt9> zCP&|z;%Y(#wks-h!`AH_6{tGKQ6F+zv29p5wQ^5b-80;gi{`vl&L*DzV%svLYs2kk z!d0Dz*M!2waU0h12YX)Iuuz0zyxp@wMfD-Tw!L~f4-cUV&TOGZV0&N>(@;C(9w20@ zaL$3WN)b-Kz|9 zFsrFO?2dCJk}1F*U8>qfbZGlc+kH!{j-DG@6;5S6jm&CPu(ymJO6^)N z+2#gnWDi#g_GF;Sfq!QIAcyHo;pXrpt2NQ7Y!-cKI!+dL&rlb|MVanrLaJgMcU?!! zD$AwKJxkfK5zdKMsvaB|J&8QW=;6pba||%QM;dzaE_ua-+l7u?b`V1q(n@kpinn4O zGHFcis&Jr~+nj9U9@~rVL{4EdBD2Q>Ih$BHAj!5V)GS*~h#2-(ms42oY(g^?{1pI@ zxP+S{id6QtS2^bvg$ro(z&1TF9Yu+uP$TXPkkyjFz8PJz_eF43@E~i=?jFNu@2AWG zc)|UY8aFwQ$B3XhTn(&D!U<#tHKKEdZ9zCf*5KeAFPj>=ueNmsI;zU=-ilkCT_ug( z(H(`V5a8f%NpPzyy2mpjWkX;)5bj;zb9$>6Um=8>3La>^@a*X_O&fn8p(FzN;Bv=R zfDng&O#u0UD^ISu_*Fjo-=2uY#Ybq#;$LL?Dg>6o^5|zRz+cgR3~9ZzJo?v^Po94i z!RhMMtI2C!K7L0x9(?e@>ZN>ZfzUK*tao<5a)q>Q5x#PKd^NLdC|^f}Z#6!``e!;# z?WJ{P0Bf40EAjC2StCs0RN9dUU;ek_-eEi!)Jyq}Mfh%i2JM6YRxjl{Y56KG;q=+K z_X|2b?L|I@PuW=c{%nNrm0KQ0%J&e`Xm+2Hz6d^!rCwU^86Y%G+Bxur-v@+x1-Gfy zDRRLlI?4CGW0LZ#WiZBw2G`-qzSoEG;49^PE<}APLcn9#obq11+V>X0x2l(x_dfwB C6M#qn literal 0 HcmV?d00001 diff --git a/firmware/am43x-evm-scale-data.bin b/firmware/am43x-evm-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2d71341089816be7989e6dc11b265d9194ac909e GIT binary patch literal 41 hcmd-HXAn+dU{VptW>OLB0@CSBDpG9>aG{wnApnMi2I2q! literal 0 HcmV?d00001 From a6c392bc8874d12fb7d5e871a2b5bdb6aa598712 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:01:33 -0600 Subject: [PATCH 005/135] Add BeagleBoard.org DTBS: v5.4.x-ti-overlays https://github.com/beagleboard/BeagleBoard-DeviceTrees/tree/v5.4.x-ti-overlays https://github.com/beagleboard/BeagleBoard-DeviceTrees/commit/e167dbfc27fbf2c339516e6f6ff3f764c693a106 Signed-off-by: Robert Nelson --- arch/arm/boot/dts/Makefile | 6 + arch/arm/boot/dts/am335x-abbbi.dts | 164 + .../dts/am335x-bone-common-no-capemgr.dtsi | 381 +++ .../arm/boot/dts/am335x-bone-common-univ.dtsi | 2961 +++++++++++++++++ arch/arm/boot/dts/am335x-bone-common.dtsi | 131 +- arch/arm/boot/dts/am335x-bone-jtag.dtsi | 15 + arch/arm/boot/dts/am335x-bone-uboot-univ.dts | 29 + arch/arm/boot/dts/am335x-bone.dts | 6 + .../arm/boot/dts/am335x-boneblack-common.dtsi | 12 +- .../boot/dts/am335x-boneblack-uboot-univ.dts | 40 + arch/arm/boot/dts/am335x-boneblack-uboot.dts | 40 + .../boot/dts/am335x-boneblack-wireless.dts | 6 + arch/arm/boot/dts/am335x-boneblack.dts | 6 + arch/arm/boot/dts/am335x-boneblue.dts | 2 + .../arm/boot/dts/am335x-bonegreen-gateway.dts | 211 ++ ...am335x-bonegreen-wireless-common-univ.dtsi | 2823 ++++++++++++++++ .../am335x-bonegreen-wireless-uboot-univ.dts | 55 + .../boot/dts/am335x-bonegreen-wireless.dts | 6 + arch/arm/boot/dts/am335x-bonegreen.dts | 6 + arch/arm/boot/dts/am335x-osd3358-sm-red.dts | 25 +- arch/arm/boot/dts/am335x-pocketbeagle.dts | 2045 +++++++++++- arch/arm/boot/dts/am335x-sancloud-bbe.dts | 5 + arch/arm/boot/dts/am33xx-l4.dtsi | 162 + arch/arm/boot/dts/am33xx.dtsi | 4 +- arch/arm/boot/dts/am5729-beagleboneai.dts | 363 +- arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts | 5 + arch/arm/boot/dts/am57xx-beagle-x15-revc.dts | 5 + arch/arm/boot/dts/am57xx-beagle-x15.dts | 5 + arch/arm/boot/dts/overlays/BBORG_FAN-A000.dts | 29 + arch/arm/boot/dts/overlays/BONE-ADC.dts | 32 + arch/arm/boot/dts/overlays/Makefile | 11 + .../dt-bindings/board/am335x-bbw-bbb-base.h | 103 + 32 files changed, 9600 insertions(+), 94 deletions(-) create mode 100644 arch/arm/boot/dts/am335x-abbbi.dts create mode 100644 arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-common-univ.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-jtag.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-uboot-univ.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-uboot.dts create mode 100644 arch/arm/boot/dts/am335x-bonegreen-gateway.dts create mode 100644 arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi create mode 100644 arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts create mode 100644 arch/arm/boot/dts/overlays/BBORG_FAN-A000.dts create mode 100644 arch/arm/boot/dts/overlays/BONE-ADC.dts create mode 100644 arch/arm/boot/dts/overlays/Makefile create mode 100644 include/dt-bindings/board/am335x-bbw-bbb-base.h diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 0615a978e36dc..1b7be948a2bc2 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -740,6 +740,12 @@ dtb-$(CONFIG_SOC_AM33XX) += \ am335x-base0033.dtb \ am335x-bone.dtb \ am335x-boneblack.dtb \ + am335x-bonegreen-gateway.dtb \ + am335x-bonegreen-wireless-uboot-univ.dtb \ + am335x-boneblack-uboot-univ.dtb \ + am335x-boneblack-uboot.dtb \ + am335x-bone-uboot-univ.dtb \ + am335x-abbbi.dtb \ am335x-boneblack-wireless.dtb \ am335x-boneblack-prusuart.dtb \ am335x-boneblue.dtb \ diff --git a/arch/arm/boot/dts/am335x-abbbi.dts b/arch/arm/boot/dts/am335x-abbbi.dts new file mode 100644 index 0000000000000..870ce8567471b --- /dev/null +++ b/arch/arm/boot/dts/am335x-abbbi.dts @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright 2015 Konsulko Group + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" + +/ { + model = "Arrow BeagleBone Black Industrial"; + compatible = "arrow,am335x-abbbi", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-abbbi.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&am33xx_pinmux { + adi_hdmi_bbbi_pins: adi_hdmi_bbbi_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + mcasp0_pins: mcasp0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + >; + }; +}; + +&lcdc { + status = "okay"; + + /* If you want to get 24 bit RGB and 16 BGR mode instead of + * current 16 bit RGB and 24 BGR modes, set the propety + * below to "crossed" and uncomment the video-ports -property + * in tda19988 node. + */ + blue-and-red-wiring = "straight"; + + port { + lcdc_0: endpoint@0 { + remote-endpoint = <&hdmi_0>; + }; + }; +}; + +&i2c0 { + adv7511: adv7511@39 { + compatible = "adi,adv7511"; + reg = <0x39>; + + pinctrl-names = "default"; + pinctrl-0 = <&adi_hdmi_bbbi_pins>; + + /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ + /* video-ports = <0x234501>; */ + + #sound-dai-cells = <0>; + + ports { + port@0 { + hdmi_0: endpoint@0 { + remote-endpoint = <&lcdc_0>; + }; + }; + }; + }; +}; + +&rtc { + system-power-controller; +}; + +&mcasp0 { + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + status = "okay"; + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 0 0 1 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +/ { + clk_mcasp0_fixed: clk_mcasp0_fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24576000>; + }; + + clk_mcasp0: clk_mcasp0 { + #clock-cells = <0>; + compatible = "gpio-gate-clock"; + clocks = <&clk_mcasp0_fixed>; + enable-gpios = <&gpio1 27 0>; /* BeagleBone Black Clk enable on GPIO1_27 */ + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "TI BeagleBone Black"; + simple-audio-card,format = "i2s"; + simple-audio-card,bitclock-master = <&dailink0_master>; + simple-audio-card,frame-master = <&dailink0_master>; + + dailink0_master: simple-audio-card,cpu { + sound-dai = <&mcasp0>; + clocks = <&clk_mcasp0>; + }; + + simple-audio-card,codec { + sound-dai = <&adv7511>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi new file mode 100644 index 0000000000000..46bc98409fa76 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +/ { + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x10000000>; /* 256 MB */ + }; + + chosen { + stdout-path = &uart0; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led2 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led4 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led5 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&am33xx_pinmux { + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ + >; + }; + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + mmc1_pins: pinmux_mmc1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + + status = "okay"; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; +}; + +&usb1_phy { + status = "okay"; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + interrupts-extended = <&intc 18 &tps 0>; + interrupt-names = "mc", "vbus"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + + status = "okay"; + clock-frequency = <100000>; +}; + + +/include/ "tps65217.dtsi" + +&tps { + /* + * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only + * mode") at poweroff. Most BeagleBone versions do not support RTC-only + * mode and risk hardware damage if this mode is entered. + * + * For details, see linux-omap mailing list May 2015 thread + * [PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller + * In particular, messages: + * http://www.spinics.net/lists/linux-omap/msg118585.html + * http://www.spinics.net/lists/linux-omap/msg118615.html + * + * You can override this later with + * &tps { /delete-property/ ti,pmic-shutdown-controller; } + * if you want to use RTC-only mode and made sure you are not affected + * by the hardware problems. (Tip: double-check by performing a current + * measurement after shutdown: it should be less than 1 mA.) + */ + + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy-handle = <ðphy0>; + phy-mode = "mii"; +}; + +&mac { + slaves = <1>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; + + ethphy0: ethernet-phy@0 { + reg = <0>; + }; +}; + +&mmc1 { + status = "okay"; + bus-width = <0x4>; + pinctrl-names = "default"; + pinctrl-0 = <&mmc1_pins>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + clocks = <&clk_32768_ck>, <&clk_24mhz_clkctrl AM3_CLK_24MHZ_CLKDIV32K_CLKCTRL 0>; + clock-names = "ext-clk", "int-clk"; + system-power-controller; +}; + +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +&pruss_tm { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-univ.dtsi b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi new file mode 100644 index 0000000000000..c0f5bf58cf340 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi @@ -0,0 +1,2961 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) gpio0_26 */ + P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) gpio0_27 */ + P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_pwm_pin: pinmux_P8_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio1_29 */ + P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio3_16 */ + P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_pwm_pin: pinmux_P9_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/1"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/2"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/1"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/2"; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/3"; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/4"; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/5"; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/0"; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/1"; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/0"; +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/1"; +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/2"; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) */ + P8_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_14_default_pin>; + pinctrl-1 = <&P8_14_gpio_pin>; + pinctrl-2 = <&P8_14_gpio_pu_pin>; + pinctrl-3 = <&P8_14_gpio_pd_pin>; + pinctrl-4 = <&P8_14_gpio_input_pin>; + pinctrl-5 = <&P8_14_pwm_pin>; + }; + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) */ + P8_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_17_default_pin>; + pinctrl-1 = <&P8_17_gpio_pin>; + pinctrl-2 = <&P8_17_gpio_pu_pin>; + pinctrl-3 = <&P8_17_gpio_pd_pin>; + pinctrl-4 = <&P8_17_gpio_input_pin>; + pinctrl-5 = <&P8_17_pwm_pin>; + }; + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) */ + P8_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_26_default_pin>; + pinctrl-1 = <&P8_26_gpio_pin>; + pinctrl-2 = <&P8_26_gpio_pu_pin>; + pinctrl-3 = <&P8_26_gpio_pd_pin>; + pinctrl-4 = <&P8_26_gpio_input_pin>; + }; + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_30_default_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_gpio_input_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pwm_pin>; + pinctrl-7 = <&P9_30_pruout_pin>; + pinctrl-8 = <&P9_30_pruin_pin>; + }; + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_14 { + gpio-name = "P8_14"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_17 { + gpio-name = "P8_17"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_26 { + gpio-name = "P8_26"; + gpio = <&gpio1 29 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_30 { + gpio-name = "P9_30"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi index ee04b7dd07acb..3f3f885754981 100644 --- a/arch/arm/boot/dts/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/am335x-bone-common.dtsi @@ -26,14 +26,14 @@ compatible = "gpio-leds"; led2 { - label = "beaglebone:green:heartbeat"; + label = "beaglebone:green:usr0"; gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; default-state = "off"; }; led3 { - label = "beaglebone:green:mmc0"; + label = "beaglebone:green:usr1"; gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; linux,default-trigger = "mmc0"; default-state = "off"; @@ -63,9 +63,6 @@ }; &am33xx_pinmux { - pinctrl-names = "default"; - pinctrl-0 = <&clkout2_pin>; - user_leds_s0: user_leds_s0 { pinctrl-single,pins = < AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ @@ -82,12 +79,12 @@ >; }; - i2c2_pins: pinmux_i2c2_pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ - AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ - >; - }; +// i2c2_pins: pinmux_i2c2_pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ +// AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ +// >; +// }; uart0_pins: pinmux_uart0_pins { pinctrl-single,pins = < @@ -96,12 +93,6 @@ >; }; - clkout2_pin: pinmux_clkout2_pin { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* xdma_event_intr1.clkout2 */ - >; - }; - cpsw_default: cpsw_default { pinctrl-single,pins = < /* Slave 1 */ @@ -182,6 +173,50 @@ AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ >; }; + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ }; &uart0 { @@ -248,7 +283,8 @@ &i2c2 { pinctrl-names = "default"; - pinctrl-0 = <&i2c2_pins>; + //pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; status = "okay"; clock-frequency = <100000>; @@ -428,3 +464,62 @@ &pruss_tm { status = "okay"; }; + +&ocp { + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-jtag.dtsi b/arch/arm/boot/dts/am335x-bone-jtag.dtsi new file mode 100644 index 0000000000000..2cdca01417121 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-jtag.dtsi @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2015 Robert Nelson + */ + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&clkout2_pin>; + + clkout2_pin: pinmux_clkout2_pin { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* xdma_event_intr1.clkout2 */ + >; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-uboot-univ.dts b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts new file mode 100644 index 0000000000000..87559166d38ee --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone"; + compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&ldo3_reg>; +}; diff --git a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts index 43bfbce410494..c7ae3e59d29e3 100644 --- a/arch/arm/boot/dts/am335x-bone.dts +++ b/arch/arm/boot/dts/am335x-bone.dts @@ -6,10 +6,16 @@ #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone"; compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &ldo3_reg { diff --git a/arch/arm/boot/dts/am335x-boneblack-common.dtsi b/arch/arm/boot/dts/am335x-boneblack-common.dtsi index 91f93bc89716d..a9b8bd273cbf0 100644 --- a/arch/arm/boot/dts/am335x-boneblack-common.dtsi +++ b/arch/arm/boot/dts/am335x-boneblack-common.dtsi @@ -27,7 +27,6 @@ &am33xx_pinmux { nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins { pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT_PULLDOWN, MUX_MODE3) AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) @@ -51,12 +50,6 @@ >; }; - nxp_hdmi_bonelt_off_pins: nxp_hdmi_bonelt_off_pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT_PULLDOWN, MUX_MODE3) - >; - }; - mcasp0_pins: mcasp0_pins { pinctrl-single,pins = < AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ @@ -89,12 +82,9 @@ tda19988: tda19988@70 { compatible = "nxp,tda998x"; reg = <0x70>; - nxp,calib-gpios = <&gpio1 25 0>; - interrupts-extended = <&gpio1 25 IRQ_TYPE_LEVEL_LOW>; - pinctrl-names = "default", "off"; + pinctrl-names = "default"; pinctrl-0 = <&nxp_hdmi_bonelt_pins>; - pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ /* video-ports = <0x234501>; */ diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts new file mode 100644 index 0000000000000..4e5633e386945 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot.dts b/arch/arm/boot/dts/am335x-boneblack-uboot.dts new file mode 100644 index 0000000000000..285a921268545 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot.dts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wireless.dts b/arch/arm/boot/dts/am335x-boneblack-wireless.dts index 3124d94c0b3cc..f98e0cfe14ad0 100644 --- a/arch/arm/boot/dts/am335x-boneblack-wireless.dts +++ b/arch/arm/boot/dts/am335x-boneblack-wireless.dts @@ -8,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black Wireless"; compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-boneblack-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; diff --git a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts index d3928662aed4e..40b17b9561099 100644 --- a/arch/arm/boot/dts/am335x-boneblack.dts +++ b/arch/arm/boot/dts/am335x-boneblack.dts @@ -7,10 +7,16 @@ #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black"; compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &cpu0_opp_table { diff --git a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts index 2f6652ef9a157..3e2e2b43b9d93 100644 --- a/arch/arm/boot/dts/am335x-boneblue.dts +++ b/arch/arm/boot/dts/am335x-boneblue.dts @@ -14,6 +14,8 @@ chosen { stdout-path = &uart0; + base_dtb = "am335x-boneblue.dts"; + base_dtb_timestamp = __TIMESTAMP__; }; leds { diff --git a/arch/arm/boot/dts/am335x-bonegreen-gateway.dts b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts new file mode 100644 index 0000000000000..964db3920a972 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-common.dtsi" +#include +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "SeeedStudio BeagleBone Green Gateway"; + compatible = "ti,am335x-bone-green-gateway", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + aliases { + rtc0 = &extrtc; + rtc1 = &rtc; + }; + + chosen { + base_dtb = "am335x-bonegreen-gateway.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us= <70000>; + + /* WL_EN */ + gpio = <&gpio3 9 0>; + enable-active-high; + }; + + leds { + pinctrl-0 = <&user_leds_s0 &extra_led_pins>; + + led6 { + label = "beaglebone:green:usr4"; + gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "netdev"; + default-state = "off"; + }; + }; +}; + +&cpu0_opp_table { + /* + * Octavo Systems: + * The EFUSE_SMA register is not programmed for any of the AM335x wafers + * we get and we are not programming them during our production test. + * Therefore, from a DEVICE_ID revision point of view, the silicon looks + * like it is Revision 2.1. However, from an EFUSE_SMA point of view for + * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the + * EFUSE_SMA register reads as all zeros). + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&usbhost_pins>; + + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + >; + }; + + mmc3_pins: pinmux_mmc3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + uart2_grove_pins: pinmux_uart2_grove_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x90c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + AM33XX_IOPAD(0x910, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + >; + }; + + uart3_pins: pinmux_uart3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ + >; + }; + + extra_led_pins: pinmux_extra_led_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_OUTPUT, MUX_MODE7) /* WL_Active_LED / USR4 */ + >; + }; + + + wl18xx_pins: pinmux_wl18xx_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + >; + }; + + usbhost_pins: pinmux_usbhost_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd1.gpio0[21] */ + >; + }; +}; + +&mac { + status = "disabled"; +}; + +&mmc3 { + dmas = <&edma_xbar 12 0 1 + &edma_xbar 13 0 2>; + dma-names = "tx", "rx"; + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + non-removable; + cap-power-off-card; + ti,needs-special-hs-handling; + keep-power-in-suspend; + pinctrl-names = "default"; + pinctrl-0 = <&mmc3_pins &wl18xx_pins>; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@2 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_grove_pins>; + status = "okay"; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_pins &bt_pins>; + status = "okay"; + + bluetooth { + compatible = "ti,wl1835-st"; + enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c0 { + extrtc: rtc@68 { + compatible = "dallas,ds1340"; + reg = <0x68>; + }; +}; + +// (K16) gmii1_txd1.gpio0[21] +&gpio0 { + usb_reset { + gpio-hog; + gpios = <21 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb_reset"; + }; +}; + +&gpio3 { + ls_buf_en { + gpio-hog; + gpios = <10 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +&usb1 { + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb424,9512"; + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + ethernet: ethernet@1 { + compatible = "usb424,ec00"; + reg = <1>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi new file mode 100644 index 0000000000000..4662f304f3eed --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi @@ -0,0 +1,2823 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts new file mode 100644 index 0000000000000..ac63c249b1961 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-wireless-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Green Wireless"; + compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen-wireless-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&gpio1 { + ls_buf_en { + gpio-hog; + gpios = <29 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +/* BT_AUD_OUT from wl1835 has to be pulled low when WL_EN is activated.*/ +/* in case it isn't, wilink8 ends up in one of the test modes that */ +/* intruces various issues (elp wkaeup timeouts etc.) */ +/* On the BBGW this pin is routed through the level shifter (U21) that */ +/* introduces a pullup on the line and wilink8 ends up in a bad state. */ +/* use a gpio hog to force this pin low. An alternative may be adding */ +/* an external pulldown on U21 pin 4. */ + +&gpio3 { + bt_aud_in { + gpio-hog; + gpios = <16 GPIO_ACTIVE_HIGH>; + output-low; + line-name = "MCASP0_AHCLKR"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts index 4092cd193b8a6..a0a841d97165e 100644 --- a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts @@ -8,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green Wireless"; compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-bonegreen-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; diff --git a/arch/arm/boot/dts/am335x-bonegreen.dts b/arch/arm/boot/dts/am335x-bonegreen.dts index c12bb07177796..d3009aeeb640e 100644 --- a/arch/arm/boot/dts/am335x-bonegreen.dts +++ b/arch/arm/boot/dts/am335x-bonegreen.dts @@ -7,8 +7,14 @@ #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green"; compatible = "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; diff --git a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts index f47cc9fea2538..bdb783879277e 100644 --- a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts +++ b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts @@ -13,10 +13,16 @@ #include #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "Octavo Systems OSD3358-SM-RED"; compatible = "oct,osd3358-sm-refdesign", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-osd3358-sm-red.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &ldo3_reg { @@ -40,7 +46,6 @@ &am33xx_pinmux { nxp_hdmi_bonelt_pins: nxp-hdmi-bonelt-pins { pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT_PULLDOWN, MUX_MODE3) AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) @@ -64,12 +69,6 @@ >; }; - nxp_hdmi_bonelt_off_pins: nxp-hdmi-bonelt-off-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT_PULLDOWN, MUX_MODE3) - >; - }; - mcasp0_pins: mcasp0-pins { pinctrl-single,pins = < AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) @@ -124,9 +123,8 @@ compatible = "nxp,tda998x"; reg = <0x70>; - pinctrl-names = "default", "off"; + pinctrl-names = "default"; pinctrl-0 = <&nxp_hdmi_bonelt_pins>; - pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ /* video-ports = <0x234501>; */ @@ -264,9 +262,6 @@ }; &am33xx_pinmux { - pinctrl-names = "default"; - pinctrl-0 = <&clkout2_pin>; - user_leds_s0: user-leds-s0 { pinctrl-single,pins = < AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ @@ -290,12 +285,6 @@ >; }; - clkout2_pin: pinmux-clkout2-pin { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* xdma_event_intr1.clkout2 */ - >; - }; - cpsw_default: cpsw-default { pinctrl-single,pins = < /* Slave 1 */ diff --git a/arch/arm/boot/dts/am335x-pocketbeagle.dts b/arch/arm/boot/dts/am335x-pocketbeagle.dts index abf2badce53d9..6c91d8558d0f4 100644 --- a/arch/arm/boot/dts/am335x-pocketbeagle.dts +++ b/arch/arm/boot/dts/am335x-pocketbeagle.dts @@ -15,6 +15,8 @@ chosen { stdout-path = &uart0; + base_dtb = "am335x-pocketbeagle.dts"; + base_dtb_timestamp = __TIMESTAMP__; }; leds { @@ -60,24 +62,24 @@ }; &am33xx_pinmux { - i2c2_pins: pinmux-i2c2-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ - AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ - >; - }; +// i2c2_pins: pinmux-i2c2-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ +// AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ +// >; +// }; - ehrpwm0_pins: pinmux-ehrpwm0-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* (A13) mcasp0_aclkx.ehrpwm0A */ - >; - }; +// ehrpwm0_pins: pinmux-ehrpwm0-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* (A13) mcasp0_aclkx.ehrpwm0A */ +// >; +// }; - ehrpwm1_pins: pinmux-ehrpwm1-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_GPMC_A2, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U14) gpmc_a2.ehrpwm1A */ - >; - }; +// ehrpwm1_pins: pinmux-ehrpwm1-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_GPMC_A2, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U14) gpmc_a2.ehrpwm1A */ +// >; +// }; mmc0_pins: pinmux-mmc0-pins { pinctrl-single,pins = < @@ -91,23 +93,23 @@ >; }; - spi0_pins: pinmux-spi0-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT_PULLUP, MUX_MODE0) - AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_INPUT_PULLUP, MUX_MODE0) - AM33XX_PADCONF(AM335X_PIN_SPI0_D1, PIN_INPUT_PULLUP, MUX_MODE0) - AM33XX_PADCONF(AM335X_PIN_SPI0_CS0, PIN_INPUT_PULLUP, MUX_MODE0) - >; - }; +// spi0_pins: pinmux-spi0-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT_PULLUP, MUX_MODE0) /* (A17) spi0_sclk.spi0_sclk */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_INPUT_PULLUP, MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_D1, PIN_INPUT_PULLUP, MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_CS0, PIN_INPUT_PULLUP, MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */ +// >; +// }; - spi1_pins: pinmux-spi1-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_ECAP0_IN_PWM0_OUT, PIN_INPUT_PULLUP, MUX_MODE4) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */ - AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E18) uart0_ctsn.spi1_d0 */ - AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E17) uart0_rtsn.spi1_d1 */ - AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_INPUT_PULLUP, MUX_MODE4) /* (A15) xdma_event_intr0.spi1_cs1 */ - >; - }; +// spi1_pins: pinmux-spi1-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_ECAP0_IN_PWM0_OUT, PIN_INPUT_PULLUP, MUX_MODE4) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */ +// AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E18) uart0_ctsn.spi1_d0 */ +// AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E17) uart0_rtsn.spi1_d1 */ +// AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_INPUT_PULLUP, MUX_MODE4) /* (A15) xdma_event_intr0.spi1_cs1 */ +// >; +// }; usr_leds_pins: pinmux-usr-leds-pins { pinctrl-single,pins = < @@ -125,12 +127,839 @@ >; }; - uart4_pins: pinmux-uart4-pins { - pinctrl-single,pins = < - AM33XX_PADCONF(AM335X_PIN_GPMC_WAIT0, PIN_INPUT_PULLUP, MUX_MODE6) /* (T17) gpmc_wait0.uart4_rxd */ - AM33XX_PADCONF(AM335X_PIN_GPMC_WPN, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U17) gpmc_wpn.uart4_txd */ - >; - }; +// uart4_pins: pinmux-uart4-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_GPMC_WAIT0, PIN_INPUT_PULLUP, MUX_MODE6) /* (T17) gpmc_wait0.uart4_rxd */ +// AM33XX_PADCONF(AM335X_PIN_GPMC_WPN, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U17) gpmc_wpn.uart4_txd */ +// >; +// }; + + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio2_23 */ + P1_02_default_pin: pinmux_P1_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pin: pinmux_P1_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pu_pin: pinmux_P1_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pd_pin: pinmux_P1_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_input_pin: pinmux_P1_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_pruout_pin: pinmux_P1_02_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P1_02_pruin_pin: pinmux_P1_02_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) gpio2_25 */ + P1_04_default_pin: pinmux_P1_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pin: pinmux_P1_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pu_pin: pinmux_P1_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pd_pin: pinmux_P1_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_input_pin: pinmux_P1_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_pruout_pin: pinmux_P1_04_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P1_04_pruin_pin: pinmux_P1_04_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi0_cs0 */ + P1_06_default_pin: pinmux_P1_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_gpio_pin: pinmux_P1_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pu_pin: pinmux_P1_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pd_pin: pinmux_P1_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_input_pin: pinmux_P1_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_spi_cs_pin: pinmux_P1_06_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_i2c_pin: pinmux_P1_06_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P1_06_pwm_pin: pinmux_P1_06_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P1_06_pru_uart_pin: pinmux_P1_06_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi0_sclk */ + P1_08_default_pin: pinmux_P1_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_gpio_pin: pinmux_P1_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pu_pin: pinmux_P1_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pd_pin: pinmux_P1_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_input_pin: pinmux_P1_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_spi_sclk_pin: pinmux_P1_08_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_uart_pin: pinmux_P1_08_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P1_08_i2c_pin: pinmux_P1_08_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P1_08_pwm_pin: pinmux_P1_08_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P1_08_pru_uart_pin: pinmux_P1_08_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi0_d0 */ + P1_10_default_pin: pinmux_P1_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_gpio_pin: pinmux_P1_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pu_pin: pinmux_P1_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pd_pin: pinmux_P1_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_input_pin: pinmux_P1_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_spi_pin: pinmux_P1_10_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_uart_pin: pinmux_P1_10_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P1_10_i2c_pin: pinmux_P1_10_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P1_10_pwm_pin: pinmux_P1_10_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P1_10_pru_uart_pin: pinmux_P1_10_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi0_d1 */ + P1_12_default_pin: pinmux_P1_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_gpio_pin: pinmux_P1_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pu_pin: pinmux_P1_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pd_pin: pinmux_P1_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_input_pin: pinmux_P1_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_spi_pin: pinmux_P1_12_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_i2c_pin: pinmux_P1_12_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P1_12_pwm_pin: pinmux_P1_12_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P1_12_pru_uart_pin: pinmux_P1_12_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) gpio0_20 */ + P1_20_default_pin: pinmux_P1_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pin: pinmux_P1_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pu_pin: pinmux_P1_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pd_pin: pinmux_P1_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_input_pin: pinmux_P1_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_pruin_pin: pinmux_P1_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c2_sda */ + P1_26_default_pin: pinmux_P1_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_gpio_pin: pinmux_P1_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pu_pin: pinmux_P1_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pd_pin: pinmux_P1_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_input_pin: pinmux_P1_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_can_pin: pinmux_P1_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P1_26_i2c_pin: pinmux_P1_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_spi_cs_pin: pinmux_P1_26_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P1_26_pru_uart_pin: pinmux_P1_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c2_scl */ + P1_28_default_pin: pinmux_P1_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_gpio_pin: pinmux_P1_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pu_pin: pinmux_P1_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pd_pin: pinmux_P1_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_input_pin: pinmux_P1_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_can_pin: pinmux_P1_28_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P1_28_i2c_pin: pinmux_P1_28_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_spi_cs_pin: pinmux_P1_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P1_28_pru_uart_pin: pinmux_P1_28_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P1_29 (ZCZ ball A14) pru0_in7 */ + P1_29_default_pin: pinmux_P1_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + P1_29_gpio_pin: pinmux_P1_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pu_pin: pinmux_P1_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pd_pin: pinmux_P1_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_input_pin: pinmux_P1_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_qep_pin: pinmux_P1_29_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P1_29_pruout_pin: pinmux_P1_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P1_29_pruin_pin: pinmux_P1_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P1_30 (ZCZ ball E16) uart0_txd */ + P1_30_default_pin: pinmux_P1_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_gpio_pin: pinmux_P1_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pu_pin: pinmux_P1_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pd_pin: pinmux_P1_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_input_pin: pinmux_P1_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_uart_pin: pinmux_P1_30_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_spi_cs_pin: pinmux_P1_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_txd.spi1_cs1 */ + P1_30_can_pin: pinmux_P1_30_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_txd.dcan0_rx */ + P1_30_i2c_pin: pinmux_P1_30_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_txd.i2c2_scl */ + P1_30_pruout_pin: pinmux_P1_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_txd.pru1_out15 */ + P1_30_pruin_pin: pinmux_P1_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE6) >; }; /* uart0_txd.pru1_in15 */ + + /* P1_31 (ZCZ ball B12) pru0_in4 */ + P1_31_default_pin: pinmux_P1_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + P1_31_gpio_pin: pinmux_P1_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pu_pin: pinmux_P1_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pd_pin: pinmux_P1_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_input_pin: pinmux_P1_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_qep_pin: pinmux_P1_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P1_31_pruout_pin: pinmux_P1_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P1_31_pruin_pin: pinmux_P1_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P1_32 (ZCZ ball E15) uart0_rxd */ + P1_32_default_pin: pinmux_P1_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_gpio_pin: pinmux_P1_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pu_pin: pinmux_P1_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pd_pin: pinmux_P1_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_input_pin: pinmux_P1_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_uart_pin: pinmux_P1_32_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_spi_cs_pin: pinmux_P1_32_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rxd.spi1_cs0 */ + P1_32_can_pin: pinmux_P1_32_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rxd.dcan0_tx */ + P1_32_i2c_pin: pinmux_P1_32_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rxd.i2c2_sda */ + P1_32_pruout_pin: pinmux_P1_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_rxd.pru1_out14 */ + P1_32_pruin_pin: pinmux_P1_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE6) >; }; /* uart0_rxd.pru1_in14 */ + + /* P1_33 (ZCZ ball B13) pru0_in1 */ + P1_33_default_pin: pinmux_P1_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + P1_33_gpio_pin: pinmux_P1_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pu_pin: pinmux_P1_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pd_pin: pinmux_P1_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_input_pin: pinmux_P1_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_pwm_pin: pinmux_P1_33_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P1_33_spi_pin: pinmux_P1_33_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P1_33_pruout_pin: pinmux_P1_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P1_33_pruin_pin: pinmux_P1_33_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P1_34 (ZCZ ball T11) gpio0_26 */ + P1_34_default_pin: pinmux_P1_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pin: pinmux_P1_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pu_pin: pinmux_P1_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pd_pin: pinmux_P1_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_input_pin: pinmux_P1_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_pwm_pin: pinmux_P1_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P1_35 (ZCZ ball V5) pru1_in10 */ + P1_35_default_pin: pinmux_P1_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + P1_35_gpio_pin: pinmux_P1_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pu_pin: pinmux_P1_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pd_pin: pinmux_P1_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_input_pin: pinmux_P1_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_pruout_pin: pinmux_P1_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P1_35_pruin_pin: pinmux_P1_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P1_36 (ZCZ ball A13) ehrpwm0a */ + P1_36_default_pin: pinmux_P1_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_gpio_pin: pinmux_P1_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pu_pin: pinmux_P1_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pd_pin: pinmux_P1_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_input_pin: pinmux_P1_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_pwm_pin: pinmux_P1_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_spi_sclk_pin: pinmux_P1_36_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P1_36_pruout_pin: pinmux_P1_36_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P1_36_pruin_pin: pinmux_P1_36_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) ehrpwm1a */ + P2_01_default_pin: pinmux_P2_01_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + P2_01_gpio_pin: pinmux_P2_01_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pu_pin: pinmux_P2_01_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pd_pin: pinmux_P2_01_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_input_pin: pinmux_P2_01_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_pwm_pin: pinmux_P2_01_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P2_02 (ZCZ ball V17) gpio1_27 */ + P2_02_default_pin: pinmux_P2_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pin: pinmux_P2_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pu_pin: pinmux_P2_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pd_pin: pinmux_P2_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_input_pin: pinmux_P2_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + + /* P2_03 (ZCZ ball T10) gpio0_23 */ + P2_03_default_pin: pinmux_P2_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pin: pinmux_P2_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pu_pin: pinmux_P2_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pd_pin: pinmux_P2_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_input_pin: pinmux_P2_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_pwm_pin: pinmux_P2_03_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P2_04 (ZCZ ball T16) gpio1_26 */ + P2_04_default_pin: pinmux_P2_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pin: pinmux_P2_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pu_pin: pinmux_P2_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pd_pin: pinmux_P2_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_input_pin: pinmux_P2_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + + /* P2_05 (ZCZ ball T17) uart4_rxd */ + P2_05_default_pin: pinmux_P2_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + P2_05_gpio_pin: pinmux_P2_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pu_pin: pinmux_P2_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pd_pin: pinmux_P2_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_input_pin: pinmux_P2_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_uart_pin: pinmux_P2_05_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P2_06 (ZCZ ball U16) gpio1_25 */ + P2_06_default_pin: pinmux_P2_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pin: pinmux_P2_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pu_pin: pinmux_P2_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pd_pin: pinmux_P2_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_input_pin: pinmux_P2_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + + /* P2_07 (ZCZ ball U17) uart4_txd */ + P2_07_default_pin: pinmux_P2_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + P2_07_gpio_pin: pinmux_P2_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pu_pin: pinmux_P2_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pd_pin: pinmux_P2_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_input_pin: pinmux_P2_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_uart_pin: pinmux_P2_07_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P2_08 (ZCZ ball U18) gpio1_28 */ + P2_08_default_pin: pinmux_P2_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pin: pinmux_P2_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pu_pin: pinmux_P2_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pd_pin: pinmux_P2_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_input_pin: pinmux_P2_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P2_09 (ZCZ ball D15) i2c1_scl */ + P2_09_default_pin: pinmux_P2_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_gpio_pin: pinmux_P2_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pu_pin: pinmux_P2_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pd_pin: pinmux_P2_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_input_pin: pinmux_P2_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_uart_pin: pinmux_P2_09_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P2_09_can_pin: pinmux_P2_09_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P2_09_i2c_pin: pinmux_P2_09_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_pru_uart_pin: pinmux_P2_09_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P2_09_pruin_pin: pinmux_P2_09_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P2_10 (ZCZ ball R14) gpio1_20 */ + P2_10_default_pin: pinmux_P2_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pin: pinmux_P2_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pu_pin: pinmux_P2_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pd_pin: pinmux_P2_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_input_pin: pinmux_P2_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_qep_pin: pinmux_P2_10_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a4.eqep1a_in */ + + /* P2_11 (ZCZ ball D16) i2c1_sda */ + P2_11_default_pin: pinmux_P2_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_gpio_pin: pinmux_P2_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pu_pin: pinmux_P2_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pd_pin: pinmux_P2_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_input_pin: pinmux_P2_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_uart_pin: pinmux_P2_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P2_11_can_pin: pinmux_P2_11_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P2_11_i2c_pin: pinmux_P2_11_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_pru_uart_pin: pinmux_P2_11_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P2_11_pruin_pin: pinmux_P2_11_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) gpio2_1 */ + P2_17_default_pin: pinmux_P2_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pin: pinmux_P2_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pu_pin: pinmux_P2_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pd_pin: pinmux_P2_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_input_pin: pinmux_P2_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P2_18 (ZCZ ball U13) gpio1_15 */ + P2_18_default_pin: pinmux_P2_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pin: pinmux_P2_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pu_pin: pinmux_P2_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pd_pin: pinmux_P2_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_input_pin: pinmux_P2_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_qep_pin: pinmux_P2_18_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P2_18_pru_ecap_pin: pinmux_P2_18_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P2_18_pruin_pin: pinmux_P2_18_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P2_19 (ZCZ ball U12) gpio0_27 */ + P2_19_default_pin: pinmux_P2_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pin: pinmux_P2_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pu_pin: pinmux_P2_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pd_pin: pinmux_P2_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_input_pin: pinmux_P2_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_pwm_pin: pinmux_P2_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P2_20 (ZCZ ball T13) gpio2_0 */ + P2_20_default_pin: pinmux_P2_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pin: pinmux_P2_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pu_pin: pinmux_P2_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pd_pin: pinmux_P2_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_input_pin: pinmux_P2_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) gpio1_14 */ + P2_22_default_pin: pinmux_P2_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pin: pinmux_P2_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pu_pin: pinmux_P2_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pd_pin: pinmux_P2_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_input_pin: pinmux_P2_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_qep_pin: pinmux_P2_22_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P2_22_pruin_pin: pinmux_P2_22_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) gpio1_12 */ + P2_24_default_pin: pinmux_P2_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pin: pinmux_P2_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pu_pin: pinmux_P2_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pd_pin: pinmux_P2_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_input_pin: pinmux_P2_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_qep_pin: pinmux_P2_24_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P2_24_pruout_pin: pinmux_P2_24_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P2_25 (ZCZ ball E17) spi1_d1 */ + P2_25_default_pin: pinmux_P2_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_gpio_pin: pinmux_P2_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pu_pin: pinmux_P2_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pd_pin: pinmux_P2_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_input_pin: pinmux_P2_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_uart_pin: pinmux_P2_25_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rtsn.uart4_txd */ + P2_25_can_pin: pinmux_P2_25_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rtsn.dcan1_rx */ + P2_25_i2c_pin: pinmux_P2_25_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rtsn.i2c1_scl */ + P2_25_spi_pin: pinmux_P2_25_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_spi_cs_pin: pinmux_P2_25_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart0_rtsn.spi1_cs0 */ + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi1_d0 */ + P2_27_default_pin: pinmux_P2_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + P2_27_gpio_pin: pinmux_P2_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pu_pin: pinmux_P2_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pd_pin: pinmux_P2_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_input_pin: pinmux_P2_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_INPUT | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_uart_pin: pinmux_P2_27_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_ctsn.uart4_rxd */ + P2_27_can_pin: pinmux_P2_27_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_ctsn.dcan1_tx */ + P2_27_i2c_pin: pinmux_P2_27_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_ctsn.i2c1_sda */ + P2_27_spi_pin: pinmux_P2_27_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + + /* P2_28 (ZCZ ball D13) pru0_in6 */ + P2_28_default_pin: pinmux_P2_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + P2_28_gpio_pin: pinmux_P2_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pu_pin: pinmux_P2_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pd_pin: pinmux_P2_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_input_pin: pinmux_P2_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_qep_pin: pinmux_P2_28_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P2_28_pruout_pin: pinmux_P2_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P2_28_pruin_pin: pinmux_P2_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P2_29 (ZCZ ball C18) spi1_sclk */ + P2_29_default_pin: pinmux_P2_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + P2_29_gpio_pin: pinmux_P2_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pu_pin: pinmux_P2_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pd_pin: pinmux_P2_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_input_pin: pinmux_P2_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_pwm_pin: pinmux_P2_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P2_29_uart_pin: pinmux_P2_29_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P2_29_spi_cs_pin: pinmux_P2_29_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P2_29_pru_ecap_pin: pinmux_P2_29_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P2_29_spi_sclk_pin: pinmux_P2_29_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P2_30 (ZCZ ball C12) pru0_in3 */ + P2_30_default_pin: pinmux_P2_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + P2_30_gpio_pin: pinmux_P2_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pu_pin: pinmux_P2_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pd_pin: pinmux_P2_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_input_pin: pinmux_P2_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_pwm_pin: pinmux_P2_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P2_30_spi_cs_pin: pinmux_P2_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P2_30_pruout_pin: pinmux_P2_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P2_30_pruin_pin: pinmux_P2_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P2_31 (ZCZ ball A15) spi1_cs1 */ + P2_31_default_pin: pinmux_P2_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_gpio_pin: pinmux_P2_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pu_pin: pinmux_P2_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pd_pin: pinmux_P2_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_input_pin: pinmux_P2_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_spi_cs_pin: pinmux_P2_31_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_pruin_pin: pinmux_P2_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr0.pru1_in16 */ + + /* P2_32 (ZCZ ball D12) pru0_in2 */ + P2_32_default_pin: pinmux_P2_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + P2_32_gpio_pin: pinmux_P2_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pu_pin: pinmux_P2_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pd_pin: pinmux_P2_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_input_pin: pinmux_P2_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_pwm_pin: pinmux_P2_32_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P2_32_spi_pin: pinmux_P2_32_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P2_32_pruout_pin: pinmux_P2_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P2_32_pruin_pin: pinmux_P2_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P2_33 (ZCZ ball R12) gpio1_13 */ + P2_33_default_pin: pinmux_P2_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pin: pinmux_P2_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pu_pin: pinmux_P2_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pd_pin: pinmux_P2_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_input_pin: pinmux_P2_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_qep_pin: pinmux_P2_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P2_33_pruout_pin: pinmux_P2_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P2_34 (ZCZ ball C13) pru0_in5 */ + P2_34_default_pin: pinmux_P2_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + P2_34_gpio_pin: pinmux_P2_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pu_pin: pinmux_P2_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pd_pin: pinmux_P2_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_input_pin: pinmux_P2_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_qep_pin: pinmux_P2_34_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P2_34_pruout_pin: pinmux_P2_34_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P2_34_pruin_pin: pinmux_P2_34_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P2_35 (ZCZ ball U5) gpio2_22 */ + P2_35_default_pin: pinmux_P2_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pin: pinmux_P2_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pu_pin: pinmux_P2_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pd_pin: pinmux_P2_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_input_pin: pinmux_P2_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_pruout_pin: pinmux_P2_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P2_35_pruin_pin: pinmux_P2_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P2_36 (ZCZ ball C9) AIN7 */ }; &epwmss0 { @@ -140,7 +969,8 @@ &ehrpwm0 { status = "okay"; pinctrl-names = "default"; - pinctrl-0 = <&ehrpwm0_pins>; + //pinctrl-0 = <&ehrpwm0_pins>; + pinctrl-0 = <>; }; &epwmss1 { @@ -150,7 +980,18 @@ &ehrpwm1 { status = "okay"; pinctrl-names = "default"; - pinctrl-0 = <&ehrpwm1_pins>; + //pinctrl-0 = <&ehrpwm1_pins>; + pinctrl-0 = <>; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; }; &i2c0 { @@ -160,9 +1001,18 @@ }; }; +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <>; + + status = "okay"; + clock-frequency = <400000>; +}; + &i2c2 { pinctrl-names = "default"; - pinctrl-0 = <&i2c2_pins>; +// pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; status = "okay"; clock-frequency = <400000>; @@ -181,6 +1031,10 @@ system-power-controller; }; +&pruss_tm { + status = "okay"; +}; + &tscadc { status = "okay"; adc { @@ -193,14 +1047,30 @@ &uart0 { pinctrl-names = "default"; - pinctrl-0 = <&uart0_pins>; + //pinctrl-0 = <&uart0_pins>; + pinctrl-0 = <>; + + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <>; + + status = "okay"; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <>; status = "okay"; }; &uart4 { pinctrl-names = "default"; - pinctrl-0 = <&uart4_pins>; + //pinctrl-0 = <&uart4_pins>; + pinctrl-0 = <>; status = "okay"; }; @@ -234,3 +1104,1092 @@ &cppi41dma { status = "okay"; }; + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.1"; + reg = <1>; + spi-max-frequency = <24000000>; + status = "disabled"; + }; +}; + +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ocp { + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio_input */ + P1_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_02_default_pin>; + pinctrl-1 = <&P1_02_gpio_pin>; + pinctrl-2 = <&P1_02_gpio_pu_pin>; + pinctrl-3 = <&P1_02_gpio_pd_pin>; + pinctrl-4 = <&P1_02_gpio_input_pin>; + pinctrl-5 = <&P1_02_pruout_pin>; + pinctrl-6 = <&P1_02_pruin_pin>; + }; + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) */ + P1_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_04_default_pin>; + pinctrl-1 = <&P1_04_gpio_pin>; + pinctrl-2 = <&P1_04_gpio_pu_pin>; + pinctrl-3 = <&P1_04_gpio_pd_pin>; + pinctrl-4 = <&P1_04_gpio_input_pin>; + pinctrl-5 = <&P1_04_pruout_pin>; + pinctrl-6 = <&P1_04_pruin_pin>; + }; + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi_cs */ + P1_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_06_default_pin>; + pinctrl-1 = <&P1_06_gpio_pin>; + pinctrl-2 = <&P1_06_gpio_pu_pin>; + pinctrl-3 = <&P1_06_gpio_pd_pin>; + pinctrl-4 = <&P1_06_gpio_input_pin>; + pinctrl-5 = <&P1_06_spi_cs_pin>; + pinctrl-6 = <&P1_06_i2c_pin>; + pinctrl-7 = <&P1_06_pwm_pin>; + pinctrl-8 = <&P1_06_pru_uart_pin>; + }; + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi_sclk */ + P1_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_08_default_pin>; + pinctrl-1 = <&P1_08_gpio_pin>; + pinctrl-2 = <&P1_08_gpio_pu_pin>; + pinctrl-3 = <&P1_08_gpio_pd_pin>; + pinctrl-4 = <&P1_08_gpio_input_pin>; + pinctrl-5 = <&P1_08_spi_sclk_pin>; + pinctrl-6 = <&P1_08_uart_pin>; + pinctrl-7 = <&P1_08_i2c_pin>; + pinctrl-8 = <&P1_08_pwm_pin>; + pinctrl-9 = <&P1_08_pru_uart_pin>; + }; + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi */ + P1_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_10_default_pin>; + pinctrl-1 = <&P1_10_gpio_pin>; + pinctrl-2 = <&P1_10_gpio_pu_pin>; + pinctrl-3 = <&P1_10_gpio_pd_pin>; + pinctrl-4 = <&P1_10_gpio_input_pin>; + pinctrl-5 = <&P1_10_spi_pin>; + pinctrl-6 = <&P1_10_uart_pin>; + pinctrl-7 = <&P1_10_i2c_pin>; + pinctrl-8 = <&P1_10_pwm_pin>; + pinctrl-9 = <&P1_10_pru_uart_pin>; + }; + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi */ + P1_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_12_default_pin>; + pinctrl-1 = <&P1_12_gpio_pin>; + pinctrl-2 = <&P1_12_gpio_pu_pin>; + pinctrl-3 = <&P1_12_gpio_pd_pin>; + pinctrl-4 = <&P1_12_gpio_input_pin>; + pinctrl-5 = <&P1_12_spi_pin>; + pinctrl-6 = <&P1_12_i2c_pin>; + pinctrl-7 = <&P1_12_pwm_pin>; + pinctrl-8 = <&P1_12_pru_uart_pin>; + }; + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) */ + P1_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin"; + pinctrl-0 = <&P1_20_default_pin>; + pinctrl-1 = <&P1_20_gpio_pin>; + pinctrl-2 = <&P1_20_gpio_pu_pin>; + pinctrl-3 = <&P1_20_gpio_pd_pin>; + pinctrl-4 = <&P1_20_gpio_input_pin>; + pinctrl-5 = <&P1_20_pruin_pin>; + }; + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c */ + P1_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_26_default_pin>; + pinctrl-1 = <&P1_26_gpio_pin>; + pinctrl-2 = <&P1_26_gpio_pu_pin>; + pinctrl-3 = <&P1_26_gpio_pd_pin>; + pinctrl-4 = <&P1_26_gpio_input_pin>; + pinctrl-5 = <&P1_26_spi_cs_pin>; + pinctrl-6 = <&P1_26_can_pin>; + pinctrl-7 = <&P1_26_i2c_pin>; + pinctrl-8 = <&P1_26_pru_uart_pin>; + }; + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c */ + P1_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_28_default_pin>; + pinctrl-1 = <&P1_28_gpio_pin>; + pinctrl-2 = <&P1_28_gpio_pu_pin>; + pinctrl-3 = <&P1_28_gpio_pd_pin>; + pinctrl-4 = <&P1_28_gpio_input_pin>; + pinctrl-5 = <&P1_28_spi_cs_pin>; + pinctrl-6 = <&P1_28_can_pin>; + pinctrl-7 = <&P1_28_i2c_pin>; + pinctrl-8 = <&P1_28_pru_uart_pin>; + }; + + /* P1_29 (ZCZ ball A14) pruin */ + P1_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_29_default_pin>; + pinctrl-1 = <&P1_29_gpio_pin>; + pinctrl-2 = <&P1_29_gpio_pu_pin>; + pinctrl-3 = <&P1_29_gpio_pd_pin>; + pinctrl-4 = <&P1_29_gpio_input_pin>; + pinctrl-5 = <&P1_29_qep_pin>; + pinctrl-6 = <&P1_29_pruout_pin>; + pinctrl-7 = <&P1_29_pruin_pin>; + }; + + /* P1_30 (ZCZ ball E16) uart */ + P1_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_30_default_pin>; + pinctrl-1 = <&P1_30_gpio_pin>; + pinctrl-2 = <&P1_30_gpio_pu_pin>; + pinctrl-3 = <&P1_30_gpio_pd_pin>; + pinctrl-4 = <&P1_30_gpio_input_pin>; + pinctrl-5 = <&P1_30_spi_cs_pin>; + pinctrl-6 = <&P1_30_uart_pin>; + pinctrl-7 = <&P1_30_can_pin>; + pinctrl-8 = <&P1_30_i2c_pin>; + pinctrl-9 = <&P1_30_pruout_pin>; + pinctrl-10 = <&P1_30_pruin_pin>; + }; + + /* P1_31 (ZCZ ball B12) pruin */ + P1_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_31_default_pin>; + pinctrl-1 = <&P1_31_gpio_pin>; + pinctrl-2 = <&P1_31_gpio_pu_pin>; + pinctrl-3 = <&P1_31_gpio_pd_pin>; + pinctrl-4 = <&P1_31_gpio_input_pin>; + pinctrl-5 = <&P1_31_qep_pin>; + pinctrl-6 = <&P1_31_pruout_pin>; + pinctrl-7 = <&P1_31_pruin_pin>; + }; + + /* P1_32 (ZCZ ball E15) uart */ + P1_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_32_default_pin>; + pinctrl-1 = <&P1_32_gpio_pin>; + pinctrl-2 = <&P1_32_gpio_pu_pin>; + pinctrl-3 = <&P1_32_gpio_pd_pin>; + pinctrl-4 = <&P1_32_gpio_input_pin>; + pinctrl-5 = <&P1_32_spi_cs_pin>; + pinctrl-6 = <&P1_32_uart_pin>; + pinctrl-7 = <&P1_32_can_pin>; + pinctrl-8 = <&P1_32_i2c_pin>; + pinctrl-9 = <&P1_32_pruout_pin>; + pinctrl-10 = <&P1_32_pruin_pin>; + }; + + /* P1_33 (ZCZ ball B13) pruin */ + P1_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_33_default_pin>; + pinctrl-1 = <&P1_33_gpio_pin>; + pinctrl-2 = <&P1_33_gpio_pu_pin>; + pinctrl-3 = <&P1_33_gpio_pd_pin>; + pinctrl-4 = <&P1_33_gpio_input_pin>; + pinctrl-5 = <&P1_33_spi_pin>; + pinctrl-6 = <&P1_33_pwm_pin>; + pinctrl-7 = <&P1_33_pruout_pin>; + pinctrl-8 = <&P1_33_pruin_pin>; + }; + + /* P1_34 (ZCZ ball T11) */ + P1_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P1_34_default_pin>; + pinctrl-1 = <&P1_34_gpio_pin>; + pinctrl-2 = <&P1_34_gpio_pu_pin>; + pinctrl-3 = <&P1_34_gpio_pd_pin>; + pinctrl-4 = <&P1_34_gpio_input_pin>; + pinctrl-5 = <&P1_34_pwm_pin>; + }; + + /* P1_35 (ZCZ ball V5) pruin */ + P1_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_35_default_pin>; + pinctrl-1 = <&P1_35_gpio_pin>; + pinctrl-2 = <&P1_35_gpio_pu_pin>; + pinctrl-3 = <&P1_35_gpio_pd_pin>; + pinctrl-4 = <&P1_35_gpio_input_pin>; + pinctrl-5 = <&P1_35_pruout_pin>; + pinctrl-6 = <&P1_35_pruin_pin>; + }; + + /* P1_36 (ZCZ ball A13) pwm */ + P1_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_36_default_pin>; + pinctrl-1 = <&P1_36_gpio_pin>; + pinctrl-2 = <&P1_36_gpio_pu_pin>; + pinctrl-3 = <&P1_36_gpio_pd_pin>; + pinctrl-4 = <&P1_36_gpio_input_pin>; + pinctrl-5 = <&P1_36_spi_sclk_pin>; + pinctrl-6 = <&P1_36_pwm_pin>; + pinctrl-7 = <&P1_36_pruout_pin>; + pinctrl-8 = <&P1_36_pruin_pin>; + }; + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) pwm */ + P2_01_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_01_default_pin>; + pinctrl-1 = <&P2_01_gpio_pin>; + pinctrl-2 = <&P2_01_gpio_pu_pin>; + pinctrl-3 = <&P2_01_gpio_pd_pin>; + pinctrl-4 = <&P2_01_gpio_input_pin>; + pinctrl-5 = <&P2_01_pwm_pin>; + }; + + /* P2_02 (ZCZ ball V17) */ + P2_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_02_default_pin>; + pinctrl-1 = <&P2_02_gpio_pin>; + pinctrl-2 = <&P2_02_gpio_pu_pin>; + pinctrl-3 = <&P2_02_gpio_pd_pin>; + pinctrl-4 = <&P2_02_gpio_input_pin>; + }; + + /* P2_03 (ZCZ ball T10) */ + P2_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_03_default_pin>; + pinctrl-1 = <&P2_03_gpio_pin>; + pinctrl-2 = <&P2_03_gpio_pu_pin>; + pinctrl-3 = <&P2_03_gpio_pd_pin>; + pinctrl-4 = <&P2_03_gpio_input_pin>; + pinctrl-5 = <&P2_03_pwm_pin>; + }; + + /* P2_04 (ZCZ ball T16) */ + P2_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_04_default_pin>; + pinctrl-1 = <&P2_04_gpio_pin>; + pinctrl-2 = <&P2_04_gpio_pu_pin>; + pinctrl-3 = <&P2_04_gpio_pd_pin>; + pinctrl-4 = <&P2_04_gpio_input_pin>; + }; + + /* P2_05 (ZCZ ball T17) uart */ + P2_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_05_default_pin>; + pinctrl-1 = <&P2_05_gpio_pin>; + pinctrl-2 = <&P2_05_gpio_pu_pin>; + pinctrl-3 = <&P2_05_gpio_pd_pin>; + pinctrl-4 = <&P2_05_gpio_input_pin>; + pinctrl-5 = <&P2_05_uart_pin>; + }; + + /* P2_06 (ZCZ ball U16) */ + P2_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_06_default_pin>; + pinctrl-1 = <&P2_06_gpio_pin>; + pinctrl-2 = <&P2_06_gpio_pu_pin>; + pinctrl-3 = <&P2_06_gpio_pd_pin>; + pinctrl-4 = <&P2_06_gpio_input_pin>; + }; + + /* P2_07 (ZCZ ball U17) uart */ + P2_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_07_default_pin>; + pinctrl-1 = <&P2_07_gpio_pin>; + pinctrl-2 = <&P2_07_gpio_pu_pin>; + pinctrl-3 = <&P2_07_gpio_pd_pin>; + pinctrl-4 = <&P2_07_gpio_input_pin>; + pinctrl-5 = <&P2_07_uart_pin>; + }; + + /* P2_08 (ZCZ ball U18) */ + P2_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_08_default_pin>; + pinctrl-1 = <&P2_08_gpio_pin>; + pinctrl-2 = <&P2_08_gpio_pu_pin>; + pinctrl-3 = <&P2_08_gpio_pd_pin>; + pinctrl-4 = <&P2_08_gpio_input_pin>; + }; + + /* P2_09 (ZCZ ball D15) i2c */ + P2_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_09_default_pin>; + pinctrl-1 = <&P2_09_gpio_pin>; + pinctrl-2 = <&P2_09_gpio_pu_pin>; + pinctrl-3 = <&P2_09_gpio_pd_pin>; + pinctrl-4 = <&P2_09_gpio_input_pin>; + pinctrl-5 = <&P2_09_uart_pin>; + pinctrl-6 = <&P2_09_can_pin>; + pinctrl-7 = <&P2_09_i2c_pin>; + pinctrl-8 = <&P2_09_pru_uart_pin>; + pinctrl-9 = <&P2_09_pruin_pin>; + }; + + /* P2_10 (ZCZ ball R14) */ + P2_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P2_10_default_pin>; + pinctrl-1 = <&P2_10_gpio_pin>; + pinctrl-2 = <&P2_10_gpio_pu_pin>; + pinctrl-3 = <&P2_10_gpio_pd_pin>; + pinctrl-4 = <&P2_10_gpio_input_pin>; + pinctrl-5 = <&P2_10_qep_pin>; + }; + + /* P2_11 (ZCZ ball D16) i2c */ + P2_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_11_default_pin>; + pinctrl-1 = <&P2_11_gpio_pin>; + pinctrl-2 = <&P2_11_gpio_pu_pin>; + pinctrl-3 = <&P2_11_gpio_pd_pin>; + pinctrl-4 = <&P2_11_gpio_input_pin>; + pinctrl-5 = <&P2_11_uart_pin>; + pinctrl-6 = <&P2_11_can_pin>; + pinctrl-7 = <&P2_11_i2c_pin>; + pinctrl-8 = <&P2_11_pru_uart_pin>; + pinctrl-9 = <&P2_11_pruin_pin>; + }; + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) */ + P2_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_17_default_pin>; + pinctrl-1 = <&P2_17_gpio_pin>; + pinctrl-2 = <&P2_17_gpio_pu_pin>; + pinctrl-3 = <&P2_17_gpio_pd_pin>; + pinctrl-4 = <&P2_17_gpio_input_pin>; + }; + + /* P2_18 (ZCZ ball U13) */ + P2_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P2_18_default_pin>; + pinctrl-1 = <&P2_18_gpio_pin>; + pinctrl-2 = <&P2_18_gpio_pu_pin>; + pinctrl-3 = <&P2_18_gpio_pd_pin>; + pinctrl-4 = <&P2_18_gpio_input_pin>; + pinctrl-5 = <&P2_18_qep_pin>; + pinctrl-6 = <&P2_18_pru_ecap_pin>; + pinctrl-7 = <&P2_18_pruin_pin>; + }; + + /* P2_19 (ZCZ ball U12) */ + P2_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_19_default_pin>; + pinctrl-1 = <&P2_19_gpio_pin>; + pinctrl-2 = <&P2_19_gpio_pu_pin>; + pinctrl-3 = <&P2_19_gpio_pd_pin>; + pinctrl-4 = <&P2_19_gpio_input_pin>; + pinctrl-5 = <&P2_19_pwm_pin>; + }; + + /* P2_20 (ZCZ ball T13) */ + P2_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_20_default_pin>; + pinctrl-1 = <&P2_20_gpio_pin>; + pinctrl-2 = <&P2_20_gpio_pu_pin>; + pinctrl-3 = <&P2_20_gpio_pd_pin>; + pinctrl-4 = <&P2_20_gpio_input_pin>; + }; + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) */ + P2_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P2_22_default_pin>; + pinctrl-1 = <&P2_22_gpio_pin>; + pinctrl-2 = <&P2_22_gpio_pu_pin>; + pinctrl-3 = <&P2_22_gpio_pd_pin>; + pinctrl-4 = <&P2_22_gpio_input_pin>; + pinctrl-5 = <&P2_22_qep_pin>; + pinctrl-6 = <&P2_22_pruin_pin>; + }; + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) */ + P2_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_24_default_pin>; + pinctrl-1 = <&P2_24_gpio_pin>; + pinctrl-2 = <&P2_24_gpio_pu_pin>; + pinctrl-3 = <&P2_24_gpio_pd_pin>; + pinctrl-4 = <&P2_24_gpio_input_pin>; + pinctrl-5 = <&P2_24_qep_pin>; + pinctrl-6 = <&P2_24_pruout_pin>; + }; + + /* P2_25 (ZCZ ball E17) spi */ + P2_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c"; + pinctrl-0 = <&P2_25_default_pin>; + pinctrl-1 = <&P2_25_gpio_pin>; + pinctrl-2 = <&P2_25_gpio_pu_pin>; + pinctrl-3 = <&P2_25_gpio_pd_pin>; + pinctrl-4 = <&P2_25_gpio_input_pin>; + pinctrl-5 = <&P2_25_spi_pin>; + pinctrl-6 = <&P2_25_spi_cs_pin>; + pinctrl-7 = <&P2_25_uart_pin>; + pinctrl-8 = <&P2_25_can_pin>; + pinctrl-9 = <&P2_25_i2c_pin>; + }; + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi */ + P2_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c"; + pinctrl-0 = <&P2_27_default_pin>; + pinctrl-1 = <&P2_27_gpio_pin>; + pinctrl-2 = <&P2_27_gpio_pu_pin>; + pinctrl-3 = <&P2_27_gpio_pd_pin>; + pinctrl-4 = <&P2_27_gpio_input_pin>; + pinctrl-5 = <&P2_27_spi_pin>; + pinctrl-6 = <&P2_27_uart_pin>; + pinctrl-7 = <&P2_27_can_pin>; + pinctrl-8 = <&P2_27_i2c_pin>; + }; + + /* P2_28 (ZCZ ball D13) pruin */ + P2_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_28_default_pin>; + pinctrl-1 = <&P2_28_gpio_pin>; + pinctrl-2 = <&P2_28_gpio_pu_pin>; + pinctrl-3 = <&P2_28_gpio_pd_pin>; + pinctrl-4 = <&P2_28_gpio_input_pin>; + pinctrl-5 = <&P2_28_qep_pin>; + pinctrl-6 = <&P2_28_pruout_pin>; + pinctrl-7 = <&P2_28_pruin_pin>; + }; + + /* P2_29 (ZCZ ball C18) spi_sclk */ + P2_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P2_29_default_pin>; + pinctrl-1 = <&P2_29_gpio_pin>; + pinctrl-2 = <&P2_29_gpio_pu_pin>; + pinctrl-3 = <&P2_29_gpio_pd_pin>; + pinctrl-4 = <&P2_29_gpio_input_pin>; + pinctrl-5 = <&P2_29_spi_cs_pin>; + pinctrl-6 = <&P2_29_spi_sclk_pin>; + pinctrl-7 = <&P2_29_uart_pin>; + pinctrl-8 = <&P2_29_pwm_pin>; + pinctrl-9 = <&P2_29_pru_ecap_pin>; + }; + + /* P2_30 (ZCZ ball C12) pruin */ + P2_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_30_default_pin>; + pinctrl-1 = <&P2_30_gpio_pin>; + pinctrl-2 = <&P2_30_gpio_pu_pin>; + pinctrl-3 = <&P2_30_gpio_pd_pin>; + pinctrl-4 = <&P2_30_gpio_input_pin>; + pinctrl-5 = <&P2_30_spi_cs_pin>; + pinctrl-6 = <&P2_30_pwm_pin>; + pinctrl-7 = <&P2_30_pruout_pin>; + pinctrl-8 = <&P2_30_pruin_pin>; + }; + + /* P2_31 (ZCZ ball A15) spi_cs */ + P2_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin"; + pinctrl-0 = <&P2_31_default_pin>; + pinctrl-1 = <&P2_31_gpio_pin>; + pinctrl-2 = <&P2_31_gpio_pu_pin>; + pinctrl-3 = <&P2_31_gpio_pd_pin>; + pinctrl-4 = <&P2_31_gpio_input_pin>; + pinctrl-5 = <&P2_31_spi_cs_pin>; + pinctrl-6 = <&P2_31_pruin_pin>; + }; + + /* P2_32 (ZCZ ball D12) pruin */ + P2_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_32_default_pin>; + pinctrl-1 = <&P2_32_gpio_pin>; + pinctrl-2 = <&P2_32_gpio_pu_pin>; + pinctrl-3 = <&P2_32_gpio_pd_pin>; + pinctrl-4 = <&P2_32_gpio_input_pin>; + pinctrl-5 = <&P2_32_spi_pin>; + pinctrl-6 = <&P2_32_pwm_pin>; + pinctrl-7 = <&P2_32_pruout_pin>; + pinctrl-8 = <&P2_32_pruin_pin>; + }; + + /* P2_33 (ZCZ ball R12) */ + P2_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_33_default_pin>; + pinctrl-1 = <&P2_33_gpio_pin>; + pinctrl-2 = <&P2_33_gpio_pu_pin>; + pinctrl-3 = <&P2_33_gpio_pd_pin>; + pinctrl-4 = <&P2_33_gpio_input_pin>; + pinctrl-5 = <&P2_33_qep_pin>; + pinctrl-6 = <&P2_33_pruout_pin>; + }; + + /* P2_34 (ZCZ ball C13) pruin */ + P2_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_34_default_pin>; + pinctrl-1 = <&P2_34_gpio_pin>; + pinctrl-2 = <&P2_34_gpio_pu_pin>; + pinctrl-3 = <&P2_34_gpio_pd_pin>; + pinctrl-4 = <&P2_34_gpio_input_pin>; + pinctrl-5 = <&P2_34_qep_pin>; + pinctrl-6 = <&P2_34_pruout_pin>; + pinctrl-7 = <&P2_34_pruin_pin>; + }; + + /* P2_35 (ZCZ ball U5) gpio_input */ + P2_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P2_35_default_pin>; + pinctrl-1 = <&P2_35_gpio_pin>; + pinctrl-2 = <&P2_35_gpio_pu_pin>; + pinctrl-3 = <&P2_35_gpio_pd_pin>; + pinctrl-4 = <&P2_35_gpio_input_pin>; + pinctrl-5 = <&P2_35_pruout_pin>; + pinctrl-6 = <&P2_35_pruin_pin>; + }; + + /* P2_36 (ZCZ ball C9) AIN7 */ + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P1_02 { + gpio-name = "P1_02"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P1_04 { + gpio-name = "P1_04"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P1_06 { + gpio-name = "P1_06"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P1_08 { + gpio-name = "P1_08"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P1_10 { + gpio-name = "P1_10"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P1_12 { + gpio-name = "P1_12"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P1_20 { + gpio-name = "P1_20"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P1_26 { + gpio-name = "P1_26"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P1_28 { + gpio-name = "P1_28"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P1_29 { + gpio-name = "P1_29"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P1_30 { + gpio-name = "P1_30"; + gpio = <&gpio1 11 0>; + input; + dir-changeable; + }; + + P1_31 { + gpio-name = "P1_31"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + P1_32 { + gpio-name = "P1_32"; + gpio = <&gpio1 10 0>; + input; + dir-changeable; + }; + + P1_33 { + gpio-name = "P1_33"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P1_34 { + gpio-name = "P1_34"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P1_35 { + gpio-name = "P1_35"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P1_36 { + gpio-name = "P1_36"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P2_01 { + gpio-name = "P2_01"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P2_02 { + gpio-name = "P2_02"; + gpio = <&gpio1 27 0>; + input; + dir-changeable; + }; + + P2_03 { + gpio-name = "P2_03"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P2_04 { + gpio-name = "P2_04"; + gpio = <&gpio1 26 0>; + input; + dir-changeable; + }; + + P2_05 { + gpio-name = "P2_05"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P2_06 { + gpio-name = "P2_06"; + gpio = <&gpio1 25 0>; + input; + dir-changeable; + }; + + P2_07 { + gpio-name = "P2_07"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P2_08 { + gpio-name = "P2_08"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P2_09 { + gpio-name = "P2_09"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P2_10 { + gpio-name = "P2_10"; + gpio = <&gpio1 20 0>; + input; + dir-changeable; + }; + + P2_11 { + gpio-name = "P2_11"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P2_17 { + gpio-name = "P2_17"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P2_18 { + gpio-name = "P2_18"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P2_19 { + gpio-name = "P2_19"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P2_20 { + gpio-name = "P2_20"; + gpio = <&gpio2 0 0>; + input; + dir-changeable; + }; + + P2_22 { + gpio-name = "P2_22"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P2_24 { + gpio-name = "P2_24"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P2_25 { + gpio-name = "P2_25"; + gpio = <&gpio1 9 0>; + input; + dir-changeable; + }; + + P2_27 { + gpio-name = "P2_27"; + gpio = <&gpio1 8 0>; + input; + dir-changeable; + }; + + P2_28 { + gpio-name = "P2_28"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P2_29 { + gpio-name = "P2_29"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P2_30 { + gpio-name = "P2_30"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P2_31 { + gpio-name = "P2_31"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + + P2_32 { + gpio-name = "P2_32"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P2_33 { + gpio-name = "P2_33"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P2_34 { + gpio-name = "P2_34"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P2_35 { + gpio-name = "P2_35"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe.dts b/arch/arm/boot/dts/am335x-sancloud-bbe.dts index e5fdb7abb0d54..820ff38cdf00d 100644 --- a/arch/arm/boot/dts/am335x-sancloud-bbe.dts +++ b/arch/arm/boot/dts/am335x-sancloud-bbe.dts @@ -12,6 +12,11 @@ / { model = "SanCloud BeagleBone Enhanced"; compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-sancloud-bbe.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &am33xx_pinmux { diff --git a/arch/arm/boot/dts/am33xx-l4.dtsi b/arch/arm/boot/dts/am33xx-l4.dtsi index 54605e1bd164b..6a4520b99d301 100644 --- a/arch/arm/boot/dts/am33xx-l4.dtsi +++ b/arch/arm/boot/dts/am33xx-l4.dtsi @@ -158,6 +158,39 @@ #interrupt-cells = <2>; reg = <0x0 0x1000>; interrupts = <96>; + gpio-line-names = + "MDIO_DATA", // 0 + "MDIO_CLK", // 1 + "SPI0_SCLK", // 2 + "SPI0_D0", // 3 + "SPI0_D1", // 4 + "SPI0_CS0", // 5 + "SPI0_CS1", // 6 + "ECAP0_IN_PWM0_OUT", // 7 + "LCD_DATA12", // 8 + "LCD_DATA13", // 9 + "LCD_DATA14", // 10 + "LCD_DATA15", // 11 + "UART1_CTSN", // 12 + "UART1_RTSN", // 13 + "UART1_RXD", // 14 + "UART1_TXD", // 15 + "GMII1_TXD3", // 16 + "GMII1_TXD2", // 17 + "USB0_DRVVBUS", // 18 + "XDMA_EVENT_INTR0", // 19 + "XDMA_EVENT_INTR1", // 20 + "GMII1_TXD1", // 21 + "GPMC_AD8", // 22 + "GPMC_AD9", // 23 + "NC", // 24 + "NC", // 25 + "GPMC_AD10", // 26 + "GPMC_AD11", // 27 + "GMII1_TXD0", // 28 + "RMII1_REFCLK", // 29 + "GPMC_WAIT0", // 30 + "GPMC_WPN"; // 31 }; }; @@ -1415,6 +1448,39 @@ #interrupt-cells = <2>; reg = <0x0 0x1000>; interrupts = <98>; + gpio-line-names = + "GPMC_AD0", // 0 + "GPMC_AD1", // 1 + "GPMC_AD2", // 2 + "GPMC_AD3", // 3 + "GPMC_AD4", // 4 + "GPMC_AD5", // 5 + "GPMC_AD6", // 6 + "GPMC_AD7", // 7 + "UART0_CTSN", // 8 + "UART0_RTSN", // 9 + "UART0_RXD", // 10 + "UART0_TXD", // 11 + "GPMC_AD12", // 12 + "GPMC_AD13", // 13 + "GPMC_AD14", // 14 + "GPMC_AD15", // 15 + "GPMC_A0", // 16 + "GPMC_A1", // 17 + "GPMC_A2", // 18 + "GPMC_A3", // 19 + "GPMC_A4", // 20 + "GPMC_A5", // 21 + "GPMC_A6", // 22 + "GPMC_A7", // 23 + "GPMC_A8", // 24 + "GPMC_A9", // 25 + "GPMC_A10", // 26 + "GPMC_A11", // 27 + "GPMC_BE1N", // 28 + "GPMC_CSN0", // 29 + "GPMC_CSN1", // 30 + "GPMC_CSN2"; // 31 }; }; @@ -1827,6 +1893,39 @@ #interrupt-cells = <2>; reg = <0x0 0x1000>; interrupts = <32>; + gpio-line-names = + "GPMC_CSN3", // 0 + "GPMC_CLK", // 1 + "GPMC_ADVN_ALE", // 2 + "GPMC_OEN_REN", // 3 + "GPMC_WEN", // 4 + "GPMC_BE0N_CLE", // 5 + "LCD_DATA0", // 6 + "LCD_DATA1", // 7 + "LCD_DATA2", // 8 + "LCD_DATA3", // 9 + "LCD_DATA4", // 10 + "LCD_DATA5", // 11 + "LCD_DATA6", // 12 + "LCD_DATA7", // 13 + "LCD_DATA8", // 14 + "LCD_DATA9", // 15 + "LCD_DATA10", // 16 + "LCD_DATA11", // 17 + "GMII1_RXD3", // 18 + "GMII1_RXD2", // 19 + "GMII1_RXD1", // 20 + "GMII1_RXD0", // 21 + "LCD_VSYNC", // 22 + "LCD_HSYNC", // 23 + "LCD_PCLK", // 24 + "LCD_AC_BIAS_EN", // 25 + "MMC0_DAT3", // 26 + "MMC0_DAT2", // 27 + "MMC0_DAT1", // 28 + "MMC0_DAT0", // 29 + "MMC0_CLK", // 30 + "MMC0_CMD"; // 31 }; }; @@ -1861,6 +1960,39 @@ #interrupt-cells = <2>; reg = <0x0 0x1000>; interrupts = <62>; + gpio-line-names = + "GMII1_COL", // 0 + "GMII1_CRS", // 1 + "GMII1_RXER", // 2 + "GMII1_TXEN", // 3 + "GMII1_RXDV", // 4 + "I2C0_SDA", // 5 + "I2C0_SCL", // 6 + "EMU0", // 7 + "EMU1", // 8 + "GMII1_TXCLK", // 9 + "GMII1_RXCLK", // 10 + "NC", // 11 + "NC", // 12 + "USB1_DRVVBUS", // 13 + "MCASP0_ACLKX", // 14 + "MCASP0_FSX", // 15 + "MCASP0_AXR0", // 16 + "MCASP0_AHCLKR", // 17 + "MCASP0_ACLKR", // 18 + "MCASP0_FSR", // 19 + "MCASP0_AXR1", // 20 + "MCASP0_AHCLKX", // 21 + "NC", // 22 + "NC", // 23 + "NC", // 24 + "NC", // 25 + "NC", // 26 + "NC", // 27 + "NC", // 28 + "NC", // 29 + "NC", // 30 + "NC"; // 31 }; }; @@ -2030,6 +2162,16 @@ status = "disabled"; }; + eqep0: eqep@180 { + compatible = "ti,am33xx-eqep"; + reg = <0x180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupts = <79>; + interrupt-names = "eqep0"; + status = "disabled"; + }; + ehrpwm0: pwm@200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -2083,6 +2225,16 @@ status = "disabled"; }; + eqep1: eqep@180 { + compatible = "ti,am33xx-eqep"; + reg = <0x180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupts = <88>; + interrupt-names = "eqep1"; + status = "disabled"; + }; + ehrpwm1: pwm@200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -2136,6 +2288,16 @@ status = "disabled"; }; + eqep2: eqep@180 { + compatible = "ti,am33xx-eqep"; + reg = <0x180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupts = <89>; + interrupt-names = "eqep2"; + status = "disabled"; + }; + ehrpwm2: pwm@200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index dffa30fd5dd0d..c451fd0bb8be0 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -20,7 +20,7 @@ #size-cells = <1>; chosen { }; - aliases { + aliases: aliases { i2c0 = &i2c0; i2c1 = &i2c1; i2c2 = &i2c2; @@ -172,7 +172,7 @@ * for the moment, just use a fake OCP bus entry to represent * the whole bus hierarchy. */ - ocp { + ocp: ocp { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm/boot/dts/am5729-beagleboneai.dts b/arch/arm/boot/dts/am5729-beagleboneai.dts index a7c73acc7d4e5..c24471e231867 100644 --- a/arch/arm/boot/dts/am5729-beagleboneai.dts +++ b/arch/arm/boot/dts/am5729-beagleboneai.dts @@ -24,6 +24,8 @@ chosen { stdout-path = &uart1; + base_dtb = "am5729-beagleboneai.dts"; + base_dtb_timestamp = __TIMESTAMP__; }; memory@0 { @@ -66,6 +68,8 @@ leds { compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&led_pins_default>; led0 { label = "beaglebone:green:usr0"; @@ -149,18 +153,348 @@ emmc_pwrseq: emmc_pwrseq { compatible = "mmc-pwrseq-emmc"; reset-gpios = <&gpio5 7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pwrseq_pins_default>; }; brcmf_pwrseq: brcmf_pwrseq { compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&brcmf_pwrseq_pins_default>; reset-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>, /* BT-REG-ON */ <&gpio3 18 GPIO_ACTIVE_LOW>; /* WL-REG-ON */ }; + unused_pins: unused_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&unused_pins_default>; + }; + + cape_pins: cape_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&cape_pins_default>; + }; + extcon_usb1: extcon_usb1 { compatible = "linux,extcon-usb-gpio"; ti,enable-id-detection; id-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&extcon_usb1_pins_default>; + }; +}; + +&dra7_pmx_core { + extcon_usb1_pins_default: extcon_usb1_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3518, PIN_INPUT | MUX_MODE14) /* AG2: vin1a_d9.gpio3_13 - USR0 */ + >; + }; + + led_pins_default: led_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3528, PIN_OUTPUT | MUX_MODE14) /* AF6: vin1a_d13.gpio3_17 - USR0 */ + DRA7XX_CORE_IOPAD(0x36c0, PIN_OUTPUT | MUX_MODE14) /* J11: mcasp1_axr3.gpio5_5 - USR1 */ + DRA7XX_CORE_IOPAD(0x3520, PIN_OUTPUT | MUX_MODE14) /* AG5: vin1a_d12.gpio3_15 - USR2 */ + DRA7XX_CORE_IOPAD(0x351c, PIN_OUTPUT | MUX_MODE14) /* AG3: vin1a_d10.gpio3_14 - USR3 */ + DRA7XX_CORE_IOPAD(0x3500, PIN_OUTPUT | MUX_MODE14) /* AH6: vin1a_d3.gpio3_7 - USR4 */ + >; + }; + + microsd_extra_pins_default: microsd_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT_PULLUP | MUX_MODE14) /* W7: mmc1_sdcd.gpio6_27 - MMC1_SDCD */ + >; + }; + + ethphy_extra_pins_default: ethphy_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34c4, PIN_OUTPUT_PULLUP | MUX_MODE14) /* N1: gpmc_advn_ale.gpio2_23 - RGMII_RST */ + DRA7XX_CORE_IOPAD(0x364c, PIN_INPUT | MUX_MODE14) /* Y1: uart3_txd.gpio5_19 - MII0_INT */ + >; + }; + + emmc_pwrseq_pins_default: emmc_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36c8, PIN_OUTPUT_PULLUP | MUX_MODE14) /* F13: mcasp1_axr5.gpio5_7 - eMMC_RSTn */ + >; + }; + + brcmf_pwrseq_pins_default: brcmf_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x352c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AF3: vin1a_d14.gpio3_18 - WL_REG_ON */ + DRA7XX_CORE_IOPAD(0x353c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AE5: vin1a_d18.gpio3_22 - BT_REG_ON */ + >; + }; + + wifibt_extra_pins_default: wifibt_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3540, PIN_INPUT | MUX_MODE14) /* AE1: vin1a_d19.gpio3_23 - WL_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3450, PIN_INPUT | MUX_MODE8) /* P6: vin1a_d20.uart6_rxd - UART6_RXD */ + DRA7XX_CORE_IOPAD(0x3454, PIN_INPUT | MUX_MODE8) /* R9: vin1a_d21.uart6_txd - UART6_TXD */ + DRA7XX_CORE_IOPAD(0x3458, PIN_INPUT | MUX_MODE8) /* R5: vin1a_d22.uart6_ctsn - UART6_CTSN */ + DRA7XX_CORE_IOPAD(0x345c, PIN_INPUT | MUX_MODE8) /* P5: vin1a_d23.uart6_rtsn - UART6_RTSN */ + DRA7XX_CORE_IOPAD(0x3534, PIN_INPUT_PULLDOWN | MUX_MODE14) /* AF1: vin1a_d16.gpio3_20 - BT_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3538, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* AE3: vin1a_d6.gpio3_21 - BT_WAKE */ + >; + }; + + adc_pins_default: adc_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3550, PIN_OUTPUT | MUX_MODE14) /* AD3: vin1a_d23.gpio3_27 - VDD_ADC_SEL */ + DRA7XX_CORE_IOPAD(0x34DC, PIN_INPUT_PULLUP | MUX_MODE14) /* AG8: vin1a_clk0.gpio2_30 - INT_ADC */ + >; + }; + + pmic_pins_default: pmic_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3690, PIN_INPUT_PULLUP | MUX_MODE14) /* F21: gpio6_16.gpio6_16 - PMIC_INT */ + >; + }; + + hdmi_pins_default: hdmi_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* C25: i2c2_sda.hdmi1_ddc_scl - HDMI_DDC_SCL */ + DRA7XX_CORE_IOPAD(0x380C, PIN_INPUT | MUX_MODE1) /* F17: i2c2_scl.hdmi1_ddc_sda - HDMI_DDC_SDA */ + DRA7XX_CORE_IOPAD(0x37BC, PIN_INPUT | MUX_MODE6) /* B20: spi1_cs3.hdmi1_cec - HDMI_DDC_CEC */ +#if 0 + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE6) /* B21: spi1_cs2.hdmi1_hpd - HDMI_DDC_HPD */ +#else + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE14) /* B21: spi1_cs2.gpio7_12 - HDMI_DDC_HPD */ +#endif + >; + }; + + unused_pins_default: unused_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3400, MUX_MODE15) /* M6: sysboot0 */ + DRA7XX_CORE_IOPAD(0x3404, MUX_MODE15) /* M2: sysboot1 */ + DRA7XX_CORE_IOPAD(0x3408, MUX_MODE15) /* L5: sysboot2 */ + DRA7XX_CORE_IOPAD(0x340C, MUX_MODE15) /* M1: sysboot3 */ + DRA7XX_CORE_IOPAD(0x3410, MUX_MODE15) /* L6: sysboot4 */ + DRA7XX_CORE_IOPAD(0x3414, MUX_MODE15) /* L4: sysboot5 */ + DRA7XX_CORE_IOPAD(0x3418, MUX_MODE15) /* L3: sysboot6 */ + DRA7XX_CORE_IOPAD(0x341C, MUX_MODE15) /* L2: sysboot7 */ + DRA7XX_CORE_IOPAD(0x3420, MUX_MODE15) /* L1: sysboot8 */ + DRA7XX_CORE_IOPAD(0x3424, MUX_MODE15) /* K2: sysboot9 */ + DRA7XX_CORE_IOPAD(0x3428, MUX_MODE15) /* J1: sysboot10 */ + DRA7XX_CORE_IOPAD(0x342C, MUX_MODE15) /* J2: sysboot11 */ + DRA7XX_CORE_IOPAD(0x3430, MUX_MODE15) /* H1: sysboot12 */ + DRA7XX_CORE_IOPAD(0x3434, MUX_MODE15) /* J3: sysboot13 */ + DRA7XX_CORE_IOPAD(0x3438, MUX_MODE15) /* H2: sysboot14 */ + DRA7XX_CORE_IOPAD(0x343C, MUX_MODE15) /* H3: sysboot15 */ + + DRA7XX_CORE_IOPAD(0x3448, MUX_MODE15) /* T6: N/C */ + DRA7XX_CORE_IOPAD(0x344C, MUX_MODE15) /* T7: N/C */ + + DRA7XX_CORE_IOPAD(0x3460, MUX_MODE15) /* N7: N/C */ + DRA7XX_CORE_IOPAD(0x3464, MUX_MODE15) /* R4: N/C */ + DRA7XX_CORE_IOPAD(0x3468, MUX_MODE15) /* N9: N/C */ + DRA7XX_CORE_IOPAD(0x346C, MUX_MODE15) /* P9: N/C */ + DRA7XX_CORE_IOPAD(0x3470, MUX_MODE15) /* P4: N/C */ + DRA7XX_CORE_IOPAD(0x3474, MUX_MODE15) /* R3: N/C */ + DRA7XX_CORE_IOPAD(0x3478, MUX_MODE15) /* T2: N/C */ + DRA7XX_CORE_IOPAD(0x347C, MUX_MODE15) /* U2: N/C */ + DRA7XX_CORE_IOPAD(0x3480, MUX_MODE15) /* U1: N/C */ + DRA7XX_CORE_IOPAD(0x3484, MUX_MODE15) /* P3: N/C */ + DRA7XX_CORE_IOPAD(0x3488, MUX_MODE15) /* R2: N/C */ + + DRA7XX_CORE_IOPAD(0x34B4, MUX_MODE15) /* T1: N/C */ + DRA7XX_CORE_IOPAD(0x34B8, MUX_MODE15) /* P2: N/C */ + DRA7XX_CORE_IOPAD(0x34BC, MUX_MODE15) /* P1: N/C */ + DRA7XX_CORE_IOPAD(0x34C0, MUX_MODE15) /* P7: N/C */ + DRA7XX_CORE_IOPAD(0x34C4, MUX_MODE15) /* N1: N/C */ + DRA7XX_CORE_IOPAD(0x34C8, MUX_MODE15) /* M5: N/C */ + DRA7XX_CORE_IOPAD(0x34CC, MUX_MODE15) /* M3: N/C */ + DRA7XX_CORE_IOPAD(0x34D0, MUX_MODE15) /* N6: N/C */ + DRA7XX_CORE_IOPAD(0x34D4, MUX_MODE15) /* M4: N/C */ + DRA7XX_CORE_IOPAD(0x34D8, MUX_MODE15) /* N2: N/C */ + + DRA7XX_CORE_IOPAD(0x34E0, MUX_MODE15) /* AH7: N/C */ + + DRA7XX_CORE_IOPAD(0x34EC, MUX_MODE15) /* AE9: N/C */ + + DRA7XX_CORE_IOPAD(0x34F4, MUX_MODE15) /* AE8: N/C */ + DRA7XX_CORE_IOPAD(0x34F8, MUX_MODE15) /* AD8: N/C */ + DRA7XX_CORE_IOPAD(0x34FC, MUX_MODE15) /* AG7: N/C */ + + DRA7XX_CORE_IOPAD(0x3504, MUX_MODE15) /* AH3: N/C */ + DRA7XX_CORE_IOPAD(0x3508, MUX_MODE15) /* AH5: N/C */ + + DRA7XX_CORE_IOPAD(0x3524, MUX_MODE15) /* AF2: N/C */ + + DRA7XX_CORE_IOPAD(0x3530, MUX_MODE15) /* AF4: N/C */ + + DRA7XX_CORE_IOPAD(0x354C, MUX_MODE15) /* AD2: N/C */ + + DRA7XX_CORE_IOPAD(0x3554, MUX_MODE15) /* E1: N/C */ + DRA7XX_CORE_IOPAD(0x3558, MUX_MODE15) /* G2: N/C */ + DRA7XX_CORE_IOPAD(0x355C, MUX_MODE15) /* H7: N/C */ + DRA7XX_CORE_IOPAD(0x3560, MUX_MODE15) /* G1: N/C */ + + DRA7XX_CORE_IOPAD(0x356C, MUX_MODE15) /* F3: N/C */ + + DRA7XX_CORE_IOPAD(0x3574, MUX_MODE15) /* E2: N/C */ + + DRA7XX_CORE_IOPAD(0x3584, MUX_MODE15) /* E4: N/C */ + + DRA7XX_CORE_IOPAD(0x3594, MUX_MODE15) /* F6: N/C */ + + DRA7XX_CORE_IOPAD(0x35A4, MUX_MODE15) /* C4: N/C */ + DRA7XX_CORE_IOPAD(0x35A8, MUX_MODE15) /* B2: N/C */ + + DRA7XX_CORE_IOPAD(0x35C0, MUX_MODE15) /* B5: N/C */ + DRA7XX_CORE_IOPAD(0x35C4, MUX_MODE15) /* A4: N/C */ + + DRA7XX_CORE_IOPAD(0x35D0, MUX_MODE15) /* B11: N/C */ + + DRA7XX_CORE_IOPAD(0x3644, MUX_MODE15) /* U3: N/C */ + + DRA7XX_CORE_IOPAD(0x3680, MUX_MODE15) /* AB10: N/C */ + + DRA7XX_CORE_IOPAD(0x36BC, MUX_MODE15) /* G13: N/C */ + + DRA7XX_CORE_IOPAD(0x36C4, MUX_MODE15) /* E12: N/C */ + + DRA7XX_CORE_IOPAD(0x36CC, MUX_MODE15) /* C12: N/C */ + DRA7XX_CORE_IOPAD(0x36D0, MUX_MODE15) /* D12: N/C */ + + DRA7XX_CORE_IOPAD(0x36FC, MUX_MODE15) /* E15: N/C */ + DRA7XX_CORE_IOPAD(0x3700, MUX_MODE15) /* A20: N/C */ + + DRA7XX_CORE_IOPAD(0x370C, MUX_MODE15) /* C15: N/C */ + DRA7XX_CORE_IOPAD(0x3710, MUX_MODE15) /* A16: N/C */ + DRA7XX_CORE_IOPAD(0x3714, MUX_MODE15) /* D15: N/C */ + DRA7XX_CORE_IOPAD(0x3718, MUX_MODE15) /* B16: N/C */ + DRA7XX_CORE_IOPAD(0x371C, MUX_MODE15) /* B17: N/C */ + DRA7XX_CORE_IOPAD(0x3720, MUX_MODE15) /* A17: N/C */ + DRA7XX_CORE_IOPAD(0x3724, MUX_MODE15) /* B18: N/C */ + DRA7XX_CORE_IOPAD(0x3728, MUX_MODE15) /* F15: N/C */ + + DRA7XX_CORE_IOPAD(0x3744, MUX_MODE15) /* AA3: N/C */ + DRA7XX_CORE_IOPAD(0x3748, MUX_MODE15) /* AB9: N/C */ + DRA7XX_CORE_IOPAD(0x374C, MUX_MODE15) /* AB3: N/C */ + DRA7XX_CORE_IOPAD(0x3750, MUX_MODE15) /* AA4: N/C */ + + DRA7XX_CORE_IOPAD(0x3770, MUX_MODE15) /* Y9: N/C */ + DRA7XX_CORE_IOPAD(0x3774, MUX_MODE15) /* AC5: N/C */ + DRA7XX_CORE_IOPAD(0x3778, MUX_MODE15) /* AB4: N/C */ + + DRA7XX_CORE_IOPAD(0x37A4, MUX_MODE15) /* A25: N/C */ + DRA7XX_CORE_IOPAD(0x37A8, MUX_MODE15) /* F16: N/C */ + DRA7XX_CORE_IOPAD(0x37AC, MUX_MODE15) /* B25: N/C */ + DRA7XX_CORE_IOPAD(0x37B0, MUX_MODE15) /* A24: N/C */ + + DRA7XX_CORE_IOPAD(0x37D0, MUX_MODE15) /* G20: N/C */ + DRA7XX_CORE_IOPAD(0x37D4, MUX_MODE15) /* G19: N/C */ + + DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15) /* AD17: N/C */ + DRA7XX_CORE_IOPAD(0x381C, MUX_MODE15) /* AC17: N/C */ + DRA7XX_CORE_IOPAD(0x3820, MUX_MODE15) /* AB16: N/C */ + DRA7XX_CORE_IOPAD(0x3824, MUX_MODE15) /* AC16: N/C */ + >; + }; + + cape_pins_default: cape_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, MUX_MODE14) /* AB8: P8.3: mmc3_dat6.off */ + DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE14) /* AB5: P8.4: mmc3_dat7.off */ + DRA7XX_CORE_IOPAD(0x378C, MUX_MODE14) /* AC9: P8.5: mmc3_dat2.off */ + DRA7XX_CORE_IOPAD(0x3790, MUX_MODE14) /* AC3: P8.6: mmc3_dat3.off */ + DRA7XX_CORE_IOPAD(0x36EC, MUX_MODE14) /* G14: P8.7: mcasp1_axr14.off */ + DRA7XX_CORE_IOPAD(0x36F0, MUX_MODE14) /* F14: P8.8: mcasp1_axr15.off */ + DRA7XX_CORE_IOPAD(0x3698, MUX_MODE14) /* E17: P8.9: xref_clk1.off */ + DRA7XX_CORE_IOPAD(0x36E8, MUX_MODE14) /* A13: P8.10: mcasp1_axr13.off */ + DRA7XX_CORE_IOPAD(0x3510, MUX_MODE14) /* AH4: P8.11: vin1a_d7.off */ + DRA7XX_CORE_IOPAD(0x350C, MUX_MODE14) /* AG6: P8.12: vin1a_d6.off */ + DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE12) /* D3: P8.13: vin2a_d10.off */ + DRA7XX_CORE_IOPAD(0x3598, MUX_MODE14) /* D5: P8.14: vin2a_d12.off */ + DRA7XX_CORE_IOPAD(0x3570, MUX_MODE14) /* D1: P8.15a: vin2a_d2.off */ + DRA7XX_CORE_IOPAD(0x35B4, MUX_MODE13) /* A3: P8.15b: vin2a_d19.off */ + DRA7XX_CORE_IOPAD(0x35BC, MUX_MODE13) /* B4: P8.16: vin2a_d21.off */ + DRA7XX_CORE_IOPAD(0x3624, MUX_MODE14) /* A7: P8.17: vout1_d18.off */ + DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE12) /* F5: P8.18: vin2a_d8.off */ + DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE12) /* E6: P8.19: vin2a_d9.off */ + DRA7XX_CORE_IOPAD(0x3780, MUX_MODE14) /* AC4: P8.20: mmc3_cmd.off */ + DRA7XX_CORE_IOPAD(0x377C, MUX_MODE14) /* AD4: P8.21: mmc3_clk.off */ + DRA7XX_CORE_IOPAD(0x3798, MUX_MODE14) /* AD6: P8.22: mmc3_dat5.off */ + DRA7XX_CORE_IOPAD(0x3794, MUX_MODE14) /* AC8: P8.23: mmc3_dat4.off */ + DRA7XX_CORE_IOPAD(0x3788, MUX_MODE14) /* AC6: P8.24: mmc3_dat1.off */ + DRA7XX_CORE_IOPAD(0x3784, MUX_MODE14) /* AC7: P8.25: mmc3_dat0.off */ + DRA7XX_CORE_IOPAD(0x35B8, MUX_MODE13) /* B3: P8.26: vin2a_d20.off */ + DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE14) /* E11: P8.27a: vout1_vsync.off */ + DRA7XX_CORE_IOPAD(0x3628, MUX_MODE14) /* A8: P8.27b: vout1_d19.off */ + DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE14) /* D11: P8.28a: vout1_clk.off */ + DRA7XX_CORE_IOPAD(0x362C, MUX_MODE14) /* C9: P8.28b: vout1_d20.off */ + DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE14) /* C11: P8.29a: vout1_hsync.off */ + DRA7XX_CORE_IOPAD(0x3630, MUX_MODE14) /* A9: P8.29b: vout1_d21.off */ + DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE14) /* B10: P8.30a: vout1_de.off */ + DRA7XX_CORE_IOPAD(0x3634, MUX_MODE14) /* B9: P8.30b: vout1_d22.off */ + DRA7XX_CORE_IOPAD(0x3614, MUX_MODE14) /* C8: P8.31a: vout1_d14.off */ + DRA7XX_CORE_IOPAD(0x373C, MUX_MODE14) /* G16: P8.31b: mcasp4_axr0.off */ + DRA7XX_CORE_IOPAD(0x3618, MUX_MODE14) /* C7: P8.32a: vout1_d15.off */ + DRA7XX_CORE_IOPAD(0x3740, MUX_MODE14) /* D17: P8.32b: mcasp4_axr1.off */ + DRA7XX_CORE_IOPAD(0x3610, MUX_MODE14) /* C6: P8.33a: vout1_d13.off */ + DRA7XX_CORE_IOPAD(0x34E8, MUX_MODE14) /* AF9: P8.33b: vin1a_fld0.off */ + DRA7XX_CORE_IOPAD(0x3608, MUX_MODE14) /* D8: P8.34a: vout1_d11.off */ + DRA7XX_CORE_IOPAD(0x3564, MUX_MODE14) /* G6: P8.34b: vin2a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x360C, MUX_MODE14) /* A5: P8.35a: vout1_d12.off */ + DRA7XX_CORE_IOPAD(0x34E4, MUX_MODE14) /* AD9: P8.35b: vin1a_de0.off */ + DRA7XX_CORE_IOPAD(0x3604, MUX_MODE14) /* D7: P8.36a: vout1_d10.off */ + DRA7XX_CORE_IOPAD(0x3568, MUX_MODE14) /* F2: P8.36b: vin2a_d0.off */ + DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE14) /* E8: P8.37a: vout1_d8.off */ + DRA7XX_CORE_IOPAD(0x3738, MUX_MODE14) /* A21: P8.37b: mcasp4_fsx.off */ + DRA7XX_CORE_IOPAD(0x3600, MUX_MODE14) /* D9: P8.38a: vout1_d9.off */ + DRA7XX_CORE_IOPAD(0x3734, MUX_MODE14) /* C18: P8.38b: mcasp4_aclkx.off */ + DRA7XX_CORE_IOPAD(0x35F4, MUX_MODE14) /* F8: P8.39: vout1_d6.off */ + DRA7XX_CORE_IOPAD(0x35F8, MUX_MODE14) /* E7: P8.40: vout1_d7.off */ + DRA7XX_CORE_IOPAD(0x35EC, MUX_MODE14) /* E9: P8.41: vout1_d4.off */ + DRA7XX_CORE_IOPAD(0x35F0, MUX_MODE14) /* F9: P8.42: vout1_d5.off */ + DRA7XX_CORE_IOPAD(0x35E4, MUX_MODE14) /* F10: P8.43: vout1_d2.off */ + DRA7XX_CORE_IOPAD(0x35E8, MUX_MODE14) /* G11: P8.44: vout1_d3.off */ + DRA7XX_CORE_IOPAD(0x35DC, MUX_MODE14) /* F11: P8.45a: vout1_d0.off */ + DRA7XX_CORE_IOPAD(0x361C, MUX_MODE14) /* B7: P8.45b: vout1_d16.off */ + DRA7XX_CORE_IOPAD(0x35E0, MUX_MODE14) /* G10: P8.46a: vout1_d1.off */ + DRA7XX_CORE_IOPAD(0x3638, MUX_MODE14) /* A10: P8.46b: vout1_d23.off */ + DRA7XX_CORE_IOPAD(0x372C, MUX_MODE14) /* B19: P9.11a: mcasp3_axr0.off */ + DRA7XX_CORE_IOPAD(0x3620, MUX_MODE14) /* B8: P9.11b: vout1_d17.off */ + DRA7XX_CORE_IOPAD(0x36AC, MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.off */ + DRA7XX_CORE_IOPAD(0x3730, MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off */ + DRA7XX_CORE_IOPAD(0x35AC, MUX_MODE10) /* D6: P9.14: vin2a_d17.off */ + DRA7XX_CORE_IOPAD(0x3514, MUX_MODE14) /* AG4: P9.15: vin1a_d8.off */ + DRA7XX_CORE_IOPAD(0x35B0, MUX_MODE13) /* C5: P9.16: vin2a_d18.off */ + DRA7XX_CORE_IOPAD(0x37CC, MUX_MODE14) /* B24: P9.17a: spi2_cs0.off */ + DRA7XX_CORE_IOPAD(0x36B8, MUX_MODE14) /* F12: P9.17b: mcasp1_axr1.off */ + DRA7XX_CORE_IOPAD(0x37C8, MUX_MODE14) /* G17: P9.18a: spi2_d0.off */ + DRA7XX_CORE_IOPAD(0x36B4, MUX_MODE14) /* G12: P9.18b: mcasp1_axr0.off */ + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT_PULLUP | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl */ + DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT_PULLUP | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */ + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT_PULLUP | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda */ + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT_PULLUP | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */ + DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE14) /* AF8: P9.21a: vin1a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x37C4, MUX_MODE14) /* B22: P9.21b: spi2_d1.off */ + DRA7XX_CORE_IOPAD(0x369C, MUX_MODE14) /* B26: P9.22a: xref_clk2.off */ + DRA7XX_CORE_IOPAD(0x37C0, MUX_MODE14) /* A26: P9.22b: spi2_sclk.off */ + DRA7XX_CORE_IOPAD(0x37B4, MUX_MODE14) /* A22: P9.23: spi1_cs1.off */ + DRA7XX_CORE_IOPAD(0x368C, MUX_MODE14) /* F20: P9.24: gpio6_15.off */ + DRA7XX_CORE_IOPAD(0x3694, MUX_MODE14) /* D18: P9.25: xref_clk0.off */ + DRA7XX_CORE_IOPAD(0x3688, MUX_MODE14) /* E21: P9.26a: gpio6_14.off */ + DRA7XX_CORE_IOPAD(0x3544, MUX_MODE14) /* AE2: P9.26b: vin1a_d20.off */ + DRA7XX_CORE_IOPAD(0x35A0, MUX_MODE14) /* C3: P9.27a: vin2a_d14.off */ + DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE14) /* J14: P9.27b: mcasp1_fsr.off */ + DRA7XX_CORE_IOPAD(0x36E0, MUX_MODE14) /* A12: P9.28: mcasp1_axr11.off */ + DRA7XX_CORE_IOPAD(0x36D8, MUX_MODE14) /* A11: P9.29a: mcasp1_axr9.off */ + DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE14) /* D14: P9.29b: mcasp1_fsx.off */ + DRA7XX_CORE_IOPAD(0x36DC, MUX_MODE14) /* B13: P9.30: mcasp1_axr10.off */ + DRA7XX_CORE_IOPAD(0x36D4, MUX_MODE14) /* B12: P9.31a: mcasp1_axr8.off */ + DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE14) /* C14: P9.31b: mcasp1_aclkx.off */ + DRA7XX_CORE_IOPAD(0x36A0, MUX_MODE14) /* C23: P9.41a: xref_clk3.off */ + DRA7XX_CORE_IOPAD(0x3580, MUX_MODE14) /* C1: P9.41b: vin2a_d6.off */ + DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE14) /* E14: P9.42a: mcasp1_axr12.off */ + DRA7XX_CORE_IOPAD(0x359C, MUX_MODE14) /* C2: P9.42b: vin2a_d13.off */ + >; }; }; @@ -178,6 +512,9 @@ interrupt-parent = <&gpio6>; interrupts = <16 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_pins_default>; + #interrupt-cells = <2>; interrupt-controller; @@ -375,6 +712,11 @@ #gpio-cells = <2>; }; }; + + eeprom: eeprom@50 { + compatible = "atmel,24c32"; + reg = <0x50>; + }; }; &mcspi3 { @@ -398,7 +740,7 @@ }; &gpu { - status = "ok"; + status = "okay"; }; &uart1 { @@ -423,7 +765,7 @@ &cpsw_emac0 { phy-handle = <&phy0>; - phy-mode = "rgmii"; + phy-mode = "rgmii-rxid"; }; &mmc1 { @@ -466,6 +808,10 @@ /* DDR50: DDR up to 50 MHz (1.8 V signaling). */ status = "okay"; + pinctrl-names = "default", "hs"; + pinctrl-0 = <&mmc4_pins_default &wifibt_extra_pins_default>; + pinctrl-1 = <&mmc4_pins_hs &wifibt_extra_pins_default>; + ti,needs-special-reset; vmmc-supply = <&vdd_3v3>; cap-power-off-card; @@ -525,6 +871,8 @@ &hdmi { status = "okay"; vdda-supply = <&vdd_1v8_phy_ldo4>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_pins_default>; port { hdmi_out: endpoint { @@ -579,3 +927,14 @@ status = "okay"; clock-frequency = <100000>; }; + +/* thermal hacks */ +&cpu0_opp_table { + opp_slow-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <860000 850000 1100000>, + <860000 850000 1100000>; + opp-supported-hw = <0xFF 0x01>; + opp-suspend; + }; +}; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts index 39d1c4ff57498..da47c1389fdfa 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts @@ -7,6 +7,11 @@ / { model = "TI AM5728 BeagleBoard-X15 rev B1"; + + chosen { + base_dtb = "am57xx-beagle-x15-revb1.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts index 4187a9729f961..131773f978bfb 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts @@ -7,6 +7,11 @@ / { model = "TI AM5728 BeagleBoard-X15 rev C"; + + chosen { + base_dtb = "am57xx-beagle-x15-revc.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts index a5c24ed4d12f3..dbda396f92b47 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts @@ -8,6 +8,11 @@ / { /* NOTE: This describes the "original" pre-production A2 revision */ model = "TI AM5728 BeagleBoard-X15"; + + chosen { + base_dtb = "am57xx-beagle-x15.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/overlays/BBORG_FAN-A000.dts b/arch/arm/boot/dts/overlays/BBORG_FAN-A000.dts new file mode 100644 index 0000000000000..1b46782a76620 --- /dev/null +++ b/arch/arm/boot/dts/overlays/BBORG_FAN-A000.dts @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2020 Robert Nelson + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/dts-v1/; +/plugin/; + +/* +* Helper to show loaded overlays under: /proc/device-tree/chosen/overlays/ +*/ +&{/chosen} { + overlays { + BBORG_FAN-A000 = __TIMESTAMP__; + }; +}; + +&cpu0_opp_table { + opp_slow-400000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <1060000 850000 1150000>, + <1060000 850000 1150000>; + opp-supported-hw = <0xFF 0x01>; + opp-suspend; + }; +}; diff --git a/arch/arm/boot/dts/overlays/BONE-ADC.dts b/arch/arm/boot/dts/overlays/BONE-ADC.dts new file mode 100644 index 0000000000000..0e567dd15a2f8 --- /dev/null +++ b/arch/arm/boot/dts/overlays/BONE-ADC.dts @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2020 Deepak Khatri + * See Cape Interface Spec page for more info on Bone Buses + * https://elinux.org/Beagleboard:BeagleBone_cape_interface_spec + * + * Virtual cape for Bone ADC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/dts-v1/; +/plugin/; + +/* + * Helper to show loaded overlays under: /proc/device-tree/chosen/overlays/ + */ +&{/chosen} { + overlays { + BONE-ADC = __TIMESTAMP__; + }; +}; + +/* + * See these files for the phandles (&bone_*) and other bone bus nodes + * BeagleBoard-DeviceTrees/v4.19.x-ti-overlays/src/arm/bbai-bone-buses.dtsi + * BeagleBoard-DeviceTrees/v4.19.x-ti-overlays/src/arm/bbb-bone-buses.dtsi + */ +&bone_adc { + status = "okay"; +}; \ No newline at end of file diff --git a/arch/arm/boot/dts/overlays/Makefile b/arch/arm/boot/dts/overlays/Makefile new file mode 100644 index 0000000000000..bb2113de391a8 --- /dev/null +++ b/arch/arm/boot/dts/overlays/Makefile @@ -0,0 +1,11 @@ +# Overlays for the BeagleBone platform + +dtbo-$(CONFIG_SOC_AM33XX) += \ + BBORG_FAN-A000.dtbo \ + BONE-ADC.dtbo + +targets += dtbs dtbs_install +targets += $(dtbo-y) + +always := $(dtbo-y) +clean-files := *.dtbo diff --git a/include/dt-bindings/board/am335x-bbw-bbb-base.h b/include/dt-bindings/board/am335x-bbw-bbb-base.h new file mode 100644 index 0000000000000..35f6d57ef4923 --- /dev/null +++ b/include/dt-bindings/board/am335x-bbw-bbb-base.h @@ -0,0 +1,103 @@ +/* + * This header provides constants for bbw/bbb pinctrl bindings. + * + * Copyright (C) 2014 Robert Nelson + * + * Numbers Based on: https://github.com/derekmolloy/boneDeviceTree/tree/master/docs + */ + +#ifndef _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H +#define _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H + +#define BONE_P8_03 0x018 +#define BONE_P8_04 0x01C + +#define BONE_P8_05 0x008 +#define BONE_P8_06 0x00C +#define BONE_P8_07 0x090 +#define BONE_P8_08 0x094 + +#define BONE_P8_09 0x09C +#define BONE_P8_10 0x098 +#define BONE_P8_11 0x034 +#define BONE_P8_12 0x030 + +#define BONE_P8_13 0x024 +#define BONE_P8_14 0x028 +#define BONE_P8_15 0x03C +#define BONE_P8_16 0x038 + +#define BONE_P8_17 0x02C +#define BONE_P8_18 0x08C +#define BONE_P8_19 0x020 +#define BONE_P8_20 0x084 + +#define BONE_P8_21 0x080 +#define BONE_P8_22 0x014 +#define BONE_P8_23 0x010 +#define BONE_P8_24 0x004 + +#define BONE_P8_25 0x000 +#define BONE_P8_26 0x07C +#define BONE_P8_27 0x0E0 +#define BONE_P8_28 0x0E8 + +#define BONE_P8_29 0x0E4 +#define BONE_P8_30 0x0EC +#define BONE_P8_31 0x0D8 +#define BONE_P8_32 0x0DC + +#define BONE_P8_33 0x0D4 +#define BONE_P8_34 0x0CC +#define BONE_P8_35 0x0D0 +#define BONE_P8_36 0x0C8 + +#define BONE_P8_37 0x0C0 +#define BONE_P8_38 0x0C4 +#define BONE_P8_39 0x0B8 +#define BONE_P8_40 0x0BC + +#define BONE_P8_41 0x0B0 +#define BONE_P8_42 0x0B4 +#define BONE_P8_43 0x0A8 +#define BONE_P8_44 0x0AC + +#define BONE_P8_45 0x0A0 +#define BONE_P8_46 0x0A4 + +#define BONE_P9_11 0x070 +#define BONE_P9_12 0x078 + +#define BONE_P9_13 0x074 +#define BONE_P9_14 0x048 +#define BONE_P9_15 0x040 +#define BONE_P9_16 0x04C + +#define BONE_P9_17 0x15C +#define BONE_P9_18 0x158 +#define BONE_P9_19 0x17C +#define BONE_P9_20 0x178 + +#define BONE_P9_21 0x154 +#define BONE_P9_22 0x150 +#define BONE_P9_23 0x044 +#define BONE_P9_24 0x184 + +#define BONE_P9_25 0x1AC +#define BONE_P9_26 0x180 +#define BONE_P9_27 0x1A4 +#define BONE_P9_28 0x19C + +#define BONE_P9_29 0x194 +#define BONE_P9_30 0x198 +#define BONE_P9_31 0x190 + +/* Shared P21 of P11 */ +#define BONE_P9_41A 0x1B4 +#define BONE_P9_41B 0x1A8 + +/* Shared P22 of P11 */ +#define BONE_P9_42A 0x164 +#define BONE_P9_42B 0x1A0 + +#endif From 1428f1c1ed5f59b6ff590e6c4aea2be130b4352a Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:03:14 -0600 Subject: [PATCH 006/135] backports: greybus: from: linux.git Reference: v5.8.18 Signed-off-by: Robert Nelson --- drivers/greybus/Kconfig | 10 ++--- drivers/greybus/arpc.h | 2 +- drivers/greybus/connection.c | 3 -- drivers/staging/greybus/Kconfig | 40 +++++++++---------- drivers/staging/greybus/audio_apbridgea.h | 2 +- drivers/staging/greybus/gpio.c | 15 +++---- drivers/staging/greybus/hid.c | 3 +- drivers/staging/greybus/i2c.c | 16 -------- drivers/staging/greybus/loopback.c | 2 +- drivers/staging/greybus/raw.c | 2 +- drivers/staging/greybus/sdio.c | 1 - drivers/staging/greybus/tools/.gitignore | 1 + drivers/staging/greybus/tools/loopback_test.c | 3 +- drivers/staging/greybus/uart.c | 19 ++------- 14 files changed, 43 insertions(+), 76 deletions(-) diff --git a/drivers/greybus/Kconfig b/drivers/greybus/Kconfig index b84fcaf8b105c..78ba3c3083d54 100644 --- a/drivers/greybus/Kconfig +++ b/drivers/greybus/Kconfig @@ -2,8 +2,8 @@ menuconfig GREYBUS tristate "Greybus support" depends on SYSFS - ---help--- - This option enables the Greybus driver core. Greybus is an + help + This option enables the Greybus driver core. Greybus is a hardware protocol that was designed to provide Unipro with a sane application layer. It was originally designed for the ARA project, a module phone system, but has shown up in other @@ -12,7 +12,7 @@ menuconfig GREYBUS Say Y here to enable support for these types of drivers. - To compile this code as a module, chose M here: the module + To compile this code as a module, choose M here: the module will be called greybus.ko if GREYBUS @@ -20,12 +20,12 @@ if GREYBUS config GREYBUS_ES2 tristate "Greybus ES3 USB host controller" depends on USB - ---help--- + help Select this option if you have a Toshiba ES3 USB device that acts as a Greybus "host controller". This device is a bridge from a USB device to a Unipro network. - To compile this code as a module, chose M here: the module + To compile this code as a module, choose M here: the module will be called gb-es2.ko endif # GREYBUS diff --git a/drivers/greybus/arpc.h b/drivers/greybus/arpc.h index c8b83c5cfa79f..b9ea81b55b29a 100644 --- a/drivers/greybus/arpc.h +++ b/drivers/greybus/arpc.h @@ -21,7 +21,7 @@ struct arpc_request_message { __le16 id; /* RPC unique id */ __le16 size; /* Size in bytes of header + payload */ __u8 type; /* RPC type */ - __u8 data[0]; /* ARPC data */ + __u8 data[]; /* ARPC data */ } __packed; struct arpc_response_message { diff --git a/drivers/greybus/connection.c b/drivers/greybus/connection.c index fc8f57f97ce62..e3799a53a1930 100644 --- a/drivers/greybus/connection.c +++ b/drivers/greybus/connection.c @@ -361,9 +361,6 @@ static int gb_connection_hd_cport_quiesce(struct gb_connection *connection) if (connection->mode_switch) peer_space += sizeof(struct gb_operation_msg_hdr); - if (!hd->driver->cport_quiesce) - return 0; - ret = hd->driver->cport_quiesce(hd, connection->hd_cport_id, peer_space, GB_CONNECTION_CPORT_QUIESCE_TIMEOUT); diff --git a/drivers/staging/greybus/Kconfig b/drivers/staging/greybus/Kconfig index d4777f5a8b90d..9389e7a922fa6 100644 --- a/drivers/staging/greybus/Kconfig +++ b/drivers/staging/greybus/Kconfig @@ -4,7 +4,7 @@ if GREYBUS config GREYBUS_AUDIO tristate "Greybus Audio Class driver" depends on SOUND - ---help--- + help Select this option if you have a device that follows the Greybus Audio Class specification. @@ -13,7 +13,7 @@ config GREYBUS_AUDIO config GREYBUS_BOOTROM tristate "Greybus Bootrom Class driver" - ---help--- + help Select this option if you have a device that follows the Greybus Bootrom Class specification. @@ -23,7 +23,7 @@ config GREYBUS_BOOTROM config GREYBUS_CAMERA tristate "Greybus Camera Class driver" depends on MEDIA_SUPPORT && LEDS_CLASS_FLASH && BROKEN - ---help--- + help Select this option if you have a device that follows the Greybus Camera Class specification. @@ -33,7 +33,7 @@ config GREYBUS_CAMERA config GREYBUS_FIRMWARE tristate "Greybus Firmware Download Class driver" depends on SPI - ---help--- + help Select this option if you have a device that follows the Greybus Firmware Download Class specification. @@ -43,7 +43,7 @@ config GREYBUS_FIRMWARE config GREYBUS_HID tristate "Greybus HID Class driver" depends on HID && INPUT - ---help--- + help Select this option if you have a device that follows the Greybus HID Class specification. @@ -53,7 +53,7 @@ config GREYBUS_HID config GREYBUS_LIGHT tristate "Greybus LED Class driver" depends on LEDS_CLASS - ---help--- + help Select this option if you have a device that follows the Greybus LED Class specification. @@ -62,7 +62,7 @@ config GREYBUS_LIGHT config GREYBUS_LOG tristate "Greybus Debug Log Class driver" - ---help--- + help Select this option if you have a device that follows the Greybus Debug Log Class specification. @@ -71,7 +71,7 @@ config GREYBUS_LOG config GREYBUS_LOOPBACK tristate "Greybus Loopback Class driver" - ---help--- + help Select this option if you have a device that follows the Greybus Debug Log Class specification. @@ -81,7 +81,7 @@ config GREYBUS_LOOPBACK config GREYBUS_POWER tristate "Greybus Powersupply Class driver" depends on POWER_SUPPLY - ---help--- + help Select this option if you have a device that follows the Greybus Powersupply Class specification. @@ -90,7 +90,7 @@ config GREYBUS_POWER config GREYBUS_RAW tristate "Greybus Raw Class driver" - ---help--- + help Select this option if you have a device that follows the Greybus Raw Class specification. @@ -99,7 +99,7 @@ config GREYBUS_RAW config GREYBUS_VIBRATOR tristate "Greybus Vibrator Motor Class driver" - ---help--- + help Select this option if you have a device that follows the Greybus Vibrator Motor Class specification. @@ -108,7 +108,7 @@ config GREYBUS_VIBRATOR menuconfig GREYBUS_BRIDGED_PHY tristate "Greybus Bridged PHY Class drivers" - ---help--- + help Select this option to pick from a variety of Greybus Bridged PHY class drivers. These drivers emulate a number of different "traditional" busses by tunneling them over Greybus. @@ -123,7 +123,7 @@ config GREYBUS_GPIO tristate "Greybus GPIO Bridged PHY driver" depends on GPIOLIB select GPIOLIB_IRQCHIP - ---help--- + help Select this option if you have a device that follows the Greybus GPIO Bridged PHY Class specification. @@ -133,7 +133,7 @@ config GREYBUS_GPIO config GREYBUS_I2C tristate "Greybus I2C Bridged PHY driver" depends on I2C - ---help--- + help Select this option if you have a device that follows the Greybus I2C Bridged PHY Class specification. @@ -143,7 +143,7 @@ config GREYBUS_I2C config GREYBUS_PWM tristate "Greybus PWM Bridged PHY driver" depends on PWM - ---help--- + help Select this option if you have a device that follows the Greybus PWM Bridged PHY Class specification. @@ -153,7 +153,7 @@ config GREYBUS_PWM config GREYBUS_SDIO tristate "Greybus SDIO Bridged PHY driver" depends on MMC - ---help--- + help Select this option if you have a device that follows the Greybus SDIO Bridged PHY Class specification. @@ -163,7 +163,7 @@ config GREYBUS_SDIO config GREYBUS_SPI tristate "Greybus SPI Bridged PHY driver" depends on SPI - ---help--- + help Select this option if you have a device that follows the Greybus SPI Bridged PHY Class specification. @@ -173,7 +173,7 @@ config GREYBUS_SPI config GREYBUS_UART tristate "Greybus UART Bridged PHY driver" depends on TTY - ---help--- + help Select this option if you have a device that follows the Greybus UART Bridged PHY Class specification. @@ -183,7 +183,7 @@ config GREYBUS_UART config GREYBUS_USB tristate "Greybus USB Host Bridged PHY driver" depends on USB - ---help--- + help Select this option if you have a device that follows the Greybus USB Host Bridged PHY Class specification. @@ -195,7 +195,7 @@ endif # GREYBUS_BRIDGED_PHY config GREYBUS_ARCHE tristate "Greybus Arche Platform driver" depends on USB_HSIC_USB3613 || COMPILE_TEST - ---help--- + help Select this option if you have an Arche device. To compile this code as a module, chose M here: the module diff --git a/drivers/staging/greybus/audio_apbridgea.h b/drivers/staging/greybus/audio_apbridgea.h index 3f1f4dd2c61a3..efec0f815efd1 100644 --- a/drivers/staging/greybus/audio_apbridgea.h +++ b/drivers/staging/greybus/audio_apbridgea.h @@ -65,7 +65,7 @@ struct audio_apbridgea_hdr { __u8 type; __le16 i2s_port; - __u8 data[0]; + __u8 data[]; } __packed; struct audio_apbridgea_set_config_request { diff --git a/drivers/staging/greybus/gpio.c b/drivers/staging/greybus/gpio.c index 1ff34abd5692e..36d99f9e419ef 100644 --- a/drivers/staging/greybus/gpio.c +++ b/drivers/staging/greybus/gpio.c @@ -364,8 +364,7 @@ static int gb_gpio_request_handler(struct gb_operation *op) struct gb_message *request; struct gb_gpio_irq_event_request *event; u8 type = op->type; - int irq; - struct irq_desc *desc; + int irq, ret; if (type != GB_GPIO_TYPE_IRQ_EVENT) { dev_err(dev, "unsupported unsolicited request: %u\n", type); @@ -391,17 +390,15 @@ static int gb_gpio_request_handler(struct gb_operation *op) dev_err(dev, "failed to find IRQ\n"); return -EINVAL; } - desc = irq_to_desc(irq); - if (!desc) { - dev_err(dev, "failed to look up irq\n"); - return -EINVAL; - } local_irq_disable(); - generic_handle_irq_desc(desc); + ret = generic_handle_irq(irq); local_irq_enable(); - return 0; + if (ret) + dev_err(dev, "failed to invoke irq handler\n"); + + return ret; } static int gb_gpio_request(struct gpio_chip *chip, unsigned int offset) diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c index 04bfd91105025..ed706f39e87a1 100644 --- a/drivers/staging/greybus/hid.c +++ b/drivers/staging/greybus/hid.c @@ -290,9 +290,8 @@ static int gb_hid_parse(struct hid_device *hid) } rdesc = kzalloc(rsize, GFP_KERNEL); - if (!rdesc) { + if (!rdesc) return -ENOMEM; - } ret = gb_hid_get_report_desc(ghid, rdesc); if (ret) { diff --git a/drivers/staging/greybus/i2c.c b/drivers/staging/greybus/i2c.c index ab06fc3b9e7e4..de2f6516da095 100644 --- a/drivers/staging/greybus/i2c.c +++ b/drivers/staging/greybus/i2c.c @@ -215,20 +215,6 @@ static int gb_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, return gb_i2c_transfer_operation(gb_i2c_dev, msgs, msg_count); } -#if 0 -/* Later */ -static int gb_i2c_smbus_xfer(struct i2c_adapter *adap, - u16 addr, unsigned short flags, char read_write, - u8 command, int size, union i2c_smbus_data *data) -{ - struct gb_i2c_device *gb_i2c_dev; - - gb_i2c_dev = i2c_get_adapdata(adap); - - return 0; -} -#endif - static u32 gb_i2c_functionality(struct i2c_adapter *adap) { struct gb_i2c_device *gb_i2c_dev = i2c_get_adapdata(adap); @@ -238,7 +224,6 @@ static u32 gb_i2c_functionality(struct i2c_adapter *adap) static const struct i2c_algorithm gb_i2c_algorithm = { .master_xfer = gb_i2c_master_xfer, - /* .smbus_xfer = gb_i2c_smbus_xfer, */ .functionality = gb_i2c_functionality, }; @@ -281,7 +266,6 @@ static int gb_i2c_probe(struct gbphy_device *gbphy_dev, adapter->owner = THIS_MODULE; adapter->class = I2C_CLASS_HWMON | I2C_CLASS_SPD; adapter->algo = &gb_i2c_algorithm; - /* adapter->algo_data = what? */ adapter->dev.parent = &gbphy_dev->dev; snprintf(adapter->name, sizeof(adapter->name), "Greybus i2c adapter"); diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c index 583d9708a191c..2471448ba42ab 100644 --- a/drivers/staging/greybus/loopback.c +++ b/drivers/staging/greybus/loopback.c @@ -135,7 +135,7 @@ static ssize_t name##_##field##_show(struct device *dev, \ char *buf) \ { \ struct gb_loopback *gb = dev_get_drvdata(dev); \ - /* Report 0 for min and max if no transfer successed */ \ + /* Report 0 for min and max if no transfer succeeded */ \ if (!gb->requests_completed) \ return sprintf(buf, "0\n"); \ return sprintf(buf, "%" #type "\n", gb->name.field); \ diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c index 64a17dfe3b6ee..2a375f407d389 100644 --- a/drivers/staging/greybus/raw.c +++ b/drivers/staging/greybus/raw.c @@ -29,7 +29,7 @@ struct gb_raw { struct raw_data { struct list_head entry; u32 len; - u8 data[0]; + u8 data[]; }; static struct class *raw_class; diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c index c4b16bb5c1a4f..0939f4a4c963b 100644 --- a/drivers/staging/greybus/sdio.c +++ b/drivers/staging/greybus/sdio.c @@ -67,7 +67,6 @@ static void _gb_sdio_set_host_caps(struct gb_sdio_host *host, u32 r) ((r & GB_SDIO_CAP_8_BIT_DATA) ? MMC_CAP_8_BIT_DATA : 0) | ((r & GB_SDIO_CAP_MMC_HS) ? MMC_CAP_MMC_HIGHSPEED : 0) | ((r & GB_SDIO_CAP_SD_HS) ? MMC_CAP_SD_HIGHSPEED : 0) | - ((r & GB_SDIO_CAP_ERASE) ? MMC_CAP_ERASE : 0) | ((r & GB_SDIO_CAP_1_2V_DDR) ? MMC_CAP_1_2V_DDR : 0) | ((r & GB_SDIO_CAP_1_8V_DDR) ? MMC_CAP_1_8V_DDR : 0) | ((r & GB_SDIO_CAP_POWER_OFF_CARD) ? MMC_CAP_POWER_OFF_CARD : 0) | diff --git a/drivers/staging/greybus/tools/.gitignore b/drivers/staging/greybus/tools/.gitignore index 023654c83068f..1fd364aba7747 100644 --- a/drivers/staging/greybus/tools/.gitignore +++ b/drivers/staging/greybus/tools/.gitignore @@ -1 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only loopback_test diff --git a/drivers/staging/greybus/tools/loopback_test.c b/drivers/staging/greybus/tools/loopback_test.c index 69c6dce9be314..867bf289df2e4 100644 --- a/drivers/staging/greybus/tools/loopback_test.c +++ b/drivers/staging/greybus/tools/loopback_test.c @@ -802,8 +802,9 @@ static void prepare_devices(struct loopback_test *t) write_sysfs_val(t->devices[i].sysfs_entry, "outstanding_operations_max", t->async_outstanding_operations); - } else + } else { write_sysfs_val(t->devices[i].sysfs_entry, "async", 0); + } } } diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index 4ffb334cd5cde..607378bfebb7e 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -40,14 +40,6 @@ #define GB_UART_FIRMWARE_CREDITS 4096 #define GB_UART_CREDIT_WAIT_TIMEOUT_MSEC 10000 -struct gb_tty_line_coding { - __le32 rate; - __u8 format; - __u8 parity; - __u8 data_bits; - __u8 flow_control; -}; - struct gb_tty { struct gbphy_device *gbphy_dev; struct tty_port port; @@ -66,7 +58,7 @@ struct gb_tty { struct mutex mutex; u8 ctrlin; /* input control lines */ u8 ctrlout; /* output control lines */ - struct gb_tty_line_coding line_coding; + struct gb_uart_set_line_coding_request line_coding; struct work_struct tx_work; struct kfifo write_fifo; bool close_pending; @@ -288,12 +280,9 @@ static void gb_uart_tx_write_work(struct work_struct *work) static int send_line_coding(struct gb_tty *tty) { - struct gb_uart_set_line_coding_request request; - - memcpy(&request, &tty->line_coding, - sizeof(tty->line_coding)); return gb_operation_sync(tty->connection, GB_UART_TYPE_SET_LINE_CODING, - &request, sizeof(request), NULL, 0); + &tty->line_coding, sizeof(tty->line_coding), + NULL, 0); } static int send_control(struct gb_tty *gb_tty, u8 control) @@ -493,9 +482,9 @@ static int gb_tty_break_ctl(struct tty_struct *tty, int state) static void gb_tty_set_termios(struct tty_struct *tty, struct ktermios *termios_old) { + struct gb_uart_set_line_coding_request newline; struct gb_tty *gb_tty = tty->driver_data; struct ktermios *termios = &tty->termios; - struct gb_tty_line_coding newline; u8 newctrl = gb_tty->ctrlout; newline.rate = cpu_to_le32(tty_get_baud_rate(tty)); From 112a9ca5dc55e17fcca5e466e16bbc941d6b0054 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:04:22 -0600 Subject: [PATCH 007/135] backports: exfat: from: linux.git Reference: v5.6.19 Signed-off-by: Robert Nelson --- drivers/staging/Makefile | 2 +- drivers/staging/exfat/Kconfig | 35 +- drivers/staging/exfat/Makefile | 2 +- drivers/staging/exfat/TODO | 61 +- drivers/staging/exfat/exfat.h | 277 +--- drivers/staging/exfat/exfat_blkdev.c | 44 +- drivers/staging/exfat/exfat_cache.c | 303 +--- drivers/staging/exfat/exfat_core.c | 2067 ++++++-------------------- drivers/staging/exfat/exfat_nls.c | 192 --- drivers/staging/exfat/exfat_super.c | 1047 ++++++------- 10 files changed, 1142 insertions(+), 2888 deletions(-) diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index a90f9b308c8de..7410cec2ae3d8 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -51,5 +51,5 @@ obj-$(CONFIG_KPC2000) += kpc2000/ obj-$(CONFIG_ISDN_CAPI) += isdn/ obj-$(CONFIG_UWB) += uwb/ obj-$(CONFIG_USB_WUSB) += wusbcore/ -obj-$(CONFIG_EXFAT_FS) += exfat/ +obj-$(CONFIG_STAGING_EXFAT_FS) += exfat/ obj-$(CONFIG_QLGE) += qlge/ diff --git a/drivers/staging/exfat/Kconfig b/drivers/staging/exfat/Kconfig index ce32dfe33becd..292a19dfcaf51 100644 --- a/drivers/staging/exfat/Kconfig +++ b/drivers/staging/exfat/Kconfig @@ -1,50 +1,41 @@ # SPDX-License-Identifier: GPL-2.0 -config EXFAT_FS +config STAGING_EXFAT_FS tristate "exFAT fs support" depends on BLOCK select NLS help This adds support for the exFAT file system. -config EXFAT_DONT_MOUNT_VFAT - bool "Prohibit mounting of fat/vfat filesystems by exFAT" - depends on EXFAT_FS - default y - help - By default, the exFAT driver will only mount exFAT filesystems, and refuse - to mount fat/vfat filesystems. Set this to 'n' to allow the exFAT driver - to mount these filesystems. - -config EXFAT_DISCARD +config STAGING_EXFAT_DISCARD bool "enable discard support" - depends on EXFAT_FS + depends on STAGING_EXFAT_FS default y -config EXFAT_DELAYED_SYNC +config STAGING_EXFAT_DELAYED_SYNC bool "enable delayed sync" - depends on EXFAT_FS + depends on STAGING_EXFAT_FS default n -config EXFAT_KERNEL_DEBUG +config STAGING_EXFAT_KERNEL_DEBUG bool "enable kernel debug features via ioctl" - depends on EXFAT_FS + depends on STAGING_EXFAT_FS default n -config EXFAT_DEBUG_MSG +config STAGING_EXFAT_DEBUG_MSG bool "print debug messages" - depends on EXFAT_FS + depends on STAGING_EXFAT_FS default n -config EXFAT_DEFAULT_CODEPAGE +config STAGING_EXFAT_DEFAULT_CODEPAGE int "Default codepage for exFAT" default 437 - depends on EXFAT_FS + depends on STAGING_EXFAT_FS help This option should be set to the codepage of your exFAT filesystems. -config EXFAT_DEFAULT_IOCHARSET +config STAGING_EXFAT_DEFAULT_IOCHARSET string "Default iocharset for exFAT" default "utf8" - depends on EXFAT_FS + depends on STAGING_EXFAT_FS help Set this to the default input/output character set you'd like exFAT to use. diff --git a/drivers/staging/exfat/Makefile b/drivers/staging/exfat/Makefile index 6c90aec83febf..057556eeca0c3 100644 --- a/drivers/staging/exfat/Makefile +++ b/drivers/staging/exfat/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later -obj-$(CONFIG_EXFAT_FS) += exfat.o +obj-$(CONFIG_STAGING_EXFAT_FS) += exfat.o exfat-y := exfat_core.o \ exfat_super.o \ diff --git a/drivers/staging/exfat/TODO b/drivers/staging/exfat/TODO index a3eb282f9efc2..a283ce534cf47 100644 --- a/drivers/staging/exfat/TODO +++ b/drivers/staging/exfat/TODO @@ -1,8 +1,22 @@ +A laundry list of things that need looking at, most of which will +require more work than the average checkpatch cleanup... + +Note that some of these entries may not be bugs - they're things +that need to be looked at, and *possibly* fixed. + +Clean up the ffsCamelCase function names. + +Fix (thing)->flags to not use magic numbers - multiple offenders + +Sort out all the s32/u32/u8 nonsense - most of these should be plain int. + exfat_core.c - ffsReadFile - the goto err_out seem to leak a brelse(). same for ffsWriteFile. -exfat_core.c - fs_sync(sb,0) all over the place looks fishy as hell. -There's only one place that calls it with a non-zero argument. +All the calls to fs_sync() need to be looked at, particularly in the +context of EXFAT_DELAYED_SYNC. Currently, if that's defined, we only +flush to disk when sync() gets called. We should be doing at least +metadata flushes at appropriate times. ffsTruncateFile - if (old_size <= new_size) { That doesn't look right. How did it ever work? Are they relying on lazy @@ -10,3 +24,46 @@ block allocation when actual writes happen? If nothing else, it never does the 'fid->size = new_size' and do the inode update.... ffsSetAttr() is just dangling in the breeze, not wired up at all... + +Convert global mutexes to a per-superblock mutex. + +Right now, we load exactly one UTF-8 table. Check to see +if that plays nice with different codepage and iocharset values +for simultanous mounts of different devices + +exfat_rmdir() checks for -EBUSY but ffsRemoveDir() doesn't return it. +In fact, there's a complete lack of -EBUSY testing anywhere. + +There's probably a few missing checks for -EEXIST + +check return codes of sync_dirty_buffer() + +Why is remove_file doing a num_entries++?? + +Double check a lot of can't-happen parameter checks (for null pointers for +things that have only one call site and can't pass a null, etc). + +All the DEBUG stuff can probably be tossed, including the ioctl(). Either +that, or convert to a proper fault-injection system. + +exfat_remount does exactly one thing. Fix to actually deal with remount +options, particularly handling R/O correctly. For that matter, allow +R/O mounts in the first place. + +Figure out why the VFAT code used multi_sector_(read|write) but the +exfat code doesn't use it. The difference matters on SSDs with wear leveling. + +exfat_fat_sync(), exfat_buf_sync(), and sync_alloc_bitmap() +aren't called anyplace.... + +Create helper function for exfat_set_entry_time() and exfat_set_entry_type() +because it's sort of ugly to be calling the same functionn directly and +other code calling through the fs_func struc ponters... + +clean up the remaining vol_type checks, which are of two types: +some are ?: operators with magic numbers, and the rest are places +where we're doing stuff with '.' and '..'. + +Patches to: + Greg Kroah-Hartman + Valdis Kletnieks diff --git a/drivers/staging/exfat/exfat.h b/drivers/staging/exfat/exfat.h index 4973c9edc26ec..4d87360fab351 100644 --- a/drivers/staging/exfat/exfat.h +++ b/drivers/staging/exfat/exfat.h @@ -9,7 +9,7 @@ #include #include -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG /* For Debugging Purpose */ /* IOCTL code 'f' used by * - file systems typically #0~0x1F @@ -22,14 +22,16 @@ #define EXFAT_DEBUGFLAGS_INVALID_UMOUNT 0x01 #define EXFAT_DEBUGFLAGS_ERROR_RW 0x02 -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ -#ifdef CONFIG_EXFAT_DEBUG_MSG +#ifdef CONFIG_STAGING_EXFAT_DEBUG_MSG #define DEBUG 1 #else #undef DEBUG #endif +#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ + #define DENTRY_SIZE 32 /* dir entry size */ #define DENTRY_SIZE_BITS 5 @@ -206,28 +208,6 @@ static inline u16 get_row_index(u16 i) #define FM_REGULAR 0x00 #define FM_SYMLINK 0x40 -/* return values */ -#define FFS_SUCCESS 0 -#define FFS_MEDIAERR 1 -#define FFS_FORMATERR 2 -#define FFS_MOUNTED 3 -#define FFS_NOTMOUNTED 4 -#define FFS_ALIGNMENTERR 5 -#define FFS_SEMAPHOREERR 6 -#define FFS_INVALIDPATH 7 -#define FFS_INVALIDFID 8 -#define FFS_NOTFOUND 9 -#define FFS_FILEEXIST 10 -#define FFS_PERMISSIONERR 11 -#define FFS_NOTOPENED 12 -#define FFS_MAXOPENED 13 -#define FFS_FULL 14 -#define FFS_EOF 15 -#define FFS_DIRBUSY 16 -#define FFS_MEMORYERR 17 -#define FFS_NAMETOOLONG 18 -#define FFS_ERROR 19 - #define NUM_UPCASE 2918 #define DOS_CUR_DIR_NAME ". " @@ -536,49 +516,6 @@ struct buf_cache_t { struct buffer_head *buf_bh; }; -struct fs_func { - s32 (*alloc_cluster)(struct super_block *sb, s32 num_alloc, - struct chain_t *p_chain); - void (*free_cluster)(struct super_block *sb, struct chain_t *p_chain, - s32 do_relse); - s32 (*count_used_clusters)(struct super_block *sb); - - s32 (*init_dir_entry)(struct super_block *sb, struct chain_t *p_dir, - s32 entry, u32 type, u32 start_clu, u64 size); - s32 (*init_ext_entry)(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 num_entries, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname); - s32 (*find_dir_entry)(struct super_block *sb, struct chain_t *p_dir, - struct uni_name_t *p_uniname, s32 num_entries, - struct dos_name_t *p_dosname, u32 type); - void (*delete_dir_entry)(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - s32 offset, s32 num_entries); - void (*get_uni_name_from_ext_entry)(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - u16 *uniname); - s32 (*count_ext_entries)(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - struct dentry_t *p_entry); - s32 (*calc_num_entries)(struct uni_name_t *p_uniname); - - u32 (*get_entry_type)(struct dentry_t *p_entry); - void (*set_entry_type)(struct dentry_t *p_entry, u32 type); - u32 (*get_entry_attr)(struct dentry_t *p_entry); - void (*set_entry_attr)(struct dentry_t *p_entry, u32 attr); - u8 (*get_entry_flag)(struct dentry_t *p_entry); - void (*set_entry_flag)(struct dentry_t *p_entry, u8 flag); - u32 (*get_entry_clu0)(struct dentry_t *p_entry); - void (*set_entry_clu0)(struct dentry_t *p_entry, u32 clu0); - u64 (*get_entry_size)(struct dentry_t *p_entry); - void (*set_entry_size)(struct dentry_t *p_entry, u64 size); - void (*get_entry_time)(struct dentry_t *p_entry, - struct timestamp_t *tp, u8 mode); - void (*set_entry_time)(struct dentry_t *p_entry, - struct timestamp_t *tp, u8 mode); -}; - struct fs_info_t { u32 drv; /* drive ID */ u32 vol_type; /* volume FAT type */ @@ -617,8 +554,7 @@ struct fs_info_t { u32 dev_ejected; /* block device operation error flag */ - struct fs_func *fs_func; - struct semaphore v_sem; + struct mutex v_mutex; /* FAT cache */ struct buf_cache_t FAT_cache_array[FAT_CACHE_SIZE]; @@ -681,10 +617,10 @@ struct exfat_mount_options { /* on error: continue, panic, remount-ro */ unsigned char errors; -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD /* flag on if -o dicard specified and device support discard() */ unsigned char discard; -#endif /* CONFIG_EXFAT_DISCARD */ +#endif /* CONFIG_STAGING_EXFAT_DISCARD */ }; #define EXFAT_HASH_BITS 8 @@ -720,9 +656,9 @@ struct exfat_sb_info { spinlock_t inode_hash_lock; struct hlist_head inode_hashtable[EXFAT_HASH_SIZE]; -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG long debug_flags; -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ }; /* @@ -749,14 +685,7 @@ static inline struct exfat_inode_info *EXFAT_I(struct inode *inode) /* NLS management function */ u16 nls_upper(struct super_block *sb, u16 a); -int nls_dosname_cmp(struct super_block *sb, u8 *a, u8 *b); int nls_uniname_cmp(struct super_block *sb, u16 *a, u16 *b); -void nls_uniname_to_dosname(struct super_block *sb, - struct dos_name_t *p_dosname, - struct uni_name_t *p_uniname, bool *p_lossy); -void nls_dosname_to_uniname(struct super_block *sb, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname); void nls_uniname_to_cstring(struct super_block *sb, u8 *p_cstring, struct uni_name_t *p_uniname); void nls_cstring_to_uniname(struct super_block *sb, @@ -764,48 +693,33 @@ void nls_cstring_to_uniname(struct super_block *sb, bool *p_lossy); /* buffer cache management */ -void buf_init(struct super_block *sb); -void buf_shutdown(struct super_block *sb); -int FAT_read(struct super_block *sb, u32 loc, u32 *content); -s32 FAT_write(struct super_block *sb, u32 loc, u32 content); -u8 *FAT_getblk(struct super_block *sb, sector_t sec); -void FAT_modify(struct super_block *sb, sector_t sec); -void FAT_release_all(struct super_block *sb); -void FAT_sync(struct super_block *sb); -u8 *buf_getblk(struct super_block *sb, sector_t sec); -void buf_modify(struct super_block *sb, sector_t sec); -void buf_lock(struct super_block *sb, sector_t sec); -void buf_unlock(struct super_block *sb, sector_t sec); -void buf_release(struct super_block *sb, sector_t sec); -void buf_release_all(struct super_block *sb); -void buf_sync(struct super_block *sb); +void exfat_buf_init(struct super_block *sb); +void exfat_buf_shutdown(struct super_block *sb); +int exfat_fat_read(struct super_block *sb, u32 loc, u32 *content); +s32 exfat_fat_write(struct super_block *sb, u32 loc, u32 content); +u8 *exfat_fat_getblk(struct super_block *sb, sector_t sec); +void exfat_fat_modify(struct super_block *sb, sector_t sec); +void exfat_fat_release_all(struct super_block *sb); +void exfat_fat_sync(struct super_block *sb); +u8 *exfat_buf_getblk(struct super_block *sb, sector_t sec); +void exfat_buf_modify(struct super_block *sb, sector_t sec); +void exfat_buf_lock(struct super_block *sb, sector_t sec); +void exfat_buf_unlock(struct super_block *sb, sector_t sec); +void exfat_buf_release(struct super_block *sb, sector_t sec); +void exfat_buf_release_all(struct super_block *sb); +void exfat_buf_sync(struct super_block *sb); /* fs management functions */ void fs_set_vol_flags(struct super_block *sb, u32 new_flag); void fs_error(struct super_block *sb); /* cluster management functions */ -s32 clear_cluster(struct super_block *sb, u32 clu); -s32 fat_alloc_cluster(struct super_block *sb, s32 num_alloc, - struct chain_t *p_chain); -s32 exfat_alloc_cluster(struct super_block *sb, s32 num_alloc, - struct chain_t *p_chain); -void fat_free_cluster(struct super_block *sb, struct chain_t *p_chain, - s32 do_relse); -void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, - s32 do_relse); -u32 find_last_cluster(struct super_block *sb, struct chain_t *p_chain); s32 count_num_clusters(struct super_block *sb, struct chain_t *dir); -s32 fat_count_used_clusters(struct super_block *sb); -s32 exfat_count_used_clusters(struct super_block *sb); void exfat_chain_cont_cluster(struct super_block *sb, u32 chain, s32 len); /* allocation bitmap management functions */ s32 load_alloc_bitmap(struct super_block *sb); void free_alloc_bitmap(struct super_block *sb); -s32 set_alloc_bitmap(struct super_block *sb, u32 clu); -s32 clr_alloc_bitmap(struct super_block *sb, u32 clu); -u32 test_alloc_bitmap(struct super_block *sb, u32 clu); void sync_alloc_bitmap(struct super_block *sb); /* upcase table management functions */ @@ -813,63 +727,8 @@ s32 load_upcase_table(struct super_block *sb); void free_upcase_table(struct super_block *sb); /* dir entry management functions */ -u32 fat_get_entry_type(struct dentry_t *p_entry); -u32 exfat_get_entry_type(struct dentry_t *p_entry); -void fat_set_entry_type(struct dentry_t *p_entry, u32 type); -void exfat_set_entry_type(struct dentry_t *p_entry, u32 type); -u32 fat_get_entry_attr(struct dentry_t *p_entry); -u32 exfat_get_entry_attr(struct dentry_t *p_entry); -void fat_set_entry_attr(struct dentry_t *p_entry, u32 attr); -void exfat_set_entry_attr(struct dentry_t *p_entry, u32 attr); -u8 fat_get_entry_flag(struct dentry_t *p_entry); -u8 exfat_get_entry_flag(struct dentry_t *p_entry); -void fat_set_entry_flag(struct dentry_t *p_entry, u8 flag); -void exfat_set_entry_flag(struct dentry_t *p_entry, u8 flag); -u32 fat_get_entry_clu0(struct dentry_t *p_entry); -u32 exfat_get_entry_clu0(struct dentry_t *p_entry); -void fat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu); -void exfat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu); -u64 fat_get_entry_size(struct dentry_t *p_entry); -u64 exfat_get_entry_size(struct dentry_t *p_entry); -void fat_set_entry_size(struct dentry_t *p_entry, u64 size); -void exfat_set_entry_size(struct dentry_t *p_entry, u64 size); struct timestamp_t *tm_current(struct timestamp_t *tm); -void fat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode); -void exfat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode); -void fat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode); -void exfat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode); -s32 fat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, s32 entry, - u32 type, u32 start_clu, u64 size); -s32 exfat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, u32 type, u32 start_clu, u64 size); -s32 fat_init_ext_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 num_entries, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname); -s32 exfat_init_ext_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 num_entries, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname); -void init_dos_entry(struct dos_dentry_t *ep, u32 type, u32 start_clu); -void init_ext_entry(struct ext_dentry_t *ep, s32 order, u8 chksum, - u16 *uniname); -void init_file_entry(struct file_dentry_t *ep, u32 type); -void init_strm_entry(struct strm_dentry_t *ep, u8 flags, u32 start_clu, - u64 size); -void init_name_entry(struct name_dentry_t *ep, u16 *uniname); -void fat_delete_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 order, s32 num_entries); -void exfat_delete_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 order, s32 num_entries); -s32 find_location(struct super_block *sb, struct chain_t *p_dir, s32 entry, - sector_t *sector, s32 *offset); -struct dentry_t *get_entry_with_sector(struct super_block *sb, sector_t sector, - s32 offset); struct dentry_t *get_entry_in_dir(struct super_block *sb, struct chain_t *p_dir, s32 entry, sector_t *sector); struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, @@ -877,24 +736,6 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, u32 type, struct dentry_t **file_ep); void release_entry_set(struct entry_set_cache_t *es); -s32 write_whole_entry_set(struct super_block *sb, struct entry_set_cache_t *es); -s32 write_partial_entries_in_entry_set(struct super_block *sb, - struct entry_set_cache_t *es, - struct dentry_t *ep, u32 count); -s32 search_deleted_or_unused_entry(struct super_block *sb, - struct chain_t *p_dir, s32 num_entries); -s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, - s32 num_entries); -s32 fat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, - struct uni_name_t *p_uniname, s32 num_entries, - struct dos_name_t *p_dosname, u32 type); -s32 exfat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, - struct uni_name_t *p_uniname, s32 num_entries, - struct dos_name_t *p_dosname, u32 type); -s32 fat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, - s32 entry, struct dentry_t *p_entry); -s32 exfat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, - s32 entry, struct dentry_t *p_entry); s32 count_dos_name_entries(struct super_block *sb, struct chain_t *p_dir, u32 type); void update_dir_checksum(struct super_block *sb, struct chain_t *p_dir, @@ -907,36 +748,13 @@ bool is_dir_empty(struct super_block *sb, struct chain_t *p_dir); s32 get_num_entries_and_dos_name(struct super_block *sb, struct chain_t *p_dir, struct uni_name_t *p_uniname, s32 *entries, struct dos_name_t *p_dosname); -void get_uni_name_from_dos_entry(struct super_block *sb, - struct dos_dentry_t *ep, - struct uni_name_t *p_uniname, u8 mode); -void fat_get_uni_name_from_ext_entry(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - u16 *uniname); -void exfat_get_uni_name_from_ext_entry(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - u16 *uniname); -s32 extract_uni_name_from_ext_entry(struct ext_dentry_t *ep, - u16 *uniname, s32 order); -s32 extract_uni_name_from_name_entry(struct name_dentry_t *ep, - u16 *uniname, s32 order); -s32 fat_generate_dos_name(struct super_block *sb, struct chain_t *p_dir, - struct dos_name_t *p_dosname); -void fat_attach_count_to_dos_name(u8 *dosname, s32 count); -s32 fat_calc_num_entries(struct uni_name_t *p_uniname); -s32 exfat_calc_num_entries(struct uni_name_t *p_uniname); -u8 calc_checksum_1byte(void *data, s32 len, u8 chksum); u16 calc_checksum_2byte(void *data, s32 len, u16 chksum, s32 type); -u32 calc_checksum_4byte(void *data, s32 len, u32 chksum, s32 type); /* name resolution functions */ s32 resolve_path(struct inode *inode, char *path, struct chain_t *p_dir, struct uni_name_t *p_uniname); -s32 resolve_name(u8 *name, u8 **arg); /* file operation functions */ -s32 fat16_mount(struct super_block *sb, struct pbr_sector_t *p_pbr); -s32 fat32_mount(struct super_block *sb, struct pbr_sector_t *p_pbr); s32 exfat_mount(struct super_block *sb, struct pbr_sector_t *p_pbr); s32 create_dir(struct inode *inode, struct chain_t *p_dir, struct uni_name_t *p_uniname, struct file_id_t *fid); @@ -959,13 +777,48 @@ int multi_sector_read(struct super_block *sb, sector_t sec, int multi_sector_write(struct super_block *sb, sector_t sec, struct buffer_head *bh, s32 num_secs, bool sync); -void bdev_open(struct super_block *sb); -void bdev_close(struct super_block *sb); -int bdev_read(struct super_block *sb, sector_t secno, +void exfat_bdev_open(struct super_block *sb); +void exfat_bdev_close(struct super_block *sb); +int exfat_bdev_read(struct super_block *sb, sector_t secno, struct buffer_head **bh, u32 num_secs, bool read); -int bdev_write(struct super_block *sb, sector_t secno, +int exfat_bdev_write(struct super_block *sb, sector_t secno, struct buffer_head *bh, u32 num_secs, bool sync); -int bdev_sync(struct super_block *sb); +int exfat_bdev_sync(struct super_block *sb); + +/* cluster operation functions */ +s32 exfat_alloc_cluster(struct super_block *sb, s32 num_alloc, + struct chain_t *p_chain); +void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, + s32 do_relse); +s32 exfat_count_used_clusters(struct super_block *sb); + +/* dir operation functions */ +s32 exfat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, + struct uni_name_t *p_uniname, s32 num_entries, + struct dos_name_t *p_dosname, u32 type); +void exfat_delete_dir_entry(struct super_block *sb, struct chain_t *p_dir, + s32 entry, s32 order, s32 num_entries); +void exfat_get_uni_name_from_ext_entry(struct super_block *sb, + struct chain_t *p_dir, s32 entry, + u16 *uniname); +s32 exfat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, + s32 entry, struct dentry_t *p_entry); +s32 exfat_calc_num_entries(struct uni_name_t *p_uniname); + +/* dir entry getter/setter */ +u32 exfat_get_entry_type(struct dentry_t *p_entry); +u32 exfat_get_entry_attr(struct dentry_t *p_entry); +void exfat_set_entry_attr(struct dentry_t *p_entry, u32 attr); +u8 exfat_get_entry_flag(struct dentry_t *p_entry); +void exfat_set_entry_flag(struct dentry_t *p_entry, u8 flags); +u32 exfat_get_entry_clu0(struct dentry_t *p_entry); +void exfat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu); +u64 exfat_get_entry_size(struct dentry_t *p_entry); +void exfat_set_entry_size(struct dentry_t *p_entry, u64 size); +void exfat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, + u8 mode); +void exfat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, + u8 mode); extern const u8 uni_upcase[]; #endif /* _EXFAT_H */ diff --git a/drivers/staging/exfat/exfat_blkdev.c b/drivers/staging/exfat/exfat_blkdev.c index 81d20e6241c65..0a3dc3568293b 100644 --- a/drivers/staging/exfat/exfat_blkdev.c +++ b/drivers/staging/exfat/exfat_blkdev.c @@ -8,7 +8,7 @@ #include #include "exfat.h" -void bdev_open(struct super_block *sb) +void exfat_bdev_open(struct super_block *sb) { struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); @@ -23,28 +23,28 @@ void bdev_open(struct super_block *sb) p_bd->opened = true; } -void bdev_close(struct super_block *sb) +void exfat_bdev_close(struct super_block *sb) { struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); p_bd->opened = false; } -int bdev_read(struct super_block *sb, sector_t secno, struct buffer_head **bh, - u32 num_secs, bool read) +int exfat_bdev_read(struct super_block *sb, sector_t secno, struct buffer_head **bh, + u32 num_secs, bool read) { struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG struct exfat_sb_info *sbi = EXFAT_SB(sb); long flags = sbi->debug_flags; if (flags & EXFAT_DEBUGFLAGS_ERROR_RW) - return FFS_MEDIAERR; -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ + return -EIO; +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ if (!p_bd->opened) - return FFS_MEDIAERR; + return -ENODEV; if (*bh) __brelse(*bh); @@ -62,26 +62,26 @@ int bdev_read(struct super_block *sb, sector_t secno, struct buffer_head **bh, WARN(!p_fs->dev_ejected, "[EXFAT] No bh, device seems wrong or to be ejected.\n"); - return FFS_MEDIAERR; + return -EIO; } -int bdev_write(struct super_block *sb, sector_t secno, struct buffer_head *bh, - u32 num_secs, bool sync) +int exfat_bdev_write(struct super_block *sb, sector_t secno, struct buffer_head *bh, + u32 num_secs, bool sync) { s32 count; struct buffer_head *bh2; struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG struct exfat_sb_info *sbi = EXFAT_SB(sb); long flags = sbi->debug_flags; if (flags & EXFAT_DEBUGFLAGS_ERROR_RW) - return FFS_MEDIAERR; -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ + return -EIO; +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ if (!p_bd->opened) - return FFS_MEDIAERR; + return -ENODEV; if (secno == bh->b_blocknr) { lock_buffer(bh); @@ -89,7 +89,7 @@ int bdev_write(struct super_block *sb, sector_t secno, struct buffer_head *bh, mark_buffer_dirty(bh); unlock_buffer(bh); if (sync && (sync_dirty_buffer(bh) != 0)) - return FFS_MEDIAERR; + return -EIO; } else { count = num_secs << p_bd->sector_size_bits; @@ -115,22 +115,22 @@ int bdev_write(struct super_block *sb, sector_t secno, struct buffer_head *bh, WARN(!p_fs->dev_ejected, "[EXFAT] No bh, device seems wrong or to be ejected.\n"); - return FFS_MEDIAERR; + return -EIO; } -int bdev_sync(struct super_block *sb) +int exfat_bdev_sync(struct super_block *sb) { struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG struct exfat_sb_info *sbi = EXFAT_SB(sb); long flags = sbi->debug_flags; if (flags & EXFAT_DEBUGFLAGS_ERROR_RW) - return FFS_MEDIAERR; -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ + return -EIO; +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ if (!p_bd->opened) - return FFS_MEDIAERR; + return -ENODEV; return sync_blockdev(sb->s_bdev); } diff --git a/drivers/staging/exfat/exfat_cache.c b/drivers/staging/exfat/exfat_cache.c index e1b0017187090..3fd5604058a9d 100644 --- a/drivers/staging/exfat/exfat_cache.c +++ b/drivers/staging/exfat/exfat_cache.c @@ -12,8 +12,8 @@ #define DIRTYBIT 0x02 /* Local variables */ -static DEFINE_SEMAPHORE(f_sem); -static DEFINE_SEMAPHORE(b_sem); +static DEFINE_MUTEX(f_mutex); +static DEFINE_MUTEX(b_mutex); static struct buf_cache_t *FAT_cache_find(struct super_block *sb, sector_t sec) { @@ -128,7 +128,7 @@ static void buf_cache_remove_hash(struct buf_cache_t *bp) (bp->hash_next)->hash_prev = bp->hash_prev; } -void buf_init(struct super_block *sb) +void exfat_buf_init(struct super_block *sb) { struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); @@ -189,11 +189,11 @@ void buf_init(struct super_block *sb) buf_cache_insert_hash(sb, &p_fs->buf_cache_array[i]); } -void buf_shutdown(struct super_block *sb) +void exfat_buf_shutdown(struct super_block *sb) { } -static int __FAT_read(struct super_block *sb, u32 loc, u32 *content) +static int __exfat_fat_read(struct super_block *sb, u32 loc, u32 *content) { s32 off; u32 _content; @@ -202,107 +202,22 @@ static int __FAT_read(struct super_block *sb, u32 loc, u32 *content) struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - if (p_fs->vol_type == FAT12) { - sec = p_fs->FAT1_start_sector + - ((loc + (loc >> 1)) >> p_bd->sector_size_bits); - off = (loc + (loc >> 1)) & p_bd->sector_size_mask; + sec = p_fs->FAT1_start_sector + + (loc >> (p_bd->sector_size_bits - 2)); + off = (loc << 2) & p_bd->sector_size_mask; - if (off == (p_bd->sector_size - 1)) { - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; + fat_sector = exfat_fat_getblk(sb, sec); + if (!fat_sector) + return -1; - _content = (u32)fat_sector[off]; + fat_entry = &fat_sector[off]; + _content = GET32_A(fat_entry); - fat_sector = FAT_getblk(sb, ++sec); - if (!fat_sector) - return -1; - - _content |= (u32)fat_sector[0] << 8; - } else { - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - _content = GET16(fat_entry); - } - - if (loc & 1) - _content >>= 4; - - _content &= 0x00000FFF; - - if (_content >= CLUSTER_16(0x0FF8)) { - *content = CLUSTER_32(~0); - return 0; - } - *content = CLUSTER_32(_content); - return 0; - } else if (p_fs->vol_type == FAT16) { - sec = p_fs->FAT1_start_sector + - (loc >> (p_bd->sector_size_bits - 1)); - off = (loc << 1) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - - _content = GET16_A(fat_entry); - - _content &= 0x0000FFFF; - - if (_content >= CLUSTER_16(0xFFF8)) { - *content = CLUSTER_32(~0); - return 0; - } - *content = CLUSTER_32(_content); - return 0; - } else if (p_fs->vol_type == FAT32) { - sec = p_fs->FAT1_start_sector + - (loc >> (p_bd->sector_size_bits - 2)); - off = (loc << 2) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - - _content = GET32_A(fat_entry); - - _content &= 0x0FFFFFFF; - - if (_content >= CLUSTER_32(0x0FFFFFF8)) { - *content = CLUSTER_32(~0); - return 0; - } - *content = CLUSTER_32(_content); - return 0; - } else if (p_fs->vol_type == EXFAT) { - sec = p_fs->FAT1_start_sector + - (loc >> (p_bd->sector_size_bits - 2)); - off = (loc << 2) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - _content = GET32_A(fat_entry); - - if (_content >= CLUSTER_32(0xFFFFFFF8)) { - *content = CLUSTER_32(~0); - return 0; - } - *content = CLUSTER_32(_content); + if (_content >= CLUSTER_32(0xFFFFFFF8)) { + *content = CLUSTER_32(~0); return 0; } - - /* Unknown volume type, throw in the towel and go home */ - *content = CLUSTER_32(~0); + *content = CLUSTER_32(_content); return 0; } @@ -311,18 +226,18 @@ static int __FAT_read(struct super_block *sb, u32 loc, u32 *content) * returns 0 on success * -1 on error */ -int FAT_read(struct super_block *sb, u32 loc, u32 *content) +int exfat_fat_read(struct super_block *sb, u32 loc, u32 *content) { s32 ret; - down(&f_sem); - ret = __FAT_read(sb, loc, content); - up(&f_sem); + mutex_lock(&f_mutex); + ret = __exfat_fat_read(sb, loc, content); + mutex_unlock(&f_mutex); return ret; } -static s32 __FAT_write(struct super_block *sb, u32 loc, u32 content) +static s32 __exfat_fat_write(struct super_block *sb, u32 loc, u32 content) { s32 off; sector_t sec; @@ -330,118 +245,34 @@ static s32 __FAT_write(struct super_block *sb, u32 loc, u32 content) struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - if (p_fs->vol_type == FAT12) { - content &= 0x00000FFF; - - sec = p_fs->FAT1_start_sector + - ((loc + (loc >> 1)) >> p_bd->sector_size_bits); - off = (loc + (loc >> 1)) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - if (loc & 1) { /* odd */ - content <<= 4; - - if (off == (p_bd->sector_size - 1)) { - fat_sector[off] = (u8)(content | - (fat_sector[off] & - 0x0F)); - FAT_modify(sb, sec); - - fat_sector = FAT_getblk(sb, ++sec); - if (!fat_sector) - return -1; - - fat_sector[0] = (u8)(content >> 8); - } else { - fat_entry = &fat_sector[off]; - content |= GET16(fat_entry) & 0x000F; - - SET16(fat_entry, content); - } - } else { /* even */ - fat_sector[off] = (u8)(content); - - if (off == (p_bd->sector_size - 1)) { - fat_sector[off] = (u8)(content); - FAT_modify(sb, sec); - - fat_sector = FAT_getblk(sb, ++sec); - if (!fat_sector) - return -1; - fat_sector[0] = (u8)((fat_sector[0] & 0xF0) | - (content >> 8)); - } else { - fat_entry = &fat_sector[off]; - content |= GET16(fat_entry) & 0xF000; - - SET16(fat_entry, content); - } - } - } - - else if (p_fs->vol_type == FAT16) { - content &= 0x0000FFFF; - - sec = p_fs->FAT1_start_sector + (loc >> - (p_bd->sector_size_bits - 1)); - off = (loc << 1) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - - SET16_A(fat_entry, content); - } else if (p_fs->vol_type == FAT32) { - content &= 0x0FFFFFFF; + sec = p_fs->FAT1_start_sector + (loc >> + (p_bd->sector_size_bits - 2)); + off = (loc << 2) & p_bd->sector_size_mask; - sec = p_fs->FAT1_start_sector + (loc >> - (p_bd->sector_size_bits - 2)); - off = (loc << 2) & p_bd->sector_size_mask; + fat_sector = exfat_fat_getblk(sb, sec); + if (!fat_sector) + return -1; - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; + fat_entry = &fat_sector[off]; - fat_entry = &fat_sector[off]; - - content |= GET32_A(fat_entry) & 0xF0000000; - - SET32_A(fat_entry, content); - } else { /* p_fs->vol_type == EXFAT */ - sec = p_fs->FAT1_start_sector + (loc >> - (p_bd->sector_size_bits - 2)); - off = (loc << 2) & p_bd->sector_size_mask; - - fat_sector = FAT_getblk(sb, sec); - if (!fat_sector) - return -1; - - fat_entry = &fat_sector[off]; - - SET32_A(fat_entry, content); - } + SET32_A(fat_entry, content); - FAT_modify(sb, sec); + exfat_fat_modify(sb, sec); return 0; } -int FAT_write(struct super_block *sb, u32 loc, u32 content) +int exfat_fat_write(struct super_block *sb, u32 loc, u32 content) { s32 ret; - down(&f_sem); - ret = __FAT_write(sb, loc, content); - up(&f_sem); + mutex_lock(&f_mutex); + ret = __exfat_fat_write(sb, loc, content); + mutex_unlock(&f_mutex); return ret; } -u8 *FAT_getblk(struct super_block *sb, sector_t sec) +u8 *exfat_fat_getblk(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); @@ -462,7 +293,7 @@ u8 *FAT_getblk(struct super_block *sb, sector_t sec) FAT_cache_insert_hash(sb, bp); - if (sector_read(sb, sec, &bp->buf_bh, 1) != FFS_SUCCESS) { + if (sector_read(sb, sec, &bp->buf_bh, 1) != 0) { FAT_cache_remove_hash(bp); bp->drv = -1; bp->sec = ~0; @@ -476,7 +307,7 @@ u8 *FAT_getblk(struct super_block *sb, sector_t sec) return bp->buf_bh->b_data; } -void FAT_modify(struct super_block *sb, sector_t sec) +void exfat_fat_modify(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; @@ -485,12 +316,12 @@ void FAT_modify(struct super_block *sb, sector_t sec) sector_write(sb, sec, bp->buf_bh, 0); } -void FAT_release_all(struct super_block *sb) +void exfat_fat_release_all(struct super_block *sb) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - down(&f_sem); + mutex_lock(&f_mutex); bp = p_fs->FAT_cache_lru_list.next; while (bp != &p_fs->FAT_cache_lru_list) { @@ -507,15 +338,15 @@ void FAT_release_all(struct super_block *sb) bp = bp->next; } - up(&f_sem); + mutex_unlock(&f_mutex); } -void FAT_sync(struct super_block *sb) +void exfat_fat_sync(struct super_block *sb) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - down(&f_sem); + mutex_lock(&f_mutex); bp = p_fs->FAT_cache_lru_list.next; while (bp != &p_fs->FAT_cache_lru_list) { @@ -526,7 +357,7 @@ void FAT_sync(struct super_block *sb) bp = bp->next; } - up(&f_sem); + mutex_unlock(&f_mutex); } static struct buf_cache_t *buf_cache_find(struct super_block *sb, sector_t sec) @@ -561,7 +392,7 @@ static struct buf_cache_t *buf_cache_get(struct super_block *sb, sector_t sec) return bp; } -static u8 *__buf_getblk(struct super_block *sb, sector_t sec) +static u8 *__exfat_buf_getblk(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); @@ -582,7 +413,7 @@ static u8 *__buf_getblk(struct super_block *sb, sector_t sec) buf_cache_insert_hash(sb, bp); - if (sector_read(sb, sec, &bp->buf_bh, 1) != FFS_SUCCESS) { + if (sector_read(sb, sec, &bp->buf_bh, 1) != 0) { buf_cache_remove_hash(bp); bp->drv = -1; bp->sec = ~0; @@ -596,22 +427,22 @@ static u8 *__buf_getblk(struct super_block *sb, sector_t sec) return bp->buf_bh->b_data; } -u8 *buf_getblk(struct super_block *sb, sector_t sec) +u8 *exfat_buf_getblk(struct super_block *sb, sector_t sec) { u8 *buf; - down(&b_sem); - buf = __buf_getblk(sb, sec); - up(&b_sem); + mutex_lock(&b_mutex); + buf = __exfat_buf_getblk(sb, sec); + mutex_unlock(&b_mutex); return buf; } -void buf_modify(struct super_block *sb, sector_t sec) +void exfat_buf_modify(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; - down(&b_sem); + mutex_lock(&b_mutex); bp = buf_cache_find(sb, sec); if (likely(bp)) @@ -620,14 +451,14 @@ void buf_modify(struct super_block *sb, sector_t sec) WARN(!bp, "[EXFAT] failed to find buffer_cache(sector:%llu).\n", (unsigned long long)sec); - up(&b_sem); + mutex_unlock(&b_mutex); } -void buf_lock(struct super_block *sb, sector_t sec) +void exfat_buf_lock(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; - down(&b_sem); + mutex_lock(&b_mutex); bp = buf_cache_find(sb, sec); if (likely(bp)) @@ -636,14 +467,14 @@ void buf_lock(struct super_block *sb, sector_t sec) WARN(!bp, "[EXFAT] failed to find buffer_cache(sector:%llu).\n", (unsigned long long)sec); - up(&b_sem); + mutex_unlock(&b_mutex); } -void buf_unlock(struct super_block *sb, sector_t sec) +void exfat_buf_unlock(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; - down(&b_sem); + mutex_lock(&b_mutex); bp = buf_cache_find(sb, sec); if (likely(bp)) @@ -652,15 +483,15 @@ void buf_unlock(struct super_block *sb, sector_t sec) WARN(!bp, "[EXFAT] failed to find buffer_cache(sector:%llu).\n", (unsigned long long)sec); - up(&b_sem); + mutex_unlock(&b_mutex); } -void buf_release(struct super_block *sb, sector_t sec) +void exfat_buf_release(struct super_block *sb, sector_t sec) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - down(&b_sem); + mutex_lock(&b_mutex); bp = buf_cache_find(sb, sec); if (likely(bp)) { @@ -676,15 +507,15 @@ void buf_release(struct super_block *sb, sector_t sec) move_to_lru(bp, &p_fs->buf_cache_lru_list); } - up(&b_sem); + mutex_unlock(&b_mutex); } -void buf_release_all(struct super_block *sb) +void exfat_buf_release_all(struct super_block *sb) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - down(&b_sem); + mutex_lock(&b_mutex); bp = p_fs->buf_cache_lru_list.next; while (bp != &p_fs->buf_cache_lru_list) { @@ -701,15 +532,15 @@ void buf_release_all(struct super_block *sb) bp = bp->next; } - up(&b_sem); + mutex_unlock(&b_mutex); } -void buf_sync(struct super_block *sb) +void exfat_buf_sync(struct super_block *sb) { struct buf_cache_t *bp; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - down(&b_sem); + mutex_lock(&b_mutex); bp = p_fs->buf_cache_lru_list.next; while (bp != &p_fs->buf_cache_lru_list) { @@ -720,5 +551,5 @@ void buf_sync(struct super_block *sb) bp = bp->next; } - up(&b_sem); + mutex_unlock(&b_mutex); } diff --git a/drivers/staging/exfat/exfat_core.c b/drivers/staging/exfat/exfat_core.c index f3774a1912d1e..07b460d01334c 100644 --- a/drivers/staging/exfat/exfat_core.c +++ b/drivers/staging/exfat/exfat_core.c @@ -20,15 +20,6 @@ static void __set_sb_dirty(struct super_block *sb) static u8 name_buf[MAX_PATH_LENGTH * MAX_CHARSET_SIZE]; -static char *reserved_names[] = { - "AUX ", "CON ", "NUL ", "PRN ", - "COM1 ", "COM2 ", "COM3 ", "COM4 ", - "COM5 ", "COM6 ", "COM7 ", "COM8 ", "COM9 ", - "LPT1 ", "LPT2 ", "LPT3 ", "LPT4 ", - "LPT5 ", "LPT6 ", "LPT7 ", "LPT8 ", "LPT9 ", - NULL -}; - static u8 free_bit[] = { 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, /* 0 ~ 19 */ 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, /* 20 ~ 39 */ @@ -99,25 +90,23 @@ void fs_set_vol_flags(struct super_block *sb, u32 new_flag) p_fs->vol_flag = new_flag; - if (p_fs->vol_type == EXFAT) { - if (!p_fs->pbr_bh) { - if (sector_read(sb, p_fs->PBR_sector, - &p_fs->pbr_bh, 1) != FFS_SUCCESS) - return; - } + if (!p_fs->pbr_bh) { + if (sector_read(sb, p_fs->PBR_sector, + &p_fs->pbr_bh, 1) != 0) + return; + } - p_pbr = (struct pbr_sector_t *)p_fs->pbr_bh->b_data; - p_bpb = (struct bpbex_t *)p_pbr->bpb; - SET16(p_bpb->vol_flags, (u16)new_flag); + p_pbr = (struct pbr_sector_t *)p_fs->pbr_bh->b_data; + p_bpb = (struct bpbex_t *)p_pbr->bpb; + SET16(p_bpb->vol_flags, (u16)new_flag); - /* XXX duyoung - * what can we do here? (cuz fs_set_vol_flags() is void) - */ - if ((new_flag == VOL_DIRTY) && (!buffer_dirty(p_fs->pbr_bh))) - sector_write(sb, p_fs->PBR_sector, p_fs->pbr_bh, 1); - else - sector_write(sb, p_fs->PBR_sector, p_fs->pbr_bh, 0); - } + /* XXX duyoung + * what can we do here? (cuz fs_set_vol_flags() is void) + */ + if ((new_flag == VOL_DIRTY) && (!buffer_dirty(p_fs->pbr_bh))) + sector_write(sb, p_fs->PBR_sector, p_fs->pbr_bh, 1); + else + sector_write(sb, p_fs->PBR_sector, p_fs->pbr_bh, 0); } void fs_error(struct super_block *sb) @@ -136,10 +125,10 @@ void fs_error(struct super_block *sb) * Cluster Management Functions */ -s32 clear_cluster(struct super_block *sb, u32 clu) +static s32 clear_cluster(struct super_block *sb, u32 clu) { sector_t s, n; - s32 ret = FFS_SUCCESS; + s32 ret = 0; struct buffer_head *tmp_bh = NULL; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); @@ -154,12 +143,12 @@ s32 clear_cluster(struct super_block *sb, u32 clu) for (; s < n; s++) { ret = sector_read(sb, s, &tmp_bh, 0); - if (ret != FFS_SUCCESS) + if (ret != 0) return ret; memset((char *)tmp_bh->b_data, 0x0, p_bd->sector_size); ret = sector_write(sb, s, tmp_bh, 0); - if (ret != FFS_SUCCESS) + if (ret != 0) break; } @@ -167,58 +156,97 @@ s32 clear_cluster(struct super_block *sb, u32 clu) return ret; } -s32 fat_alloc_cluster(struct super_block *sb, s32 num_alloc, - struct chain_t *p_chain) +static s32 set_alloc_bitmap(struct super_block *sb, u32 clu) { - int i, num_clusters = 0; - u32 new_clu, last_clu = CLUSTER_32(~0), read_clu; + int i, b; + sector_t sector; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); + struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - new_clu = p_chain->dir; - if (new_clu == CLUSTER_32(~0)) - new_clu = p_fs->clu_srch_ptr; - else if (new_clu >= p_fs->num_clusters) - new_clu = 2; + i = clu >> (p_bd->sector_size_bits + 3); + b = clu & ((p_bd->sector_size << 3) - 1); - __set_sb_dirty(sb); + sector = START_SECTOR(p_fs->map_clu) + i; - p_chain->dir = CLUSTER_32(~0); + exfat_bitmap_set((u8 *)p_fs->vol_amap[i]->b_data, b); - for (i = 2; i < p_fs->num_clusters; i++) { - if (FAT_read(sb, new_clu, &read_clu) != 0) - return -1; + return sector_write(sb, sector, p_fs->vol_amap[i], 0); +} - if (read_clu == CLUSTER_32(0)) { - if (FAT_write(sb, new_clu, CLUSTER_32(~0)) < 0) - return -1; - num_clusters++; +static s32 clr_alloc_bitmap(struct super_block *sb, u32 clu) +{ + int i, b; + sector_t sector; +#ifdef CONFIG_STAGING_EXFAT_DISCARD + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_mount_options *opts = &sbi->options; + int ret; +#endif /* CONFIG_STAGING_EXFAT_DISCARD */ + struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); + struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - if (p_chain->dir == CLUSTER_32(~0)) { - p_chain->dir = new_clu; - } else { - if (FAT_write(sb, last_clu, new_clu) < 0) - return -1; - } + i = clu >> (p_bd->sector_size_bits + 3); + b = clu & ((p_bd->sector_size << 3) - 1); - last_clu = new_clu; + sector = START_SECTOR(p_fs->map_clu) + i; - if ((--num_alloc) == 0) { - p_fs->clu_srch_ptr = new_clu; - if (p_fs->used_clusters != UINT_MAX) - p_fs->used_clusters += num_clusters; + exfat_bitmap_clear((u8 *)p_fs->vol_amap[i]->b_data, b); - return num_clusters; - } +#ifdef CONFIG_STAGING_EXFAT_DISCARD + if (opts->discard) { + ret = sb_issue_discard(sb, START_SECTOR(clu), + (1 << p_fs->sectors_per_clu_bits), + GFP_NOFS, 0); + if (ret == -EOPNOTSUPP) { + pr_warn("discard not supported by device, disabling"); + opts->discard = 0; + } else { + return ret; } - if ((++new_clu) >= p_fs->num_clusters) - new_clu = 2; } +#endif /* CONFIG_STAGING_EXFAT_DISCARD */ - p_fs->clu_srch_ptr = new_clu; - if (p_fs->used_clusters != UINT_MAX) - p_fs->used_clusters += num_clusters; + return sector_write(sb, sector, p_fs->vol_amap[i], 0); +} - return num_clusters; +static u32 test_alloc_bitmap(struct super_block *sb, u32 clu) +{ + int i, map_i, map_b; + u32 clu_base, clu_free; + u8 k, clu_mask; + struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); + struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); + + clu_base = (clu & ~(0x7)) + 2; + clu_mask = (1 << (clu - clu_base + 2)) - 1; + + map_i = clu >> (p_bd->sector_size_bits + 3); + map_b = (clu >> 3) & p_bd->sector_size_mask; + + for (i = 2; i < p_fs->num_clusters; i += 8) { + k = *(((u8 *)p_fs->vol_amap[map_i]->b_data) + map_b); + if (clu_mask > 0) { + k |= clu_mask; + clu_mask = 0; + } + if (k < 0xFF) { + clu_free = clu_base + free_bit[k]; + if (clu_free < p_fs->num_clusters) + return clu_free; + } + clu_base += 8; + + if (((++map_b) >= p_bd->sector_size) || + (clu_base >= p_fs->num_clusters)) { + if ((++map_i) >= p_fs->map_sectors) { + clu_base = 2; + map_i = 0; + } + map_b = 0; + } + } + + return CLUSTER_32(~0); } s32 exfat_alloc_cluster(struct super_block *sb, s32 num_alloc, @@ -251,22 +279,22 @@ s32 exfat_alloc_cluster(struct super_block *sb, s32 num_alloc, } } - if (set_alloc_bitmap(sb, new_clu - 2) != FFS_SUCCESS) - return -1; + if (set_alloc_bitmap(sb, new_clu - 2) != 0) + return -EIO; num_clusters++; if (p_chain->flags == 0x01) { - if (FAT_write(sb, new_clu, CLUSTER_32(~0)) < 0) - return -1; + if (exfat_fat_write(sb, new_clu, CLUSTER_32(~0)) < 0) + return -EIO; } if (p_chain->dir == CLUSTER_32(~0)) { p_chain->dir = new_clu; } else { if (p_chain->flags == 0x01) { - if (FAT_write(sb, last_clu, new_clu) < 0) - return -1; + if (exfat_fat_write(sb, last_clu, new_clu) < 0) + return -EIO; } } last_clu = new_clu; @@ -300,47 +328,6 @@ s32 exfat_alloc_cluster(struct super_block *sb, s32 num_alloc, return num_clusters; } -void fat_free_cluster(struct super_block *sb, struct chain_t *p_chain, - s32 do_relse) -{ - s32 num_clusters = 0; - u32 clu, prev; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - int i; - sector_t sector; - - if ((p_chain->dir == CLUSTER_32(0)) || (p_chain->dir == CLUSTER_32(~0))) - return; - __set_sb_dirty(sb); - clu = p_chain->dir; - - if (p_chain->size <= 0) - return; - - do { - if (p_fs->dev_ejected) - break; - - if (do_relse) { - sector = START_SECTOR(clu); - for (i = 0; i < p_fs->sectors_per_clu; i++) - buf_release(sb, sector + i); - } - - prev = clu; - if (FAT_read(sb, clu, &clu) == -1) - break; - - if (FAT_write(sb, prev, CLUSTER_32(0)) < 0) - break; - num_clusters++; - - } while (clu != CLUSTER_32(~0)); - - if (p_fs->used_clusters != UINT_MAX) - p_fs->used_clusters -= num_clusters; -} - void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, s32 do_relse) { @@ -367,10 +354,10 @@ void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, if (do_relse) { sector = START_SECTOR(clu); for (i = 0; i < p_fs->sectors_per_clu; i++) - buf_release(sb, sector + i); + exfat_buf_release(sb, sector + i); } - if (clr_alloc_bitmap(sb, clu - 2) != FFS_SUCCESS) + if (clr_alloc_bitmap(sb, clu - 2) != 0) break; clu++; @@ -384,13 +371,13 @@ void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, if (do_relse) { sector = START_SECTOR(clu); for (i = 0; i < p_fs->sectors_per_clu; i++) - buf_release(sb, sector + i); + exfat_buf_release(sb, sector + i); } - if (clr_alloc_bitmap(sb, clu - 2) != FFS_SUCCESS) + if (clr_alloc_bitmap(sb, clu - 2) != 0) break; - if (FAT_read(sb, clu, &clu) == -1) + if (exfat_fat_read(sb, clu, &clu) == -1) break; num_clusters++; } while ((clu != CLUSTER_32(0)) && (clu != CLUSTER_32(~0))); @@ -400,7 +387,7 @@ void exfat_free_cluster(struct super_block *sb, struct chain_t *p_chain, p_fs->used_clusters -= num_clusters; } -u32 find_last_cluster(struct super_block *sb, struct chain_t *p_chain) +static u32 find_last_cluster(struct super_block *sb, struct chain_t *p_chain) { u32 clu, next; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); @@ -410,7 +397,7 @@ u32 find_last_cluster(struct super_block *sb, struct chain_t *p_chain) if (p_chain->flags == 0x03) { clu += p_chain->size - 1; } else { - while ((FAT_read(sb, clu, &next) == 0) && + while ((exfat_fat_read(sb, clu, &next) == 0) && (next != CLUSTER_32(~0))) { if (p_fs->dev_ejected) break; @@ -437,7 +424,7 @@ s32 count_num_clusters(struct super_block *sb, struct chain_t *p_chain) } else { for (i = 2; i < p_fs->num_clusters; i++) { count++; - if (FAT_read(sb, clu, &clu) != 0) + if (exfat_fat_read(sb, clu, &clu) != 0) return 0; if (clu == CLUSTER_32(~0)) break; @@ -447,22 +434,6 @@ s32 count_num_clusters(struct super_block *sb, struct chain_t *p_chain) return count; } -s32 fat_count_used_clusters(struct super_block *sb) -{ - int i, count = 0; - u32 clu; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - for (i = 2; i < p_fs->num_clusters; i++) { - if (FAT_read(sb, i, &clu) != 0) - break; - if (clu != CLUSTER_32(0)) - count++; - } - - return count; -} - s32 exfat_count_used_clusters(struct super_block *sb) { int i, map_i, map_b, count = 0; @@ -470,7 +441,8 @@ s32 exfat_count_used_clusters(struct super_block *sb) struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - map_i = map_b = 0; + map_i = 0; + map_b = 0; for (i = 2; i < p_fs->num_clusters; i += 8) { k = *(((u8 *)p_fs->vol_amap[map_i]->b_data) + map_b); @@ -491,12 +463,12 @@ void exfat_chain_cont_cluster(struct super_block *sb, u32 chain, s32 len) return; while (len > 1) { - if (FAT_write(sb, chain, chain + 1) < 0) + if (exfat_fat_write(sb, chain, chain + 1) < 0) break; chain++; len--; } - FAT_write(sb, chain, CLUSTER_32(~0)); + exfat_fat_write(sb, chain, CLUSTER_32(~0)); } /* @@ -525,9 +497,9 @@ s32 load_alloc_bitmap(struct super_block *sb) ep = (struct bmap_dentry_t *)get_entry_in_dir(sb, &clu, i, NULL); if (!ep) - return FFS_MEDIAERR; + return -ENOENT; - type = p_fs->fs_func->get_entry_type((struct dentry_t *)ep); + type = exfat_get_entry_type((struct dentry_t *)ep); if (type == TYPE_UNUSED) break; @@ -544,14 +516,14 @@ s32 load_alloc_bitmap(struct super_block *sb) sizeof(struct buffer_head *), GFP_KERNEL); if (!p_fs->vol_amap) - return FFS_MEMORYERR; + return -ENOMEM; sector = START_SECTOR(p_fs->map_clu); for (j = 0; j < p_fs->map_sectors; j++) { p_fs->vol_amap[j] = NULL; - ret = sector_read(sb, sector + j, &(p_fs->vol_amap[j]), 1); - if (ret != FFS_SUCCESS) { + ret = sector_read(sb, sector + j, &p_fs->vol_amap[j], 1); + if (ret != 0) { /* release all buffers and free vol_amap */ i = 0; while (i < j) @@ -564,15 +536,15 @@ s32 load_alloc_bitmap(struct super_block *sb) } p_fs->pbr_bh = NULL; - return FFS_SUCCESS; + return 0; } } - if (FAT_read(sb, clu.dir, &clu.dir) != 0) - return FFS_MEDIAERR; + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) + return -EIO; } - return FFS_FORMATERR; + return -EFSCORRUPTED; } void free_alloc_bitmap(struct super_block *sb) @@ -589,97 +561,6 @@ void free_alloc_bitmap(struct super_block *sb) p_fs->vol_amap = NULL; } -s32 set_alloc_bitmap(struct super_block *sb, u32 clu) -{ - int i, b; - sector_t sector; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - - i = clu >> (p_bd->sector_size_bits + 3); - b = clu & ((p_bd->sector_size << 3) - 1); - - sector = START_SECTOR(p_fs->map_clu) + i; - - exfat_bitmap_set((u8 *)p_fs->vol_amap[i]->b_data, b); - - return sector_write(sb, sector, p_fs->vol_amap[i], 0); -} - -s32 clr_alloc_bitmap(struct super_block *sb, u32 clu) -{ - int i, b; - sector_t sector; -#ifdef CONFIG_EXFAT_DISCARD - struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct exfat_mount_options *opts = &sbi->options; - int ret; -#endif /* CONFIG_EXFAT_DISCARD */ - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - - i = clu >> (p_bd->sector_size_bits + 3); - b = clu & ((p_bd->sector_size << 3) - 1); - - sector = START_SECTOR(p_fs->map_clu) + i; - - exfat_bitmap_clear((u8 *)p_fs->vol_amap[i]->b_data, b); - - return sector_write(sb, sector, p_fs->vol_amap[i], 0); - -#ifdef CONFIG_EXFAT_DISCARD - if (opts->discard) { - ret = sb_issue_discard(sb, START_SECTOR(clu), - (1 << p_fs->sectors_per_clu_bits), - GFP_NOFS, 0); - if (ret == -EOPNOTSUPP) { - pr_warn("discard not supported by device, disabling"); - opts->discard = 0; - } - } -#endif /* CONFIG_EXFAT_DISCARD */ -} - -u32 test_alloc_bitmap(struct super_block *sb, u32 clu) -{ - int i, map_i, map_b; - u32 clu_base, clu_free; - u8 k, clu_mask; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - - clu_base = (clu & ~(0x7)) + 2; - clu_mask = (1 << (clu - clu_base + 2)) - 1; - - map_i = clu >> (p_bd->sector_size_bits + 3); - map_b = (clu >> 3) & p_bd->sector_size_mask; - - for (i = 2; i < p_fs->num_clusters; i += 8) { - k = *(((u8 *)p_fs->vol_amap[map_i]->b_data) + map_b); - if (clu_mask > 0) { - k |= clu_mask; - clu_mask = 0; - } - if (k < 0xFF) { - clu_free = clu_base + free_bit[k]; - if (clu_free < p_fs->num_clusters) - return clu_free; - } - clu_base += 8; - - if (((++map_b) >= p_bd->sector_size) || - (clu_base >= p_fs->num_clusters)) { - if ((++map_i) >= p_fs->map_sectors) { - clu_base = 2; - map_i = 0; - } - map_b = 0; - } - } - - return CLUSTER_32(~0); -} - void sync_alloc_bitmap(struct super_block *sb) { int i; @@ -698,7 +579,7 @@ void sync_alloc_bitmap(struct super_block *sb) static s32 __load_upcase_table(struct super_block *sb, sector_t sector, u32 num_sectors, u32 utbl_checksum) { - int i, ret = FFS_ERROR; + int i, ret = -EINVAL; u32 j; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); @@ -712,15 +593,15 @@ static s32 __load_upcase_table(struct super_block *sb, sector_t sector, u32 checksum = 0; - upcase_table = p_fs->vol_utbl = kmalloc(UTBL_COL_COUNT * sizeof(u16 *), - GFP_KERNEL); + upcase_table = kmalloc_array(UTBL_COL_COUNT, sizeof(u16 *), GFP_KERNEL); + p_fs->vol_utbl = upcase_table; if (!upcase_table) - return FFS_MEMORYERR; + return -ENOMEM; memset(upcase_table, 0, UTBL_COL_COUNT * sizeof(u16 *)); while (sector < end_sector) { ret = sector_read(sb, sector, &tmp_bh, 1); - if (ret != FFS_SUCCESS) { + if (ret != 0) { pr_debug("sector read (0x%llX)fail\n", (unsigned long long)sector); goto error; @@ -755,7 +636,7 @@ static s32 __load_upcase_table(struct super_block *sb, sector_t sector, upcase_table[col_index] = kmalloc_array(UTBL_ROW_COUNT, sizeof(u16), GFP_KERNEL); if (!upcase_table[col_index]) { - ret = FFS_MEMORYERR; + ret = -ENOMEM; goto error; } @@ -771,9 +652,9 @@ static s32 __load_upcase_table(struct super_block *sb, sector_t sector, if (index >= 0xFFFF && utbl_checksum == checksum) { if (tmp_bh) brelse(tmp_bh); - return FFS_SUCCESS; + return 0; } - ret = FFS_ERROR; + ret = -EINVAL; error: if (tmp_bh) brelse(tmp_bh); @@ -783,7 +664,7 @@ static s32 __load_upcase_table(struct super_block *sb, sector_t sector, static s32 __load_default_upcase_table(struct super_block *sb) { - int i, ret = FFS_ERROR; + int i, ret = -EINVAL; u32 j; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); @@ -792,10 +673,10 @@ static s32 __load_default_upcase_table(struct super_block *sb) u16 uni = 0; u16 **upcase_table; - upcase_table = p_fs->vol_utbl = kmalloc(UTBL_COL_COUNT * sizeof(u16 *), - GFP_KERNEL); + upcase_table = kmalloc_array(UTBL_COL_COUNT, sizeof(u16 *), GFP_KERNEL); + p_fs->vol_utbl = upcase_table; if (!upcase_table) - return FFS_MEMORYERR; + return -ENOMEM; memset(upcase_table, 0, UTBL_COL_COUNT * sizeof(u16 *)); for (i = 0; index <= 0xFFFF && i < NUM_UPCASE * 2; i += 2) { @@ -818,7 +699,7 @@ static s32 __load_default_upcase_table(struct super_block *sb) sizeof(u16), GFP_KERNEL); if (!upcase_table[col_index]) { - ret = FFS_MEMORYERR; + ret = -ENOMEM; goto error; } @@ -832,7 +713,7 @@ static s32 __load_default_upcase_table(struct super_block *sb) } if (index >= 0xFFFF) - return FFS_SUCCESS; + return 0; error: /* FATAL error: default upcase table has error */ @@ -855,16 +736,16 @@ s32 load_upcase_table(struct super_block *sb) clu.flags = 0x01; if (p_fs->dev_ejected) - return FFS_MEDIAERR; + return -EIO; while (clu.dir != CLUSTER_32(~0)) { for (i = 0; i < p_fs->dentries_per_clu; i++) { ep = (struct case_dentry_t *)get_entry_in_dir(sb, &clu, i, NULL); if (!ep) - return FFS_MEDIAERR; + return -ENOENT; - type = p_fs->fs_func->get_entry_type((struct dentry_t *)ep); + type = exfat_get_entry_type((struct dentry_t *)ep); if (type == TYPE_UNUSED) break; @@ -877,12 +758,12 @@ s32 load_upcase_table(struct super_block *sb) sector = START_SECTOR(tbl_clu); num_sectors = ((tbl_size - 1) >> p_bd->sector_size_bits) + 1; if (__load_upcase_table(sb, sector, num_sectors, - GET32_A(ep->checksum)) != FFS_SUCCESS) + GET32_A(ep->checksum)) != 0) break; - return FFS_SUCCESS; + return 0; } - if (FAT_read(sb, clu.dir, &clu.dir) != 0) - return FFS_MEDIAERR; + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) + return -EIO; } /* load default upcase table */ return __load_default_upcase_table(sb); @@ -906,28 +787,6 @@ void free_upcase_table(struct super_block *sb) * Directory Entry Management Functions */ -u32 fat_get_entry_type(struct dentry_t *p_entry) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - if (*(ep->name) == 0x0) - return TYPE_UNUSED; - - else if (*(ep->name) == 0xE5) - return TYPE_DELETED; - - else if (ep->attr == ATTR_EXTEND) - return TYPE_EXTEND; - - else if ((ep->attr & (ATTR_SUBDIR | ATTR_VOLUME)) == ATTR_VOLUME) - return TYPE_VOLUME; - - else if ((ep->attr & (ATTR_SUBDIR | ATTR_VOLUME)) == ATTR_SUBDIR) - return TYPE_DIR; - - return TYPE_FILE; -} - u32 exfat_get_entry_type(struct dentry_t *p_entry) { struct file_dentry_t *ep = (struct file_dentry_t *)p_entry; @@ -973,30 +832,7 @@ u32 exfat_get_entry_type(struct dentry_t *p_entry) return TYPE_BENIGN_SEC; } -void fat_set_entry_type(struct dentry_t *p_entry, u32 type) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - if (type == TYPE_UNUSED) - *(ep->name) = 0x0; - - else if (type == TYPE_DELETED) - *(ep->name) = 0xE5; - - else if (type == TYPE_EXTEND) - ep->attr = ATTR_EXTEND; - - else if (type == TYPE_DIR) - ep->attr = ATTR_SUBDIR; - - else if (type == TYPE_FILE) - ep->attr = ATTR_ARCHIVE; - - else if (type == TYPE_SYMLINK) - ep->attr = ATTR_ARCHIVE | ATTR_SYMLINK; -} - -void exfat_set_entry_type(struct dentry_t *p_entry, u32 type) +static void exfat_set_entry_type(struct dentry_t *p_entry, u32 type) { struct file_dentry_t *ep = (struct file_dentry_t *)p_entry; @@ -1026,13 +862,6 @@ void exfat_set_entry_type(struct dentry_t *p_entry, u32 type) } } -u32 fat_get_entry_attr(struct dentry_t *p_entry) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - return (u32)ep->attr; -} - u32 exfat_get_entry_attr(struct dentry_t *p_entry) { struct file_dentry_t *ep = (struct file_dentry_t *)p_entry; @@ -1040,13 +869,6 @@ u32 exfat_get_entry_attr(struct dentry_t *p_entry) return (u32)GET16_A(ep->attr); } -void fat_set_entry_attr(struct dentry_t *p_entry, u32 attr) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - ep->attr = (u8)attr; -} - void exfat_set_entry_attr(struct dentry_t *p_entry, u32 attr) { struct file_dentry_t *ep = (struct file_dentry_t *)p_entry; @@ -1054,11 +876,6 @@ void exfat_set_entry_attr(struct dentry_t *p_entry, u32 attr) SET16_A(ep->attr, (u16)attr); } -u8 fat_get_entry_flag(struct dentry_t *p_entry) -{ - return 0x01; -} - u8 exfat_get_entry_flag(struct dentry_t *p_entry) { struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; @@ -1066,10 +883,6 @@ u8 exfat_get_entry_flag(struct dentry_t *p_entry) return ep->flags; } -void fat_set_entry_flag(struct dentry_t *p_entry, u8 flags) -{ -} - void exfat_set_entry_flag(struct dentry_t *p_entry, u8 flags) { struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; @@ -1077,29 +890,13 @@ void exfat_set_entry_flag(struct dentry_t *p_entry, u8 flags) ep->flags = flags; } -u32 fat_get_entry_clu0(struct dentry_t *p_entry) +u32 exfat_get_entry_clu0(struct dentry_t *p_entry) { - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - return ((u32)GET16_A(ep->start_clu_hi) << 16) | - GET16_A(ep->start_clu_lo); -} - -u32 exfat_get_entry_clu0(struct dentry_t *p_entry) -{ - struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; + struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; return GET32_A(ep->start_clu); } -void fat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - SET16_A(ep->start_clu_lo, CLUSTER_16(start_clu)); - SET16_A(ep->start_clu_hi, CLUSTER_16(start_clu >> 16)); -} - void exfat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu) { struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; @@ -1107,13 +904,6 @@ void exfat_set_entry_clu0(struct dentry_t *p_entry, u32 start_clu) SET32_A(ep->start_clu, start_clu); } -u64 fat_get_entry_size(struct dentry_t *p_entry) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - return (u64)GET32_A(ep->size); -} - u64 exfat_get_entry_size(struct dentry_t *p_entry) { struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; @@ -1121,13 +911,6 @@ u64 exfat_get_entry_size(struct dentry_t *p_entry) return GET64_A(ep->valid_size); } -void fat_set_entry_size(struct dentry_t *p_entry, u64 size) -{ - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - SET32_A(ep->size, (u32)size); -} - void exfat_set_entry_size(struct dentry_t *p_entry, u64 size) { struct strm_dentry_t *ep = (struct strm_dentry_t *)p_entry; @@ -1136,31 +919,6 @@ void exfat_set_entry_size(struct dentry_t *p_entry, u64 size) SET64_A(ep->size, size); } -void fat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode) -{ - u16 t = 0x00, d = 0x21; - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - switch (mode) { - case TM_CREATE: - t = GET16_A(ep->create_time); - d = GET16_A(ep->create_date); - break; - case TM_MODIFY: - t = GET16_A(ep->modify_time); - d = GET16_A(ep->modify_date); - break; - } - - tp->sec = (t & 0x001F) << 1; - tp->min = (t >> 5) & 0x003F; - tp->hour = (t >> 11); - tp->day = (d & 0x001F); - tp->mon = (d >> 5) & 0x000F; - tp->year = (d >> 9); -} - void exfat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, u8 mode) { @@ -1190,27 +948,6 @@ void exfat_get_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, tp->year = (d >> 9); } -void fat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, - u8 mode) -{ - u16 t, d; - struct dos_dentry_t *ep = (struct dos_dentry_t *)p_entry; - - t = (tp->hour << 11) | (tp->min << 5) | (tp->sec >> 1); - d = (tp->year << 9) | (tp->mon << 5) | tp->day; - - switch (mode) { - case TM_CREATE: - SET16_A(ep->create_time, t); - SET16_A(ep->create_date, d); - break; - case TM_MODIFY: - SET16_A(ep->modify_time, t); - SET16_A(ep->modify_date, d); - break; - } -} - void exfat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, u8 mode) { @@ -1236,25 +973,47 @@ void exfat_set_entry_time(struct dentry_t *p_entry, struct timestamp_t *tp, } } -s32 fat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, s32 entry, - u32 type, u32 start_clu, u64 size) +static void init_file_entry(struct file_dentry_t *ep, u32 type) { - sector_t sector; - struct dos_dentry_t *dos_ep; + struct timestamp_t tm, *tp; + + exfat_set_entry_type((struct dentry_t *)ep, type); + + tp = tm_current(&tm); + exfat_set_entry_time((struct dentry_t *)ep, tp, TM_CREATE); + exfat_set_entry_time((struct dentry_t *)ep, tp, TM_MODIFY); + exfat_set_entry_time((struct dentry_t *)ep, tp, TM_ACCESS); + ep->create_time_ms = 0; + ep->modify_time_ms = 0; + ep->access_time_ms = 0; +} + +static void init_strm_entry(struct strm_dentry_t *ep, u8 flags, u32 start_clu, u64 size) +{ + exfat_set_entry_type((struct dentry_t *)ep, TYPE_STREAM); + ep->flags = flags; + SET32_A(ep->start_clu, start_clu); + SET64_A(ep->valid_size, size); + SET64_A(ep->size, size); +} - dos_ep = (struct dos_dentry_t *)get_entry_in_dir(sb, p_dir, entry, - §or); - if (!dos_ep) - return FFS_MEDIAERR; +static void init_name_entry(struct name_dentry_t *ep, u16 *uniname) +{ + int i; - init_dos_entry(dos_ep, type, start_clu); - buf_modify(sb, sector); + exfat_set_entry_type((struct dentry_t *)ep, TYPE_EXTEND); + ep->flags = 0x0; - return FFS_SUCCESS; + for (i = 0; i < 30; i++, i++) { + SET16_A(ep->unicode_0_14 + i, *uniname); + if (*uniname == 0x0) + break; + uniname++; + } } -s32 exfat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, u32 type, u32 start_clu, u64 size) +static s32 exfat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, + s32 entry, u32 type, u32 start_clu, u64 size) { sector_t sector; u8 flags; @@ -1267,71 +1026,20 @@ s32 exfat_init_dir_entry(struct super_block *sb, struct chain_t *p_dir, file_ep = (struct file_dentry_t *)get_entry_in_dir(sb, p_dir, entry, §or); if (!file_ep) - return FFS_MEDIAERR; + return -ENOENT; strm_ep = (struct strm_dentry_t *)get_entry_in_dir(sb, p_dir, entry + 1, §or); if (!strm_ep) - return FFS_MEDIAERR; + return -ENOENT; init_file_entry(file_ep, type); - buf_modify(sb, sector); + exfat_buf_modify(sb, sector); init_strm_entry(strm_ep, flags, start_clu, size); - buf_modify(sb, sector); - - return FFS_SUCCESS; -} - -static s32 fat_init_ext_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 num_entries, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname) -{ - int i; - sector_t sector; - u8 chksum; - u16 *uniname = p_uniname->name; - struct dos_dentry_t *dos_ep; - struct ext_dentry_t *ext_ep; - - dos_ep = (struct dos_dentry_t *)get_entry_in_dir(sb, p_dir, entry, - §or); - if (!dos_ep) - return FFS_MEDIAERR; - - dos_ep->lcase = p_dosname->name_case; - memcpy(dos_ep->name, p_dosname->name, DOS_NAME_LENGTH); - buf_modify(sb, sector); - - if ((--num_entries) > 0) { - chksum = calc_checksum_1byte((void *)dos_ep->name, - DOS_NAME_LENGTH, 0); - - for (i = 1; i < num_entries; i++) { - ext_ep = (struct ext_dentry_t *)get_entry_in_dir(sb, - p_dir, - entry - i, - §or); - if (!ext_ep) - return FFS_MEDIAERR; - - init_ext_entry(ext_ep, i, chksum, uniname); - buf_modify(sb, sector); - uniname += 13; - } - - ext_ep = (struct ext_dentry_t *)get_entry_in_dir(sb, p_dir, - entry - i, - §or); - if (!ext_ep) - return FFS_MEDIAERR; - - init_ext_entry(ext_ep, i + 0x40, chksum, uniname); - buf_modify(sb, sector); - } + exfat_buf_modify(sb, sector); - return FFS_SUCCESS; + return 0; } static s32 exfat_init_ext_entry(struct super_block *sb, struct chain_t *p_dir, @@ -1349,173 +1057,51 @@ static s32 exfat_init_ext_entry(struct super_block *sb, struct chain_t *p_dir, file_ep = (struct file_dentry_t *)get_entry_in_dir(sb, p_dir, entry, §or); if (!file_ep) - return FFS_MEDIAERR; + return -ENOENT; file_ep->num_ext = (u8)(num_entries - 1); - buf_modify(sb, sector); + exfat_buf_modify(sb, sector); strm_ep = (struct strm_dentry_t *)get_entry_in_dir(sb, p_dir, entry + 1, §or); if (!strm_ep) - return FFS_MEDIAERR; + return -ENOENT; strm_ep->name_len = p_uniname->name_len; SET16_A(strm_ep->name_hash, p_uniname->name_hash); - buf_modify(sb, sector); + exfat_buf_modify(sb, sector); for (i = 2; i < num_entries; i++) { name_ep = (struct name_dentry_t *)get_entry_in_dir(sb, p_dir, entry + i, §or); if (!name_ep) - return FFS_MEDIAERR; + return -ENOENT; init_name_entry(name_ep, uniname); - buf_modify(sb, sector); + exfat_buf_modify(sb, sector); uniname += 15; } update_dir_checksum(sb, p_dir, entry); - return FFS_SUCCESS; -} - -void init_dos_entry(struct dos_dentry_t *ep, u32 type, u32 start_clu) -{ - struct timestamp_t tm, *tp; - - fat_set_entry_type((struct dentry_t *)ep, type); - SET16_A(ep->start_clu_lo, CLUSTER_16(start_clu)); - SET16_A(ep->start_clu_hi, CLUSTER_16(start_clu >> 16)); - SET32_A(ep->size, 0); - - tp = tm_current(&tm); - fat_set_entry_time((struct dentry_t *)ep, tp, TM_CREATE); - fat_set_entry_time((struct dentry_t *)ep, tp, TM_MODIFY); - SET16_A(ep->access_date, 0); - ep->create_time_ms = 0; -} - -void init_ext_entry(struct ext_dentry_t *ep, s32 order, u8 chksum, u16 *uniname) -{ - int i; - bool end = false; - - fat_set_entry_type((struct dentry_t *)ep, TYPE_EXTEND); - ep->order = (u8)order; - ep->sysid = 0; - ep->checksum = chksum; - SET16_A(ep->start_clu, 0); - - for (i = 0; i < 10; i += 2) { - if (!end) { - SET16(ep->unicode_0_4 + i, *uniname); - if (*uniname == 0x0) - end = true; - else - uniname++; - } else { - SET16(ep->unicode_0_4 + i, 0xFFFF); - } - } - - for (i = 0; i < 12; i += 2) { - if (!end) { - SET16_A(ep->unicode_5_10 + i, *uniname); - if (*uniname == 0x0) - end = true; - else - uniname++; - } else { - SET16_A(ep->unicode_5_10 + i, 0xFFFF); - } - } - - for (i = 0; i < 4; i += 2) { - if (!end) { - SET16_A(ep->unicode_11_12 + i, *uniname); - if (*uniname == 0x0) - end = true; - else - uniname++; - } else { - SET16_A(ep->unicode_11_12 + i, 0xFFFF); - } - } -} - -void init_file_entry(struct file_dentry_t *ep, u32 type) -{ - struct timestamp_t tm, *tp; - - exfat_set_entry_type((struct dentry_t *)ep, type); - - tp = tm_current(&tm); - exfat_set_entry_time((struct dentry_t *)ep, tp, TM_CREATE); - exfat_set_entry_time((struct dentry_t *)ep, tp, TM_MODIFY); - exfat_set_entry_time((struct dentry_t *)ep, tp, TM_ACCESS); - ep->create_time_ms = 0; - ep->modify_time_ms = 0; - ep->access_time_ms = 0; -} - -void init_strm_entry(struct strm_dentry_t *ep, u8 flags, u32 start_clu, u64 size) -{ - exfat_set_entry_type((struct dentry_t *)ep, TYPE_STREAM); - ep->flags = flags; - SET32_A(ep->start_clu, start_clu); - SET64_A(ep->valid_size, size); - SET64_A(ep->size, size); -} - -void init_name_entry(struct name_dentry_t *ep, u16 *uniname) -{ - int i; - - exfat_set_entry_type((struct dentry_t *)ep, TYPE_EXTEND); - ep->flags = 0x0; - - for (i = 0; i < 30; i++, i++) { - SET16_A(ep->unicode_0_14 + i, *uniname); - if (*uniname == 0x0) - break; - uniname++; - } -} - -void fat_delete_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 order, s32 num_entries) -{ - int i; - sector_t sector; - struct dentry_t *ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - for (i = num_entries - 1; i >= order; i--) { - ep = get_entry_in_dir(sb, p_dir, entry - i, §or); - if (!ep) - return; - - p_fs->fs_func->set_entry_type(ep, TYPE_DELETED); - buf_modify(sb, sector); - } + return 0; } void exfat_delete_dir_entry(struct super_block *sb, struct chain_t *p_dir, - s32 entry, s32 order, s32 num_entries) + s32 entry, s32 order, s32 num_entries) { int i; sector_t sector; struct dentry_t *ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); for (i = order; i < num_entries; i++) { ep = get_entry_in_dir(sb, p_dir, entry + i, §or); if (!ep) return; - p_fs->fs_func->set_entry_type(ep, TYPE_DELETED); - buf_modify(sb, sector); + exfat_set_entry_type(ep, TYPE_DELETED); + exfat_buf_modify(sb, sector); } } @@ -1533,7 +1119,7 @@ void update_dir_checksum(struct super_block *sb, struct chain_t *p_dir, if (!file_ep) return; - buf_lock(sb, sector); + exfat_buf_lock(sb, sector); num_entries = (s32)file_ep->num_ext + 1; chksum = calc_checksum_2byte((void *)file_ep, DENTRY_SIZE, 0, @@ -1542,7 +1128,7 @@ void update_dir_checksum(struct super_block *sb, struct chain_t *p_dir, for (i = 1; i < num_entries; i++) { ep = get_entry_in_dir(sb, p_dir, entry + i, NULL); if (!ep) { - buf_unlock(sb, sector); + exfat_buf_unlock(sb, sector); return; } @@ -1551,8 +1137,75 @@ void update_dir_checksum(struct super_block *sb, struct chain_t *p_dir, } SET16_A(file_ep->checksum, chksum); - buf_modify(sb, sector); - buf_unlock(sb, sector); + exfat_buf_modify(sb, sector); + exfat_buf_unlock(sb, sector); +} + +static s32 __write_partial_entries_in_entry_set(struct super_block *sb, + struct entry_set_cache_t *es, + sector_t sec, s32 off, u32 count) +{ + s32 num_entries, buf_off = (off - es->offset); + u32 remaining_byte_in_sector, copy_entries; + struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); + struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); + u32 clu; + u8 *buf, *esbuf = (u8 *)&es->__buf; + + pr_debug("%s entered es %p sec %llu off %d count %d\n", + __func__, es, (unsigned long long)sec, off, count); + num_entries = count; + + while (num_entries) { + /* white per sector base */ + remaining_byte_in_sector = (1 << p_bd->sector_size_bits) - off; + copy_entries = min_t(s32, + remaining_byte_in_sector >> DENTRY_SIZE_BITS, + num_entries); + buf = exfat_buf_getblk(sb, sec); + if (!buf) + goto err_out; + pr_debug("es->buf %p buf_off %u\n", esbuf, buf_off); + pr_debug("copying %d entries from %p to sector %llu\n", + copy_entries, (esbuf + buf_off), + (unsigned long long)sec); + memcpy(buf + off, esbuf + buf_off, + copy_entries << DENTRY_SIZE_BITS); + exfat_buf_modify(sb, sec); + num_entries -= copy_entries; + + if (num_entries) { + /* get next sector */ + if (IS_LAST_SECTOR_IN_CLUSTER(sec)) { + clu = GET_CLUSTER_FROM_SECTOR(sec); + if (es->alloc_flag == 0x03) { + clu++; + } else { + if (exfat_fat_read(sb, clu, &clu) == -1) + goto err_out; + } + sec = START_SECTOR(clu); + } else { + sec++; + } + off = 0; + buf_off += copy_entries << DENTRY_SIZE_BITS; + } + } + + pr_debug("%s exited successfully\n", __func__); + return 0; +err_out: + pr_debug("%s failed\n", __func__); + return -EINVAL; +} + +/* write back all entries in entry set */ +static s32 write_whole_entry_set(struct super_block *sb, struct entry_set_cache_t *es) +{ + return __write_partial_entries_in_entry_set(sb, es, es->sector, + es->offset, + es->num_entries); } void update_dir_checksum_with_entry_set(struct super_block *sb, @@ -1562,7 +1215,7 @@ void update_dir_checksum_with_entry_set(struct super_block *sb, u16 chksum = 0; s32 chksum_type = CS_DIR_ENTRY, i; - ep = (struct dentry_t *)&(es->__buf); + ep = (struct dentry_t *)&es->__buf; for (i = 0; i < es->num_entries; i++) { pr_debug("%s ep %p\n", __func__, ep); chksum = calc_checksum_2byte((void *)ep, DENTRY_SIZE, chksum, @@ -1571,7 +1224,7 @@ void update_dir_checksum_with_entry_set(struct super_block *sb, chksum_type = CS_DEFAULT; } - ep = (struct dentry_t *)&(es->__buf); + ep = (struct dentry_t *)&es->__buf; SET16_A(((struct file_dentry_t *)ep)->checksum, chksum); write_whole_entry_set(sb, es); } @@ -1590,19 +1243,19 @@ static s32 _walk_fat_chain(struct super_block *sb, struct chain_t *p_dir, cur_clu += clu_offset; } else { while (clu_offset > 0) { - if (FAT_read(sb, cur_clu, &cur_clu) == -1) - return FFS_MEDIAERR; + if (exfat_fat_read(sb, cur_clu, &cur_clu) == -1) + return -EIO; clu_offset--; } } if (clu) *clu = cur_clu; - return FFS_SUCCESS; + return 0; } -s32 find_location(struct super_block *sb, struct chain_t *p_dir, s32 entry, - sector_t *sector, s32 *offset) +static s32 find_location(struct super_block *sb, struct chain_t *p_dir, s32 entry, + sector_t *sector, s32 *offset) { s32 off, ret; u32 clu = 0; @@ -1617,7 +1270,7 @@ s32 find_location(struct super_block *sb, struct chain_t *p_dir, s32 entry, *sector += p_fs->root_start_sector; } else { ret = _walk_fat_chain(sb, p_dir, off, &clu); - if (ret != FFS_SUCCESS) + if (ret != 0) return ret; /* byte offset in cluster */ @@ -1630,20 +1283,7 @@ s32 find_location(struct super_block *sb, struct chain_t *p_dir, s32 entry, *sector = off >> p_bd->sector_size_bits; *sector += START_SECTOR(clu); } - return FFS_SUCCESS; -} - -struct dentry_t *get_entry_with_sector(struct super_block *sb, sector_t sector, - s32 offset) -{ - u8 *buf; - - buf = buf_getblk(sb, sector); - - if (!buf) - return NULL; - - return (struct dentry_t *)(buf + offset); + return 0; } struct dentry_t *get_entry_in_dir(struct super_block *sb, struct chain_t *p_dir, @@ -1653,10 +1293,10 @@ struct dentry_t *get_entry_in_dir(struct super_block *sb, struct chain_t *p_dir, sector_t sec; u8 *buf; - if (find_location(sb, p_dir, entry, &sec, &off) != FFS_SUCCESS) + if (find_location(sb, p_dir, entry, &sec, &off) != 0) return NULL; - buf = buf_getblk(sb, sec); + buf = exfat_buf_getblk(sb, sec); if (!buf) return NULL; @@ -1703,11 +1343,11 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, size_t bufsize; pr_debug("%s entered p_dir dir %u flags %x size %d\n", - __func__, p_dir->dir, p_dir->flags, p_dir->size); + __func__, p_dir->dir, p_dir->flags, p_dir->size); byte_offset = entry << DENTRY_SIZE_BITS; ret = _walk_fat_chain(sb, p_dir, byte_offset, &clu); - if (ret != FFS_SUCCESS) + if (ret != 0) return NULL; /* byte offset in cluster */ @@ -1720,15 +1360,14 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, sec = byte_offset >> p_bd->sector_size_bits; sec += START_SECTOR(clu); - buf = buf_getblk(sb, sec); + buf = exfat_buf_getblk(sb, sec); if (!buf) goto err_out; ep = (struct dentry_t *)(buf + off); - entry_type = p_fs->fs_func->get_entry_type(ep); + entry_type = exfat_get_entry_type(ep); - if ((entry_type != TYPE_FILE) - && (entry_type != TYPE_DIR)) + if ((entry_type != TYPE_FILE) && (entry_type != TYPE_DIR)) goto err_out; if (type == ES_ALL_ENTRIES) @@ -1756,7 +1395,7 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, * instead of copying whole sector, we will check every entry. * this will provide minimum stablity and consistency. */ - entry_type = p_fs->fs_func->get_entry_type(ep); + entry_type = exfat_get_entry_type(ep); if ((entry_type == TYPE_UNUSED) || (entry_type == TYPE_DELETED)) goto err_out; @@ -1812,14 +1451,14 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, if (es->alloc_flag == 0x03) { clu++; } else { - if (FAT_read(sb, clu, &clu) == -1) + if (exfat_fat_read(sb, clu, &clu) == -1) goto err_out; } sec = START_SECTOR(clu); } else { sec++; } - buf = buf_getblk(sb, sec); + buf = exfat_buf_getblk(sb, sec); if (!buf) goto err_out; off = 0; @@ -1832,11 +1471,11 @@ struct entry_set_cache_t *get_entry_set_in_dir(struct super_block *sb, } if (file_ep) - *file_ep = (struct dentry_t *)&(es->__buf); + *file_ep = (struct dentry_t *)&es->__buf; pr_debug("%s exiting es %p sec %llu offset %d flags %d, num_entries %u buf ptr %p\n", - __func__, es, (unsigned long long)es->sector, es->offset, - es->alloc_flag, es->num_entries, &es->__buf); + __func__, es, (unsigned long long)es->sector, es->offset, + es->alloc_flag, es->num_entries, &es->__buf); return es; err_out: pr_debug("%s exited NULL (es %p)\n", __func__, es); @@ -1850,115 +1489,10 @@ void release_entry_set(struct entry_set_cache_t *es) kfree(es); } -static s32 __write_partial_entries_in_entry_set(struct super_block *sb, - struct entry_set_cache_t *es, - sector_t sec, s32 off, u32 count) -{ - s32 num_entries, buf_off = (off - es->offset); - u32 remaining_byte_in_sector, copy_entries; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - u32 clu; - u8 *buf, *esbuf = (u8 *)&(es->__buf); - - pr_debug("%s entered es %p sec %llu off %d count %d\n", - __func__, es, (unsigned long long)sec, off, count); - num_entries = count; - - while (num_entries) { - /* white per sector base */ - remaining_byte_in_sector = (1 << p_bd->sector_size_bits) - off; - copy_entries = min_t(s32, - remaining_byte_in_sector >> DENTRY_SIZE_BITS, - num_entries); - buf = buf_getblk(sb, sec); - if (!buf) - goto err_out; - pr_debug("es->buf %p buf_off %u\n", esbuf, buf_off); - pr_debug("copying %d entries from %p to sector %llu\n", - copy_entries, (esbuf + buf_off), - (unsigned long long)sec); - memcpy(buf + off, esbuf + buf_off, - copy_entries << DENTRY_SIZE_BITS); - buf_modify(sb, sec); - num_entries -= copy_entries; - - if (num_entries) { - /* get next sector */ - if (IS_LAST_SECTOR_IN_CLUSTER(sec)) { - clu = GET_CLUSTER_FROM_SECTOR(sec); - if (es->alloc_flag == 0x03) { - clu++; - } else { - if (FAT_read(sb, clu, &clu) == -1) - goto err_out; - } - sec = START_SECTOR(clu); - } else { - sec++; - } - off = 0; - buf_off += copy_entries << DENTRY_SIZE_BITS; - } - } - - pr_debug("%s exited successfully\n", __func__); - return FFS_SUCCESS; -err_out: - pr_debug("%s failed\n", __func__); - return FFS_ERROR; -} - -/* write back all entries in entry set */ -s32 write_whole_entry_set(struct super_block *sb, struct entry_set_cache_t *es) -{ - return __write_partial_entries_in_entry_set(sb, es, es->sector, - es->offset, - es->num_entries); -} - -/* write back some entries in entry set */ -s32 write_partial_entries_in_entry_set(struct super_block *sb, - struct entry_set_cache_t *es, struct dentry_t *ep, u32 count) -{ - s32 ret, byte_offset, off; - u32 clu = 0; - sector_t sec; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - struct chain_t dir; - - /* vaidity check */ - if (ep + count > ((struct dentry_t *)&(es->__buf)) + es->num_entries) - return FFS_ERROR; - - dir.dir = GET_CLUSTER_FROM_SECTOR(es->sector); - dir.flags = es->alloc_flag; - dir.size = 0xffffffff; /* XXX */ - - byte_offset = (es->sector - START_SECTOR(dir.dir)) << - p_bd->sector_size_bits; - byte_offset += ((void **)ep - &(es->__buf)) + es->offset; - - ret = _walk_fat_chain(sb, &dir, byte_offset, &clu); - if (ret != FFS_SUCCESS) - return ret; - - /* byte offset in cluster */ - byte_offset &= p_fs->cluster_size - 1; - - /* byte offset in sector */ - off = byte_offset & p_bd->sector_size_mask; - - /* sector offset in cluster */ - sec = byte_offset >> p_bd->sector_size_bits; - sec += START_SECTOR(clu); - return __write_partial_entries_in_entry_set(sb, es, sec, off, count); -} - -/* search EMPTY CONTINUOUS "num_entries" entries */ -s32 search_deleted_or_unused_entry(struct super_block *sb, - struct chain_t *p_dir, s32 num_entries) +/* search EMPTY CONTINUOUS "num_entries" entries */ +static s32 search_deleted_or_unused_entry(struct super_block *sb, + struct chain_t *p_dir, + s32 num_entries) { int i, dentry, num_empty = 0; s32 dentries_per_clu; @@ -2005,7 +1539,7 @@ s32 search_deleted_or_unused_entry(struct super_block *sb, if (!ep) return -1; - type = p_fs->fs_func->get_entry_type(ep); + type = exfat_get_entry_type(ep); if (type == TYPE_UNUSED) { num_empty++; @@ -2043,7 +1577,7 @@ s32 search_deleted_or_unused_entry(struct super_block *sb, else clu.dir = CLUSTER_32(~0); } else { - if (FAT_read(sb, clu.dir, &clu.dir) != 0) + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) return -1; } } @@ -2051,7 +1585,7 @@ s32 search_deleted_or_unused_entry(struct super_block *sb, return -1; } -s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries) +static s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries) { s32 ret, dentry; u32 last_clu; @@ -2070,10 +1604,8 @@ s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries if (p_fs->dev_ejected) break; - if (p_fs->vol_type == EXFAT) { - if (p_dir->dir != p_fs->root_dir) - size = i_size_read(inode); - } + if (p_dir->dir != p_fs->root_dir) + size = i_size_read(inode); last_clu = find_last_cluster(sb, p_dir); clu.dir = last_clu + 1; @@ -2081,12 +1613,12 @@ s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries clu.flags = p_dir->flags; /* (1) allocate a cluster */ - ret = p_fs->fs_func->alloc_cluster(sb, 1, &clu); + ret = exfat_alloc_cluster(sb, 1, &clu); if (ret < 1) - return -1; + return -EIO; - if (clear_cluster(sb, clu.dir) != FFS_SUCCESS) - return -1; + if (clear_cluster(sb, clu.dir) != 0) + return -EIO; /* (2) append to the FAT chain */ if (clu.flags != p_dir->flags) { @@ -2095,8 +1627,8 @@ s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries p_fs->hint_uentry.clu.flags = 0x01; } if (clu.flags == 0x01) - if (FAT_write(sb, last_clu, clu.dir) < 0) - return -1; + if (exfat_fat_write(sb, last_clu, clu.dir) < 0) + return -EIO; if (p_fs->hint_uentry.entry == -1) { p_fs->hint_uentry.dir = p_dir->dir; @@ -2110,21 +1642,19 @@ s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries p_dir->size++; /* (3) update the directory entry */ - if (p_fs->vol_type == EXFAT) { - if (p_dir->dir != p_fs->root_dir) { - size += p_fs->cluster_size; - - ep = get_entry_in_dir(sb, &fid->dir, - fid->entry + 1, §or); - if (!ep) - return -1; - p_fs->fs_func->set_entry_size(ep, size); - p_fs->fs_func->set_entry_flag(ep, p_dir->flags); - buf_modify(sb, sector); - - update_dir_checksum(sb, &(fid->dir), - fid->entry); - } + if (p_dir->dir != p_fs->root_dir) { + size += p_fs->cluster_size; + + ep = get_entry_in_dir(sb, &fid->dir, + fid->entry + 1, §or); + if (!ep) + return -ENOENT; + exfat_set_entry_size(ep, size); + exfat_set_entry_flag(ep, p_dir->flags); + exfat_buf_modify(sb, sector); + + update_dir_checksum(sb, &fid->dir, + fid->entry); } i_size_write(inode, i_size_read(inode) + p_fs->cluster_size); @@ -2137,102 +1667,21 @@ s32 find_empty_entry(struct inode *inode, struct chain_t *p_dir, s32 num_entries return dentry; } -/* return values of fat_find_dir_entry() - * >= 0 : return dir entiry position with the name in dir - * -1 : (root dir, ".") it is the root dir itself - * -2 : entry with the name does not exist - */ -s32 fat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, - struct uni_name_t *p_uniname, s32 num_entries, - struct dos_name_t *p_dosname, u32 type) +static s32 extract_uni_name_from_name_entry(struct name_dentry_t *ep, u16 *uniname, + s32 order) { - int i, dentry = 0, len; - s32 order = 0; - bool is_feasible_entry = true, has_ext_entry = false; - s32 dentries_per_clu; - u32 entry_type; - u16 entry_uniname[14], *uniname = NULL, unichar; - struct chain_t clu; - struct dentry_t *ep; - struct dos_dentry_t *dos_ep; - struct ext_dentry_t *ext_ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - if (p_dir->dir == p_fs->root_dir) { - if ((!nls_uniname_cmp(sb, p_uniname->name, - (u16 *)UNI_CUR_DIR_NAME)) || - (!nls_uniname_cmp(sb, p_uniname->name, - (u16 *)UNI_PAR_DIR_NAME))) - return -1; // special case, root directory itself - } - - if (p_dir->dir == CLUSTER_32(0)) /* FAT16 root_dir */ - dentries_per_clu = p_fs->dentries_in_root; - else - dentries_per_clu = p_fs->dentries_per_clu; - - clu.dir = p_dir->dir; - clu.flags = p_dir->flags; - - while (clu.dir != CLUSTER_32(~0)) { - if (p_fs->dev_ejected) - break; - - for (i = 0; i < dentries_per_clu; i++, dentry++) { - ep = get_entry_in_dir(sb, &clu, i, NULL); - if (!ep) - return -2; - - entry_type = p_fs->fs_func->get_entry_type(ep); - - if ((entry_type == TYPE_FILE) || (entry_type == TYPE_DIR)) { - if ((type == TYPE_ALL) || (type == entry_type)) { - if (is_feasible_entry && has_ext_entry) - return dentry; - - dos_ep = (struct dos_dentry_t *)ep; - if (!nls_dosname_cmp(sb, p_dosname->name, dos_ep->name)) - return dentry; - } - is_feasible_entry = true; - has_ext_entry = false; - } else if (entry_type == TYPE_EXTEND) { - if (is_feasible_entry) { - ext_ep = (struct ext_dentry_t *)ep; - if (ext_ep->order > 0x40) { - order = (s32)(ext_ep->order - 0x40); - uniname = p_uniname->name + 13 * (order - 1); - } else { - order = (s32)ext_ep->order; - uniname -= 13; - } - - len = extract_uni_name_from_ext_entry(ext_ep, entry_uniname, order); - - unichar = *(uniname + len); - *(uniname + len) = 0x0; - - if (nls_uniname_cmp(sb, uniname, entry_uniname)) - is_feasible_entry = false; - - *(uniname + len) = unichar; - } - has_ext_entry = true; - } else if (entry_type == TYPE_UNUSED) { - return -2; - } - is_feasible_entry = true; - has_ext_entry = false; - } - - if (p_dir->dir == CLUSTER_32(0)) - break; /* FAT16 root_dir */ + int i, len = 0; - if (FAT_read(sb, clu.dir, &clu.dir) != 0) - return -2; + for (i = 0; i < 30; i += 2) { + *uniname = GET16_A(ep->unicode_0_14 + i); + if (*uniname == 0x0) + return len; + uniname++; + len++; } - return -2; + *uniname = 0x0; + return len; } /* return values of exfat_find_dir_entry() @@ -2286,7 +1735,7 @@ s32 exfat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, if (!ep) return -2; - entry_type = p_fs->fs_func->get_entry_type(ep); + entry_type = exfat_get_entry_type(ep); step = 1; if ((entry_type == TYPE_UNUSED) || (entry_type == TYPE_DELETED)) { @@ -2375,7 +1824,7 @@ s32 exfat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, else clu.dir = CLUSTER_32(~0); } else { - if (FAT_read(sb, clu.dir, &clu.dir) != 0) + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) return -2; } } @@ -2383,36 +1832,6 @@ s32 exfat_find_dir_entry(struct super_block *sb, struct chain_t *p_dir, return -2; } -s32 fat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, - s32 entry, struct dentry_t *p_entry) -{ - s32 count = 0; - u8 chksum; - struct dos_dentry_t *dos_ep = (struct dos_dentry_t *)p_entry; - struct ext_dentry_t *ext_ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - chksum = calc_checksum_1byte((void *)dos_ep->name, DOS_NAME_LENGTH, 0); - - for (entry--; entry >= 0; entry--) { - ext_ep = (struct ext_dentry_t *)get_entry_in_dir(sb, p_dir, - entry, NULL); - if (!ext_ep) - return -1; - - if ((p_fs->fs_func->get_entry_type((struct dentry_t *)ext_ep) == - TYPE_EXTEND) && (ext_ep->checksum == chksum)) { - count++; - if (ext_ep->order > 0x40) - return count; - } else { - return count; - } - } - - return count; -} - s32 exfat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, s32 entry, struct dentry_t *p_entry) { @@ -2420,14 +1839,13 @@ s32 exfat_count_ext_entries(struct super_block *sb, struct chain_t *p_dir, u32 type; struct file_dentry_t *file_ep = (struct file_dentry_t *)p_entry; struct dentry_t *ext_ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); for (i = 0, entry++; i < file_ep->num_ext; i++, entry++) { ext_ep = get_entry_in_dir(sb, p_dir, entry, NULL); if (!ext_ep) return -1; - type = p_fs->fs_func->get_entry_type(ext_ep); + type = exfat_get_entry_type(ext_ep); if ((type == TYPE_EXTEND) || (type == TYPE_STREAM)) count++; else @@ -2463,9 +1881,9 @@ s32 count_dos_name_entries(struct super_block *sb, struct chain_t *p_dir, for (i = 0; i < dentries_per_clu; i++) { ep = get_entry_in_dir(sb, &clu, i, NULL); if (!ep) - return -1; + return -ENOENT; - entry_type = p_fs->fs_func->get_entry_type(ep); + entry_type = exfat_get_entry_type(ep); if (entry_type == TYPE_UNUSED) return count; @@ -2486,8 +1904,8 @@ s32 count_dos_name_entries(struct super_block *sb, struct chain_t *p_dir, else clu.dir = CLUSTER_32(~0); } else { - if (FAT_read(sb, clu.dir, &clu.dir) != 0) - return -1; + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) + return -EIO; } } @@ -2521,7 +1939,7 @@ bool is_dir_empty(struct super_block *sb, struct chain_t *p_dir) if (!ep) break; - type = p_fs->fs_func->get_entry_type(ep); + type = exfat_get_entry_type(ep); if (type == TYPE_UNUSED) return true; @@ -2546,7 +1964,7 @@ bool is_dir_empty(struct super_block *sb, struct chain_t *p_dir) else clu.dir = CLUSTER_32(~0); } - if (FAT_read(sb, clu.dir, &clu.dir) != 0) + if (exfat_fat_read(sb, clu.dir, &clu.dir) != 0) break; } @@ -2564,81 +1982,15 @@ s32 get_num_entries_and_dos_name(struct super_block *sb, struct chain_t *p_dir, struct uni_name_t *p_uniname, s32 *entries, struct dos_name_t *p_dosname) { - s32 ret, num_entries; - bool lossy = false; - char **r; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); + s32 num_entries; - num_entries = p_fs->fs_func->calc_num_entries(p_uniname); + num_entries = exfat_calc_num_entries(p_uniname); if (num_entries == 0) - return FFS_INVALIDPATH; - - if (p_fs->vol_type != EXFAT) { - nls_uniname_to_dosname(sb, p_dosname, p_uniname, &lossy); - - if (lossy) { - ret = fat_generate_dos_name(sb, p_dir, p_dosname); - if (ret) - return ret; - } else { - for (r = reserved_names; *r; r++) { - if (!strncmp((void *)p_dosname->name, *r, 8)) - return FFS_INVALIDPATH; - } - - if (p_dosname->name_case != 0xFF) - num_entries = 1; - } - - if (num_entries > 1) - p_dosname->name_case = 0x0; - } + return -EINVAL; *entries = num_entries; - return FFS_SUCCESS; -} - -void get_uni_name_from_dos_entry(struct super_block *sb, - struct dos_dentry_t *ep, - struct uni_name_t *p_uniname, u8 mode) -{ - struct dos_name_t dos_name; - - if (mode == 0x0) - dos_name.name_case = 0x0; - else - dos_name.name_case = ep->lcase; - - memcpy(dos_name.name, ep->name, DOS_NAME_LENGTH); - nls_dosname_to_uniname(sb, p_uniname, &dos_name); -} - -void fat_get_uni_name_from_ext_entry(struct super_block *sb, - struct chain_t *p_dir, s32 entry, - u16 *uniname) -{ - int i; - struct ext_dentry_t *ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - for (entry--, i = 1; entry >= 0; entry--, i++) { - ep = (struct ext_dentry_t *)get_entry_in_dir(sb, p_dir, entry, - NULL); - if (!ep) - return; - - if (p_fs->fs_func->get_entry_type((struct dentry_t *)ep) == - TYPE_EXTEND) { - extract_uni_name_from_ext_entry(ep, uniname, i); - if (ep->order > 0x40) - return; - } else { - return; - } - - uniname += 13; - } + return 0; } void exfat_get_uni_name_from_ext_entry(struct super_block *sb, @@ -2648,7 +2000,6 @@ void exfat_get_uni_name_from_ext_entry(struct super_block *sb, int i; struct dentry_t *ep; struct entry_set_cache_t *es; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); es = get_entry_set_in_dir(sb, p_dir, entry, ES_ALL_ENTRIES, &ep); if (!es || es->num_entries < 3) { @@ -2666,7 +2017,7 @@ void exfat_get_uni_name_from_ext_entry(struct super_block *sb, * So, the index of first file-name dentry should start from 2. */ for (i = 2; i < es->num_entries; i++, ep++) { - if (p_fs->fs_func->get_entry_type(ep) == TYPE_EXTEND) + if (exfat_get_entry_type(ep) == TYPE_EXTEND) extract_uni_name_from_name_entry((struct name_dentry_t *) ep, uniname, i); else @@ -2678,202 +2029,6 @@ void exfat_get_uni_name_from_ext_entry(struct super_block *sb, release_entry_set(es); } -s32 extract_uni_name_from_ext_entry(struct ext_dentry_t *ep, u16 *uniname, - s32 order) -{ - int i, len = 0; - - for (i = 0; i < 10; i += 2) { - *uniname = GET16(ep->unicode_0_4 + i); - if (*uniname == 0x0) - return len; - uniname++; - len++; - } - - if (order < 20) { - for (i = 0; i < 12; i += 2) { - *uniname = GET16_A(ep->unicode_5_10 + i); - if (*uniname == 0x0) - return len; - uniname++; - len++; - } - } else { - for (i = 0; i < 8; i += 2) { - *uniname = GET16_A(ep->unicode_5_10 + i); - if (*uniname == 0x0) - return len; - uniname++; - len++; - } - *uniname = 0x0; /* uniname[MAX_NAME_LENGTH-1] */ - return len; - } - - for (i = 0; i < 4; i += 2) { - *uniname = GET16_A(ep->unicode_11_12 + i); - if (*uniname == 0x0) - return len; - uniname++; - len++; - } - - *uniname = 0x0; - return len; -} - -s32 extract_uni_name_from_name_entry(struct name_dentry_t *ep, u16 *uniname, - s32 order) -{ - int i, len = 0; - - for (i = 0; i < 30; i += 2) { - *uniname = GET16_A(ep->unicode_0_14 + i); - if (*uniname == 0x0) - return len; - uniname++; - len++; - } - - *uniname = 0x0; - return len; -} - -s32 fat_generate_dos_name(struct super_block *sb, struct chain_t *p_dir, - struct dos_name_t *p_dosname) -{ - int i, j, count = 0; - bool count_begin = false; - s32 dentries_per_clu; - u32 type; - u8 bmap[128/* 1 ~ 1023 */]; - struct chain_t clu; - struct dos_dentry_t *ep; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - - memset(bmap, 0, sizeof(bmap)); - exfat_bitmap_set(bmap, 0); - - if (p_dir->dir == CLUSTER_32(0)) /* FAT16 root_dir */ - dentries_per_clu = p_fs->dentries_in_root; - else - dentries_per_clu = p_fs->dentries_per_clu; - - clu.dir = p_dir->dir; - clu.flags = p_dir->flags; - - while (clu.dir != CLUSTER_32(~0)) { - if (p_fs->dev_ejected) - break; - - for (i = 0; i < dentries_per_clu; i++) { - ep = (struct dos_dentry_t *)get_entry_in_dir(sb, &clu, - i, NULL); - if (!ep) - return FFS_MEDIAERR; - - type = p_fs->fs_func->get_entry_type((struct dentry_t *) - ep); - - if (type == TYPE_UNUSED) - break; - if ((type != TYPE_FILE) && (type != TYPE_DIR)) - continue; - - count = 0; - count_begin = false; - - for (j = 0; j < 8; j++) { - if (ep->name[j] == ' ') - break; - - if (ep->name[j] == '~') { - count_begin = true; - } else if (count_begin) { - if ((ep->name[j] >= '0') && - (ep->name[j] <= '9')) { - count = count * 10 + - (ep->name[j] - '0'); - } else { - count = 0; - count_begin = false; - } - } - } - - if ((count > 0) && (count < 1024)) - exfat_bitmap_set(bmap, count); - } - - if (p_dir->dir == CLUSTER_32(0)) - break; /* FAT16 root_dir */ - - if (FAT_read(sb, clu.dir, &clu.dir) != 0) - return FFS_MEDIAERR; - } - - count = 0; - for (i = 0; i < 128; i++) { - if (bmap[i] != 0xFF) { - for (j = 0; j < 8; j++) { - if (exfat_bitmap_test(&bmap[i], j) == 0) { - count = (i << 3) + j; - break; - } - } - if (count != 0) - break; - } - } - - if ((count == 0) || (count >= 1024)) - return FFS_FILEEXIST; - fat_attach_count_to_dos_name(p_dosname->name, count); - - /* Now dos_name has DOS~????.EXT */ - return FFS_SUCCESS; -} - -void fat_attach_count_to_dos_name(u8 *dosname, s32 count) -{ - int i, j, length; - char str_count[6]; - - snprintf(str_count, sizeof(str_count), "~%d", count); - length = strlen(str_count); - - i = 0; - j = 0; - while (j <= (8 - length)) { - i = j; - if (dosname[j] == ' ') - break; - if (dosname[j] & 0x80) - j += 2; - else - j++; - } - - for (j = 0; j < length; i++, j++) - dosname[i] = (u8)str_count[j]; - - if (i == 7) - dosname[7] = ' '; -} - -s32 fat_calc_num_entries(struct uni_name_t *p_uniname) -{ - s32 len; - - len = p_uniname->name_len; - if (len == 0) - return 0; - - /* 1 dos name entry + extended entries */ - return (len - 1) / 13 + 2; -} - s32 exfat_calc_num_entries(struct uni_name_t *p_uniname) { s32 len; @@ -2886,17 +2041,6 @@ s32 exfat_calc_num_entries(struct uni_name_t *p_uniname) return (len - 1) / 15 + 3; } -u8 calc_checksum_1byte(void *data, s32 len, u8 chksum) -{ - int i; - u8 *c = (u8 *)data; - - for (i = 0; i < len; i++, c++) - chksum = (((chksum & 1) << 7) | ((chksum & 0xFE) >> 1)) + *c; - - return chksum; -} - u16 calc_checksum_2byte(void *data, s32 len, u16 chksum, s32 type) { int i; @@ -2921,30 +2065,6 @@ u16 calc_checksum_2byte(void *data, s32 len, u16 chksum, s32 type) return chksum; } -u32 calc_checksum_4byte(void *data, s32 len, u32 chksum, s32 type) -{ - int i; - u8 *c = (u8 *)data; - - switch (type) { - case CS_PBR_SECTOR: - for (i = 0; i < len; i++, c++) { - if ((i == 106) || (i == 107) || (i == 112)) - continue; - chksum = (((chksum & 1) << 31) | - ((chksum & 0xFFFFFFFE) >> 1)) + (u32)*c; - } - break; - default - : - for (i = 0; i < len; i++, c++) - chksum = (((chksum & 1) << 31) | - ((chksum & 0xFFFFFFFE) >> 1)) + (u32)*c; - } - - return chksum; -} - /* * Name Resolution Functions */ @@ -2962,11 +2082,11 @@ s32 resolve_path(struct inode *inode, char *path, struct chain_t *p_dir, struct file_id_t *fid = &(EXFAT_I(inode)->fid); if (strscpy(name_buf, path, sizeof(name_buf)) < 0) - return FFS_INVALIDPATH; + return -EINVAL; nls_cstring_to_uniname(sb, p_uniname, name_buf, &lossy); if (lossy) - return FFS_INVALIDPATH; + return -EINVAL; fid->size = i_size_read(inode); @@ -2974,181 +2094,9 @@ s32 resolve_path(struct inode *inode, char *path, struct chain_t *p_dir, p_dir->size = (s32)(fid->size >> p_fs->cluster_size_bits); p_dir->flags = fid->flags; - return FFS_SUCCESS; -} - -/* - * File Operation Functions - */ -static struct fs_func fat_fs_func = { - .alloc_cluster = fat_alloc_cluster, - .free_cluster = fat_free_cluster, - .count_used_clusters = fat_count_used_clusters, - - .init_dir_entry = fat_init_dir_entry, - .init_ext_entry = fat_init_ext_entry, - .find_dir_entry = fat_find_dir_entry, - .delete_dir_entry = fat_delete_dir_entry, - .get_uni_name_from_ext_entry = fat_get_uni_name_from_ext_entry, - .count_ext_entries = fat_count_ext_entries, - .calc_num_entries = fat_calc_num_entries, - - .get_entry_type = fat_get_entry_type, - .set_entry_type = fat_set_entry_type, - .get_entry_attr = fat_get_entry_attr, - .set_entry_attr = fat_set_entry_attr, - .get_entry_flag = fat_get_entry_flag, - .set_entry_flag = fat_set_entry_flag, - .get_entry_clu0 = fat_get_entry_clu0, - .set_entry_clu0 = fat_set_entry_clu0, - .get_entry_size = fat_get_entry_size, - .set_entry_size = fat_set_entry_size, - .get_entry_time = fat_get_entry_time, - .set_entry_time = fat_set_entry_time, -}; - -s32 fat16_mount(struct super_block *sb, struct pbr_sector_t *p_pbr) -{ - s32 num_reserved, num_root_sectors; - struct bpb16_t *p_bpb = (struct bpb16_t *)p_pbr->bpb; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - - if (p_bpb->num_fats == 0) - return FFS_FORMATERR; - - num_root_sectors = GET16(p_bpb->num_root_entries) << DENTRY_SIZE_BITS; - num_root_sectors = ((num_root_sectors - 1) >> - p_bd->sector_size_bits) + 1; - - p_fs->sectors_per_clu = p_bpb->sectors_per_clu; - p_fs->sectors_per_clu_bits = ilog2(p_bpb->sectors_per_clu); - p_fs->cluster_size_bits = p_fs->sectors_per_clu_bits + - p_bd->sector_size_bits; - p_fs->cluster_size = 1 << p_fs->cluster_size_bits; - - p_fs->num_FAT_sectors = GET16(p_bpb->num_fat_sectors); - - p_fs->FAT1_start_sector = p_fs->PBR_sector + GET16(p_bpb->num_reserved); - if (p_bpb->num_fats == 1) - p_fs->FAT2_start_sector = p_fs->FAT1_start_sector; - else - p_fs->FAT2_start_sector = p_fs->FAT1_start_sector + - p_fs->num_FAT_sectors; - - p_fs->root_start_sector = p_fs->FAT2_start_sector + - p_fs->num_FAT_sectors; - p_fs->data_start_sector = p_fs->root_start_sector + num_root_sectors; - - p_fs->num_sectors = GET16(p_bpb->num_sectors); - if (p_fs->num_sectors == 0) - p_fs->num_sectors = GET32(p_bpb->num_huge_sectors); - - num_reserved = p_fs->data_start_sector - p_fs->PBR_sector; - p_fs->num_clusters = ((p_fs->num_sectors - num_reserved) >> - p_fs->sectors_per_clu_bits) + 2; - /* because the cluster index starts with 2 */ - - if (p_fs->num_clusters < FAT12_THRESHOLD) - p_fs->vol_type = FAT12; - else - p_fs->vol_type = FAT16; - p_fs->vol_id = GET32(p_bpb->vol_serial); - - p_fs->root_dir = 0; - p_fs->dentries_in_root = GET16(p_bpb->num_root_entries); - p_fs->dentries_per_clu = 1 << (p_fs->cluster_size_bits - - DENTRY_SIZE_BITS); - - p_fs->vol_flag = VOL_CLEAN; - p_fs->clu_srch_ptr = 2; - p_fs->used_clusters = UINT_MAX; - - p_fs->fs_func = &fat_fs_func; - - return FFS_SUCCESS; + return 0; } -s32 fat32_mount(struct super_block *sb, struct pbr_sector_t *p_pbr) -{ - s32 num_reserved; - struct bpb32_t *p_bpb = (struct bpb32_t *)p_pbr->bpb; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); - - if (p_bpb->num_fats == 0) - return FFS_FORMATERR; - - p_fs->sectors_per_clu = p_bpb->sectors_per_clu; - p_fs->sectors_per_clu_bits = ilog2(p_bpb->sectors_per_clu); - p_fs->cluster_size_bits = p_fs->sectors_per_clu_bits + - p_bd->sector_size_bits; - p_fs->cluster_size = 1 << p_fs->cluster_size_bits; - - p_fs->num_FAT_sectors = GET32(p_bpb->num_fat32_sectors); - - p_fs->FAT1_start_sector = p_fs->PBR_sector + GET16(p_bpb->num_reserved); - if (p_bpb->num_fats == 1) - p_fs->FAT2_start_sector = p_fs->FAT1_start_sector; - else - p_fs->FAT2_start_sector = p_fs->FAT1_start_sector + - p_fs->num_FAT_sectors; - - p_fs->root_start_sector = p_fs->FAT2_start_sector + - p_fs->num_FAT_sectors; - p_fs->data_start_sector = p_fs->root_start_sector; - - p_fs->num_sectors = GET32(p_bpb->num_huge_sectors); - num_reserved = p_fs->data_start_sector - p_fs->PBR_sector; - - p_fs->num_clusters = ((p_fs->num_sectors - num_reserved) >> - p_fs->sectors_per_clu_bits) + 2; - /* because the cluster index starts with 2 */ - - p_fs->vol_type = FAT32; - p_fs->vol_id = GET32(p_bpb->vol_serial); - - p_fs->root_dir = GET32(p_bpb->root_cluster); - p_fs->dentries_in_root = 0; - p_fs->dentries_per_clu = 1 << (p_fs->cluster_size_bits - - DENTRY_SIZE_BITS); - - p_fs->vol_flag = VOL_CLEAN; - p_fs->clu_srch_ptr = 2; - p_fs->used_clusters = UINT_MAX; - - p_fs->fs_func = &fat_fs_func; - - return FFS_SUCCESS; -} - -static struct fs_func exfat_fs_func = { - .alloc_cluster = exfat_alloc_cluster, - .free_cluster = exfat_free_cluster, - .count_used_clusters = exfat_count_used_clusters, - - .init_dir_entry = exfat_init_dir_entry, - .init_ext_entry = exfat_init_ext_entry, - .find_dir_entry = exfat_find_dir_entry, - .delete_dir_entry = exfat_delete_dir_entry, - .get_uni_name_from_ext_entry = exfat_get_uni_name_from_ext_entry, - .count_ext_entries = exfat_count_ext_entries, - .calc_num_entries = exfat_calc_num_entries, - - .get_entry_type = exfat_get_entry_type, - .set_entry_type = exfat_set_entry_type, - .get_entry_attr = exfat_get_entry_attr, - .set_entry_attr = exfat_set_entry_attr, - .get_entry_flag = exfat_get_entry_flag, - .set_entry_flag = exfat_set_entry_flag, - .get_entry_clu0 = exfat_get_entry_clu0, - .set_entry_clu0 = exfat_set_entry_clu0, - .get_entry_size = exfat_get_entry_size, - .set_entry_size = exfat_set_entry_size, - .get_entry_time = exfat_get_entry_time, - .set_entry_time = exfat_set_entry_time, -}; - s32 exfat_mount(struct super_block *sb, struct pbr_sector_t *p_pbr) { struct bpbex_t *p_bpb = (struct bpbex_t *)p_pbr->bpb; @@ -3156,7 +2104,7 @@ s32 exfat_mount(struct super_block *sb, struct pbr_sector_t *p_pbr) struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); if (p_bpb->num_fats == 0) - return FFS_FORMATERR; + return -EFSCORRUPTED; p_fs->sectors_per_clu = 1 << p_bpb->sectors_per_clu_bits; p_fs->sectors_per_clu_bits = p_bpb->sectors_per_clu_bits; @@ -3192,9 +2140,7 @@ s32 exfat_mount(struct super_block *sb, struct pbr_sector_t *p_pbr) p_fs->clu_srch_ptr = 2; p_fs->used_clusters = UINT_MAX; - p_fs->fs_func = &exfat_fs_func; - - return FFS_SUCCESS; + return 0; } s32 create_dir(struct inode *inode, struct chain_t *p_dir, @@ -3203,10 +2149,9 @@ s32 create_dir(struct inode *inode, struct chain_t *p_dir, s32 ret, dentry, num_entries; u64 size; struct chain_t clu; - struct dos_name_t dos_name, dot_name; + struct dos_name_t dos_name; struct super_block *sb = inode->i_sb; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; ret = get_num_entries_and_dos_name(sb, p_dir, p_uniname, &num_entries, &dos_name); @@ -3216,73 +2161,35 @@ s32 create_dir(struct inode *inode, struct chain_t *p_dir, /* find_empty_entry must be called before alloc_cluster */ dentry = find_empty_entry(inode, p_dir, num_entries); if (dentry < 0) - return FFS_FULL; + return -ENOSPC; clu.dir = CLUSTER_32(~0); clu.size = 0; clu.flags = (p_fs->vol_type == EXFAT) ? 0x03 : 0x01; /* (1) allocate a cluster */ - ret = fs_func->alloc_cluster(sb, 1, &clu); + ret = exfat_alloc_cluster(sb, 1, &clu); if (ret < 0) - return FFS_MEDIAERR; + return ret; else if (ret == 0) - return FFS_FULL; + return -ENOSPC; ret = clear_cluster(sb, clu.dir); - if (ret != FFS_SUCCESS) + if (ret != 0) return ret; - if (p_fs->vol_type == EXFAT) { - size = p_fs->cluster_size; - } else { - size = 0; - - /* initialize the . and .. entry - * Information for . points to itself - * Information for .. points to parent dir - */ - - dot_name.name_case = 0x0; - memcpy(dot_name.name, DOS_CUR_DIR_NAME, DOS_NAME_LENGTH); - - ret = fs_func->init_dir_entry(sb, &clu, 0, TYPE_DIR, clu.dir, - 0); - if (ret != FFS_SUCCESS) - return ret; - - ret = fs_func->init_ext_entry(sb, &clu, 0, 1, NULL, &dot_name); - if (ret != FFS_SUCCESS) - return ret; - - memcpy(dot_name.name, DOS_PAR_DIR_NAME, DOS_NAME_LENGTH); - - if (p_dir->dir == p_fs->root_dir) - ret = fs_func->init_dir_entry(sb, &clu, 1, TYPE_DIR, - CLUSTER_32(0), 0); - else - ret = fs_func->init_dir_entry(sb, &clu, 1, TYPE_DIR, - p_dir->dir, 0); - - if (ret != FFS_SUCCESS) - return ret; - - ret = p_fs->fs_func->init_ext_entry(sb, &clu, 1, 1, NULL, - &dot_name); - if (ret != FFS_SUCCESS) - return ret; - } + size = p_fs->cluster_size; /* (2) update the directory entry */ /* make sub-dir entry in parent directory */ - ret = fs_func->init_dir_entry(sb, p_dir, dentry, TYPE_DIR, clu.dir, - size); - if (ret != FFS_SUCCESS) + ret = exfat_init_dir_entry(sb, p_dir, dentry, TYPE_DIR, clu.dir, + size); + if (ret != 0) return ret; - ret = fs_func->init_ext_entry(sb, p_dir, dentry, num_entries, p_uniname, - &dos_name); - if (ret != FFS_SUCCESS) + ret = exfat_init_ext_entry(sb, p_dir, dentry, num_entries, p_uniname, + &dos_name); + if (ret != 0) return ret; fid->dir.dir = p_dir->dir; @@ -3299,7 +2206,7 @@ s32 create_dir(struct inode *inode, struct chain_t *p_dir, fid->rwoffset = 0; fid->hint_last_off = -1; - return FFS_SUCCESS; + return 0; } s32 create_file(struct inode *inode, struct chain_t *p_dir, @@ -3309,7 +2216,6 @@ s32 create_file(struct inode *inode, struct chain_t *p_dir, struct dos_name_t dos_name; struct super_block *sb = inode->i_sb; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; ret = get_num_entries_and_dos_name(sb, p_dir, p_uniname, &num_entries, &dos_name); @@ -3319,20 +2225,20 @@ s32 create_file(struct inode *inode, struct chain_t *p_dir, /* find_empty_entry must be called before alloc_cluster() */ dentry = find_empty_entry(inode, p_dir, num_entries); if (dentry < 0) - return FFS_FULL; + return -ENOSPC; /* (1) update the directory entry */ /* fill the dos name directory entry information of the created file. * the first cluster is not determined yet. (0) */ - ret = fs_func->init_dir_entry(sb, p_dir, dentry, TYPE_FILE | mode, - CLUSTER_32(0), 0); - if (ret != FFS_SUCCESS) + ret = exfat_init_dir_entry(sb, p_dir, dentry, TYPE_FILE | mode, + CLUSTER_32(0), 0); + if (ret != 0) return ret; - ret = fs_func->init_ext_entry(sb, p_dir, dentry, num_entries, p_uniname, - &dos_name); - if (ret != FFS_SUCCESS) + ret = exfat_init_ext_entry(sb, p_dir, dentry, num_entries, p_uniname, + &dos_name); + if (ret != 0) return ret; fid->dir.dir = p_dir->dir; @@ -3349,7 +2255,7 @@ s32 create_file(struct inode *inode, struct chain_t *p_dir, fid->rwoffset = 0; fid->hint_last_off = -1; - return FFS_SUCCESS; + return 0; } void remove_file(struct inode *inode, struct chain_t *p_dir, s32 entry) @@ -3358,27 +2264,25 @@ void remove_file(struct inode *inode, struct chain_t *p_dir, s32 entry) sector_t sector; struct dentry_t *ep; struct super_block *sb = inode->i_sb; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; ep = get_entry_in_dir(sb, p_dir, entry, §or); if (!ep) return; - buf_lock(sb, sector); + exfat_buf_lock(sb, sector); - /* buf_lock() before call count_ext_entries() */ - num_entries = fs_func->count_ext_entries(sb, p_dir, entry, ep); + /* exfat_buf_lock() before call count_ext_entries() */ + num_entries = exfat_count_ext_entries(sb, p_dir, entry, ep); if (num_entries < 0) { - buf_unlock(sb, sector); + exfat_buf_unlock(sb, sector); return; } num_entries++; - buf_unlock(sb, sector); + exfat_buf_unlock(sb, sector); /* (1) update the directory entry */ - fs_func->delete_dir_entry(sb, p_dir, entry, 0, num_entries); + exfat_delete_dir_entry(sb, p_dir, entry, 0, num_entries); } s32 exfat_rename_file(struct inode *inode, struct chain_t *p_dir, s32 oldentry, @@ -3389,101 +2293,97 @@ s32 exfat_rename_file(struct inode *inode, struct chain_t *p_dir, s32 oldentry, struct dos_name_t dos_name; struct dentry_t *epold, *epnew; struct super_block *sb = inode->i_sb; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; epold = get_entry_in_dir(sb, p_dir, oldentry, §or_old); if (!epold) - return FFS_MEDIAERR; + return -ENOENT; - buf_lock(sb, sector_old); + exfat_buf_lock(sb, sector_old); - /* buf_lock() before call count_ext_entries() */ - num_old_entries = fs_func->count_ext_entries(sb, p_dir, oldentry, - epold); + /* exfat_buf_lock() before call count_ext_entries() */ + num_old_entries = exfat_count_ext_entries(sb, p_dir, oldentry, + epold); if (num_old_entries < 0) { - buf_unlock(sb, sector_old); - return FFS_MEDIAERR; + exfat_buf_unlock(sb, sector_old); + return -ENOENT; } num_old_entries++; ret = get_num_entries_and_dos_name(sb, p_dir, p_uniname, &num_new_entries, &dos_name); if (ret) { - buf_unlock(sb, sector_old); + exfat_buf_unlock(sb, sector_old); return ret; } if (num_old_entries < num_new_entries) { newentry = find_empty_entry(inode, p_dir, num_new_entries); if (newentry < 0) { - buf_unlock(sb, sector_old); - return FFS_FULL; + exfat_buf_unlock(sb, sector_old); + return -ENOSPC; } epnew = get_entry_in_dir(sb, p_dir, newentry, §or_new); if (!epnew) { - buf_unlock(sb, sector_old); - return FFS_MEDIAERR; + exfat_buf_unlock(sb, sector_old); + return -ENOENT; } memcpy((void *)epnew, (void *)epold, DENTRY_SIZE); - if (fs_func->get_entry_type(epnew) == TYPE_FILE) { - fs_func->set_entry_attr(epnew, - fs_func->get_entry_attr(epnew) | - ATTR_ARCHIVE); + if (exfat_get_entry_type(epnew) == TYPE_FILE) { + exfat_set_entry_attr(epnew, + exfat_get_entry_attr(epnew) | + ATTR_ARCHIVE); fid->attr |= ATTR_ARCHIVE; } - buf_modify(sb, sector_new); - buf_unlock(sb, sector_old); - - if (p_fs->vol_type == EXFAT) { - epold = get_entry_in_dir(sb, p_dir, oldentry + 1, - §or_old); - buf_lock(sb, sector_old); - epnew = get_entry_in_dir(sb, p_dir, newentry + 1, - §or_new); - - if (!epold || !epnew) { - buf_unlock(sb, sector_old); - return FFS_MEDIAERR; - } + exfat_buf_modify(sb, sector_new); + exfat_buf_unlock(sb, sector_old); + + epold = get_entry_in_dir(sb, p_dir, oldentry + 1, + §or_old); + exfat_buf_lock(sb, sector_old); + epnew = get_entry_in_dir(sb, p_dir, newentry + 1, + §or_new); - memcpy((void *)epnew, (void *)epold, DENTRY_SIZE); - buf_modify(sb, sector_new); - buf_unlock(sb, sector_old); + if (!epold || !epnew) { + exfat_buf_unlock(sb, sector_old); + return -ENOENT; } - ret = fs_func->init_ext_entry(sb, p_dir, newentry, - num_new_entries, p_uniname, - &dos_name); - if (ret != FFS_SUCCESS) + memcpy((void *)epnew, (void *)epold, DENTRY_SIZE); + exfat_buf_modify(sb, sector_new); + exfat_buf_unlock(sb, sector_old); + + ret = exfat_init_ext_entry(sb, p_dir, newentry, + num_new_entries, p_uniname, + &dos_name); + if (ret != 0) return ret; - fs_func->delete_dir_entry(sb, p_dir, oldentry, 0, - num_old_entries); + exfat_delete_dir_entry(sb, p_dir, oldentry, 0, + num_old_entries); fid->entry = newentry; } else { - if (fs_func->get_entry_type(epold) == TYPE_FILE) { - fs_func->set_entry_attr(epold, - fs_func->get_entry_attr(epold) | - ATTR_ARCHIVE); + if (exfat_get_entry_type(epold) == TYPE_FILE) { + exfat_set_entry_attr(epold, + exfat_get_entry_attr(epold) | + ATTR_ARCHIVE); fid->attr |= ATTR_ARCHIVE; } - buf_modify(sb, sector_old); - buf_unlock(sb, sector_old); + exfat_buf_modify(sb, sector_old); + exfat_buf_unlock(sb, sector_old); - ret = fs_func->init_ext_entry(sb, p_dir, oldentry, - num_new_entries, p_uniname, - &dos_name); - if (ret != FFS_SUCCESS) + ret = exfat_init_ext_entry(sb, p_dir, oldentry, + num_new_entries, p_uniname, + &dos_name); + if (ret != 0) return ret; - fs_func->delete_dir_entry(sb, p_dir, oldentry, num_new_entries, - num_old_entries); + exfat_delete_dir_entry(sb, p_dir, oldentry, num_new_entries, + num_old_entries); } - return FFS_SUCCESS; + return 0; } s32 move_file(struct inode *inode, struct chain_t *p_olddir, s32 oldentry, @@ -3492,97 +2392,78 @@ s32 move_file(struct inode *inode, struct chain_t *p_olddir, s32 oldentry, { s32 ret, newentry, num_new_entries, num_old_entries; sector_t sector_mov, sector_new; - struct chain_t clu; struct dos_name_t dos_name; struct dentry_t *epmov, *epnew; struct super_block *sb = inode->i_sb; - struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; epmov = get_entry_in_dir(sb, p_olddir, oldentry, §or_mov); if (!epmov) - return FFS_MEDIAERR; + return -ENOENT; /* check if the source and target directory is the same */ - if (fs_func->get_entry_type(epmov) == TYPE_DIR && - fs_func->get_entry_clu0(epmov) == p_newdir->dir) - return FFS_INVALIDPATH; + if (exfat_get_entry_type(epmov) == TYPE_DIR && + exfat_get_entry_clu0(epmov) == p_newdir->dir) + return -EINVAL; - buf_lock(sb, sector_mov); + exfat_buf_lock(sb, sector_mov); - /* buf_lock() before call count_ext_entries() */ - num_old_entries = fs_func->count_ext_entries(sb, p_olddir, oldentry, - epmov); + /* exfat_buf_lock() before call count_ext_entries() */ + num_old_entries = exfat_count_ext_entries(sb, p_olddir, oldentry, + epmov); if (num_old_entries < 0) { - buf_unlock(sb, sector_mov); - return FFS_MEDIAERR; + exfat_buf_unlock(sb, sector_mov); + return -ENOENT; } num_old_entries++; ret = get_num_entries_and_dos_name(sb, p_newdir, p_uniname, &num_new_entries, &dos_name); if (ret) { - buf_unlock(sb, sector_mov); + exfat_buf_unlock(sb, sector_mov); return ret; } newentry = find_empty_entry(inode, p_newdir, num_new_entries); if (newentry < 0) { - buf_unlock(sb, sector_mov); - return FFS_FULL; + exfat_buf_unlock(sb, sector_mov); + return -ENOSPC; } epnew = get_entry_in_dir(sb, p_newdir, newentry, §or_new); if (!epnew) { - buf_unlock(sb, sector_mov); - return FFS_MEDIAERR; + exfat_buf_unlock(sb, sector_mov); + return -ENOENT; } memcpy((void *)epnew, (void *)epmov, DENTRY_SIZE); - if (fs_func->get_entry_type(epnew) == TYPE_FILE) { - fs_func->set_entry_attr(epnew, fs_func->get_entry_attr(epnew) | - ATTR_ARCHIVE); + if (exfat_get_entry_type(epnew) == TYPE_FILE) { + exfat_set_entry_attr(epnew, exfat_get_entry_attr(epnew) | + ATTR_ARCHIVE); fid->attr |= ATTR_ARCHIVE; } - buf_modify(sb, sector_new); - buf_unlock(sb, sector_mov); - - if (p_fs->vol_type == EXFAT) { - epmov = get_entry_in_dir(sb, p_olddir, oldentry + 1, - §or_mov); - buf_lock(sb, sector_mov); - epnew = get_entry_in_dir(sb, p_newdir, newentry + 1, - §or_new); - if (!epmov || !epnew) { - buf_unlock(sb, sector_mov); - return FFS_MEDIAERR; - } + exfat_buf_modify(sb, sector_new); + exfat_buf_unlock(sb, sector_mov); - memcpy((void *)epnew, (void *)epmov, DENTRY_SIZE); - buf_modify(sb, sector_new); - buf_unlock(sb, sector_mov); - } else if (fs_func->get_entry_type(epnew) == TYPE_DIR) { - /* change ".." pointer to new parent dir */ - clu.dir = fs_func->get_entry_clu0(epnew); - clu.flags = 0x01; - - epnew = get_entry_in_dir(sb, &clu, 1, §or_new); - if (!epnew) - return FFS_MEDIAERR; - - if (p_newdir->dir == p_fs->root_dir) - fs_func->set_entry_clu0(epnew, CLUSTER_32(0)); - else - fs_func->set_entry_clu0(epnew, p_newdir->dir); - buf_modify(sb, sector_new); + epmov = get_entry_in_dir(sb, p_olddir, oldentry + 1, + §or_mov); + exfat_buf_lock(sb, sector_mov); + epnew = get_entry_in_dir(sb, p_newdir, newentry + 1, + §or_new); + if (!epmov || !epnew) { + exfat_buf_unlock(sb, sector_mov); + return -ENOENT; } - ret = fs_func->init_ext_entry(sb, p_newdir, newentry, num_new_entries, - p_uniname, &dos_name); - if (ret != FFS_SUCCESS) + memcpy((void *)epnew, (void *)epmov, DENTRY_SIZE); + exfat_buf_modify(sb, sector_new); + exfat_buf_unlock(sb, sector_mov); + + ret = exfat_init_ext_entry(sb, p_newdir, newentry, num_new_entries, + p_uniname, &dos_name); + if (ret != 0) return ret; - fs_func->delete_dir_entry(sb, p_olddir, oldentry, 0, num_old_entries); + exfat_delete_dir_entry(sb, p_olddir, oldentry, 0, num_old_entries); fid->dir.dir = p_newdir->dir; fid->dir.size = p_newdir->size; @@ -3590,7 +2471,7 @@ s32 move_file(struct inode *inode, struct chain_t *p_olddir, s32 oldentry, fid->entry = newentry; - return FFS_SUCCESS; + return 0; } /* @@ -3600,7 +2481,7 @@ s32 move_file(struct inode *inode, struct chain_t *p_olddir, s32 oldentry, int sector_read(struct super_block *sb, sector_t sec, struct buffer_head **bh, bool read) { - s32 ret = FFS_MEDIAERR; + s32 ret = -EIO; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); if ((sec >= (p_fs->PBR_sector + p_fs->num_sectors)) && @@ -3612,8 +2493,8 @@ int sector_read(struct super_block *sb, sector_t sec, struct buffer_head **bh, } if (!p_fs->dev_ejected) { - ret = bdev_read(sb, sec, bh, 1, read); - if (ret != FFS_SUCCESS) + ret = exfat_bdev_read(sb, sec, bh, 1, read); + if (ret != 0) p_fs->dev_ejected = 1; } @@ -3623,7 +2504,7 @@ int sector_read(struct super_block *sb, sector_t sec, struct buffer_head **bh, int sector_write(struct super_block *sb, sector_t sec, struct buffer_head *bh, bool sync) { - s32 ret = FFS_MEDIAERR; + s32 ret = -EIO; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); if (sec >= (p_fs->PBR_sector + p_fs->num_sectors) && @@ -3641,8 +2522,8 @@ int sector_write(struct super_block *sb, sector_t sec, struct buffer_head *bh, } if (!p_fs->dev_ejected) { - ret = bdev_write(sb, sec, bh, 1, sync); - if (ret != FFS_SUCCESS) + ret = exfat_bdev_write(sb, sec, bh, 1, sync); + if (ret != 0) p_fs->dev_ejected = 1; } @@ -3652,7 +2533,7 @@ int sector_write(struct super_block *sb, sector_t sec, struct buffer_head *bh, int multi_sector_read(struct super_block *sb, sector_t sec, struct buffer_head **bh, s32 num_secs, bool read) { - s32 ret = FFS_MEDIAERR; + s32 ret = -EIO; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); if (((sec + num_secs) > (p_fs->PBR_sector + p_fs->num_sectors)) && @@ -3664,8 +2545,8 @@ int multi_sector_read(struct super_block *sb, sector_t sec, } if (!p_fs->dev_ejected) { - ret = bdev_read(sb, sec, bh, num_secs, read); - if (ret != FFS_SUCCESS) + ret = exfat_bdev_read(sb, sec, bh, num_secs, read); + if (ret != 0) p_fs->dev_ejected = 1; } @@ -3675,7 +2556,7 @@ int multi_sector_read(struct super_block *sb, sector_t sec, int multi_sector_write(struct super_block *sb, sector_t sec, struct buffer_head *bh, s32 num_secs, bool sync) { - s32 ret = FFS_MEDIAERR; + s32 ret = -EIO; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); if ((sec + num_secs) > (p_fs->PBR_sector + p_fs->num_sectors) && @@ -3692,8 +2573,8 @@ int multi_sector_write(struct super_block *sb, sector_t sec, } if (!p_fs->dev_ejected) { - ret = bdev_write(sb, sec, bh, num_secs, sync); - if (ret != FFS_SUCCESS) + ret = exfat_bdev_write(sb, sec, bh, num_secs, sync); + if (ret != 0) p_fs->dev_ejected = 1; } diff --git a/drivers/staging/exfat/exfat_nls.c b/drivers/staging/exfat/exfat_nls.c index a5c4b68925fba..91e8b0c4dce78 100644 --- a/drivers/staging/exfat/exfat_nls.c +++ b/drivers/staging/exfat/exfat_nls.c @@ -7,13 +7,6 @@ #include #include "exfat.h" -static u16 bad_dos_chars[] = { - /* + , ; = [ ] */ - 0x002B, 0x002C, 0x003B, 0x003D, 0x005B, 0x005D, - 0xFF0B, 0xFF0C, 0xFF1B, 0xFF1D, 0xFF3B, 0xFF3D, - 0 -}; - static u16 bad_uni_chars[] = { /* " * / : < > ? \ | */ 0x0022, 0x002A, 0x002F, 0x003A, @@ -96,11 +89,6 @@ static u16 *nls_wstrchr(u16 *str, u16 wchar) return NULL; } -int nls_dosname_cmp(struct super_block *sb, u8 *a, u8 *b) -{ - return strncmp(a, b, DOS_NAME_LENGTH); -} - int nls_uniname_cmp(struct super_block *sb, u16 *a, u16 *b) { int i; @@ -114,186 +102,6 @@ int nls_uniname_cmp(struct super_block *sb, u16 *a, u16 *b) return 0; } -void nls_uniname_to_dosname(struct super_block *sb, - struct dos_name_t *p_dosname, - struct uni_name_t *p_uniname, bool *p_lossy) -{ - int i, j, len; - bool lossy = false; - u8 buf[MAX_CHARSET_SIZE]; - u8 lower = 0, upper = 0; - u8 *dosname = p_dosname->name; - u16 *uniname = p_uniname->name; - u16 *p, *last_period; - struct nls_table *nls = EXFAT_SB(sb)->nls_disk; - - for (i = 0; i < DOS_NAME_LENGTH; i++) - *(dosname + i) = ' '; - - if (!nls_uniname_cmp(sb, uniname, (u16 *)UNI_CUR_DIR_NAME)) { - *(dosname) = '.'; - p_dosname->name_case = 0x0; - if (p_lossy) - *p_lossy = false; - return; - } - - if (!nls_uniname_cmp(sb, uniname, (u16 *)UNI_PAR_DIR_NAME)) { - *(dosname) = '.'; - *(dosname + 1) = '.'; - p_dosname->name_case = 0x0; - if (p_lossy) - *p_lossy = false; - return; - } - - /* search for the last embedded period */ - last_period = NULL; - for (p = uniname; *p; p++) { - if (*p == (u16)'.') - last_period = p; - } - - i = 0; - while (i < DOS_NAME_LENGTH) { - if (i == 8) { - if (!last_period) - break; - - if (uniname <= last_period) { - if (uniname < last_period) - lossy = true; - uniname = last_period + 1; - } - } - - if (*uniname == (u16)'\0') { - break; - } else if (*uniname == (u16)' ') { - lossy = true; - } else if (*uniname == (u16)'.') { - if (uniname < last_period) - lossy = true; - else - i = 8; - } else if (nls_wstrchr(bad_dos_chars, *uniname)) { - lossy = true; - *(dosname + i) = '_'; - i++; - } else { - len = convert_uni_to_ch(nls, buf, *uniname, &lossy); - - if (len > 1) { - if ((i >= 8) && ((i + len) > DOS_NAME_LENGTH)) - break; - - if ((i < 8) && ((i + len) > 8)) { - i = 8; - continue; - } - - lower = 0xFF; - - for (j = 0; j < len; j++, i++) - *(dosname + i) = *(buf + j); - } else { /* len == 1 */ - if ((*buf >= 'a') && (*buf <= 'z')) { - *(dosname + i) = *buf - ('a' - 'A'); - - if (i < 8) - lower |= 0x08; - else - lower |= 0x10; - } else if ((*buf >= 'A') && (*buf <= 'Z')) { - *(dosname + i) = *buf; - - if (i < 8) - upper |= 0x08; - else - upper |= 0x10; - } else { - *(dosname + i) = *buf; - } - i++; - } - } - - uniname++; - } - - if (*dosname == 0xE5) - *dosname = 0x05; - - if (*uniname != 0x0) - lossy = true; - - if (upper & lower) - p_dosname->name_case = 0xFF; - else - p_dosname->name_case = lower; - - if (p_lossy) - *p_lossy = lossy; -} - -void nls_dosname_to_uniname(struct super_block *sb, - struct uni_name_t *p_uniname, - struct dos_name_t *p_dosname) -{ - int i = 0, j, n = 0; - u8 buf[DOS_NAME_LENGTH + 2]; - u8 *dosname = p_dosname->name; - u16 *uniname = p_uniname->name; - struct nls_table *nls = EXFAT_SB(sb)->nls_disk; - - if (*dosname == 0x05) { - *buf = 0xE5; - i++; - n++; - } - - for (; i < 8; i++, n++) { - if (*(dosname + i) == ' ') - break; - - if ((*(dosname + i) >= 'A') && (*(dosname + i) <= 'Z') && - (p_dosname->name_case & 0x08)) - *(buf + n) = *(dosname + i) + ('a' - 'A'); - else - *(buf + n) = *(dosname + i); - } - if (*(dosname + 8) != ' ') { - *(buf + n) = '.'; - n++; - } - - for (i = 8; i < DOS_NAME_LENGTH; i++, n++) { - if (*(dosname + i) == ' ') - break; - - if ((*(dosname + i) >= 'A') && (*(dosname + i) <= 'Z') && - (p_dosname->name_case & 0x10)) - *(buf + n) = *(dosname + i) + ('a' - 'A'); - else - *(buf + n) = *(dosname + i); - } - *(buf + n) = '\0'; - - i = 0; - j = 0; - while (j < (MAX_NAME_LENGTH - 1)) { - if (*(buf + i) == '\0') - break; - - i += convert_ch_to_uni(nls, uniname, (buf + i), NULL); - - uniname++; - j++; - } - - *uniname = (u16)'\0'; -} - void nls_uniname_to_cstring(struct super_block *sb, u8 *p_cstring, struct uni_name_t *p_uniname) { diff --git a/drivers/staging/exfat/exfat_super.c b/drivers/staging/exfat/exfat_super.c index 58c7d66060f7e..b81d2a87b82ed 100644 --- a/drivers/staging/exfat/exfat_super.c +++ b/drivers/staging/exfat/exfat_super.c @@ -26,7 +26,7 @@ #include #include #include - +#include #include #include #include @@ -38,8 +38,8 @@ static struct kmem_cache *exfat_inode_cachep; -static int exfat_default_codepage = CONFIG_EXFAT_DEFAULT_CODEPAGE; -static char exfat_default_iocharset[] = CONFIG_EXFAT_DEFAULT_IOCHARSET; +static int exfat_default_codepage = CONFIG_STAGING_EXFAT_DEFAULT_CODEPAGE; +static char exfat_default_iocharset[] = CONFIG_STAGING_EXFAT_DEFAULT_IOCHARSET; #define INC_IVERSION(x) (inode_inc_iversion(x)) #define GET_IVERSION(x) (inode_peek_iversion_raw(x)) @@ -284,12 +284,12 @@ static const struct dentry_operations exfat_dentry_ops = { .d_compare = exfat_cmp, }; -static DEFINE_SEMAPHORE(z_sem); +static DEFINE_MUTEX(z_mutex); static inline void fs_sync(struct super_block *sb, bool do_sync) { if (do_sync) - bdev_sync(sb); + exfat_bdev_sync(sb); } /* @@ -353,26 +353,28 @@ static int ffsMountVol(struct super_block *sb) pr_info("[EXFAT] trying to mount...\n"); - down(&z_sem); + mutex_lock(&z_mutex); - buf_init(sb); + exfat_buf_init(sb); - sema_init(&p_fs->v_sem, 1); + mutex_init(&p_fs->v_mutex); p_fs->dev_ejected = 0; /* open the block device */ - bdev_open(sb); + exfat_bdev_open(sb); if (p_bd->sector_size < sb->s_blocksize) { - ret = FFS_MEDIAERR; + printk(KERN_INFO "EXFAT: mount failed - sector size %d less than blocksize %ld\n", + p_bd->sector_size, sb->s_blocksize); + ret = -EINVAL; goto out; } if (p_bd->sector_size > sb->s_blocksize) sb_set_blocksize(sb, p_bd->sector_size); /* read Sector 0 */ - if (sector_read(sb, 0, &tmp_bh, 1) != FFS_SUCCESS) { - ret = FFS_MEDIAERR; + if (sector_read(sb, 0, &tmp_bh, 1) != 0) { + ret = -EIO; goto out; } @@ -383,8 +385,8 @@ static int ffsMountVol(struct super_block *sb) /* check the validity of PBR */ if (GET16_A(p_pbr->signature) != PBR_SIGNATURE) { brelse(tmp_bh); - bdev_close(sb); - ret = FFS_FORMATERR; + exfat_bdev_close(sb); + ret = -EFSCORRUPTED; goto out; } @@ -394,16 +396,10 @@ static int ffsMountVol(struct super_block *sb) break; if (i < 53) { -#ifdef CONFIG_EXFAT_DONT_MOUNT_VFAT + /* Not sure how we'd get here, but complain if it does */ ret = -EINVAL; - printk(KERN_INFO "EXFAT: Attempted to mount VFAT filesystem\n"); + pr_info("EXFAT: Attempted to mount VFAT filesystem\n"); goto out; -#else - if (GET16(p_pbr->bpb + 11)) /* num_fat_sectors */ - ret = fat16_mount(sb, p_pbr); - else - ret = fat32_mount(sb, p_pbr); -#endif } else { ret = exfat_mount(sb, p_pbr); } @@ -411,38 +407,34 @@ static int ffsMountVol(struct super_block *sb) brelse(tmp_bh); if (ret) { - bdev_close(sb); + exfat_bdev_close(sb); goto out; } - if (p_fs->vol_type == EXFAT) { - ret = load_alloc_bitmap(sb); - if (ret) { - bdev_close(sb); - goto out; - } - ret = load_upcase_table(sb); - if (ret) { - free_alloc_bitmap(sb); - bdev_close(sb); - goto out; - } + ret = load_alloc_bitmap(sb); + if (ret) { + exfat_bdev_close(sb); + goto out; + } + ret = load_upcase_table(sb); + if (ret) { + free_alloc_bitmap(sb); + exfat_bdev_close(sb); + goto out; } if (p_fs->dev_ejected) { - if (p_fs->vol_type == EXFAT) { - free_upcase_table(sb); - free_alloc_bitmap(sb); - } - bdev_close(sb); - ret = FFS_MEDIAERR; + free_upcase_table(sb); + free_alloc_bitmap(sb); + exfat_bdev_close(sb); + ret = -EIO; goto out; } pr_info("[EXFAT] mounted successfully\n"); out: - up(&z_sem); + mutex_unlock(&z_mutex); return ret; } @@ -450,39 +442,37 @@ static int ffsMountVol(struct super_block *sb) static int ffsUmountVol(struct super_block *sb) { struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - int err = FFS_SUCCESS; + int err = 0; pr_info("[EXFAT] trying to unmount...\n"); - down(&z_sem); + mutex_lock(&z_mutex); /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); - fs_sync(sb, false); + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); - if (p_fs->vol_type == EXFAT) { - free_upcase_table(sb); - free_alloc_bitmap(sb); - } + free_upcase_table(sb); + free_alloc_bitmap(sb); - FAT_release_all(sb); - buf_release_all(sb); + exfat_fat_release_all(sb); + exfat_buf_release_all(sb); /* close the block device */ - bdev_close(sb); + exfat_bdev_close(sb); if (p_fs->dev_ejected) { pr_info("[EXFAT] unmounted with media errors. Device is already ejected.\n"); - err = FFS_MEDIAERR; + err = -EIO; } - buf_shutdown(sb); + exfat_buf_shutdown(sb); /* release the lock for file system critical section */ - up(&p_fs->v_sem); - up(&z_sem); + mutex_unlock(&p_fs->v_mutex); + mutex_unlock(&z_mutex); pr_info("[EXFAT] unmounted successfully\n"); @@ -491,18 +481,18 @@ static int ffsUmountVol(struct super_block *sb) static int ffsGetVolInfo(struct super_block *sb, struct vol_info_t *info) { - int err = FFS_SUCCESS; + int err = 0; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); /* check the validity of pointer parameters */ if (!info) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); if (p_fs->used_clusters == UINT_MAX) - p_fs->used_clusters = p_fs->fs_func->count_used_clusters(sb); + p_fs->used_clusters = exfat_count_used_clusters(sb); info->FatType = p_fs->vol_type; info->ClusterSize = p_fs->cluster_size; @@ -511,31 +501,31 @@ static int ffsGetVolInfo(struct super_block *sb, struct vol_info_t *info) info->FreeClusters = info->NumClusters - info->UsedClusters; if (p_fs->dev_ejected) - err = FFS_MEDIAERR; + err = -EIO; /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return err; } static int ffsSyncVol(struct super_block *sb, bool do_sync) { - int err = FFS_SUCCESS; + int err = 0; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* synchronize the file system */ fs_sync(sb, do_sync); fs_set_vol_flags(sb, VOL_CLEAN); if (p_fs->dev_ejected) - err = FFS_MEDIAERR; + err = -EIO; /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return err; } @@ -559,10 +549,10 @@ static int ffsLookupFile(struct inode *inode, char *path, struct file_id_t *fid) /* check the validity of pointer parameters */ if (!fid || !path || (*path == '\0')) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check the validity of directory name in the given pathname */ ret = resolve_path(inode, path, &dir, &uni_name); @@ -575,10 +565,10 @@ static int ffsLookupFile(struct inode *inode, char *path, struct file_id_t *fid) goto out; /* search the file name for directories */ - dentry = p_fs->fs_func->find_dir_entry(sb, &dir, &uni_name, num_entries, - &dos_name, TYPE_ALL); + dentry = exfat_find_dir_entry(sb, &dir, &uni_name, num_entries, + &dos_name, TYPE_ALL); if (dentry < -1) { - ret = FFS_NOTFOUND; + ret = -ENOENT; goto out; } @@ -597,46 +587,36 @@ static int ffsLookupFile(struct inode *inode, char *path, struct file_id_t *fid) fid->size = 0; fid->start_clu = p_fs->root_dir; } else { - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &dir, dentry, - ES_2_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep + 1; - } else { - ep = get_entry_in_dir(sb, &dir, dentry, NULL); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep; + es = get_entry_set_in_dir(sb, &dir, dentry, + ES_2_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } + ep2 = ep + 1; - fid->type = p_fs->fs_func->get_entry_type(ep); + fid->type = exfat_get_entry_type(ep); fid->rwoffset = 0; fid->hint_last_off = -1; - fid->attr = p_fs->fs_func->get_entry_attr(ep); + fid->attr = exfat_get_entry_attr(ep); - fid->size = p_fs->fs_func->get_entry_size(ep2); + fid->size = exfat_get_entry_size(ep2); if ((fid->type == TYPE_FILE) && (fid->size == 0)) { fid->flags = (p_fs->vol_type == EXFAT) ? 0x03 : 0x01; fid->start_clu = CLUSTER_32(~0); } else { - fid->flags = p_fs->fs_func->get_entry_flag(ep2); - fid->start_clu = p_fs->fs_func->get_entry_clu0(ep2); + fid->flags = exfat_get_entry_flag(ep2); + fid->start_clu = exfat_get_entry_clu0(ep2); } - if (p_fs->vol_type == EXFAT) - release_entry_set(es); + release_entry_set(es); } if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -648,14 +628,14 @@ static int ffsCreateFile(struct inode *inode, char *path, u8 mode, struct uni_name_t uni_name; struct super_block *sb = inode->i_sb; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - int ret; + int ret = 0; /* check the validity of pointer parameters */ if (!fid || !path || (*path == '\0')) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check the validity of directory name in the given pathname */ ret = resolve_path(inode, path, &dir, &uni_name); @@ -667,17 +647,17 @@ static int ffsCreateFile(struct inode *inode, char *path, u8 mode, /* create a new file */ ret = create_file(inode, &dir, &uni_name, mode, fid); -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -697,18 +677,18 @@ static int ffsReadFile(struct inode *inode, struct file_id_t *fid, void *buffer, /* check the validity of the given file id */ if (!fid) - return FFS_INVALIDFID; + return -EINVAL; /* check the validity of pointer parameters */ if (!buffer) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check if the given file ID is opened */ if (fid->type != TYPE_FILE) { - ret = FFS_PERMISSIONERR; + ret = -EPERM; goto out; } @@ -721,7 +701,7 @@ static int ffsReadFile(struct inode *inode, struct file_id_t *fid, void *buffer, if (count == 0) { if (rcount) *rcount = 0; - ret = FFS_EOF; + ret = 0; goto out; } @@ -742,9 +722,11 @@ static int ffsReadFile(struct inode *inode, struct file_id_t *fid, void *buffer, } while (clu_offset > 0) { - /* clu = FAT_read(sb, clu); */ - if (FAT_read(sb, clu, &clu) == -1) - return FFS_MEDIAERR; + /* clu = exfat_fat_read(sb, clu); */ + if (exfat_fat_read(sb, clu, &clu) == -1) { + ret = -EIO; + goto out; + } clu_offset--; } @@ -772,13 +754,13 @@ static int ffsReadFile(struct inode *inode, struct file_id_t *fid, void *buffer, if ((offset == 0) && (oneblkread == p_bd->sector_size)) { if (sector_read(sb, LogSector, &tmp_bh, 1) != - FFS_SUCCESS) + 0) goto err_out; memcpy((char *)buffer + read_bytes, (char *)tmp_bh->b_data, (s32)oneblkread); } else { if (sector_read(sb, LogSector, &tmp_bh, 1) != - FFS_SUCCESS) + 0) goto err_out; memcpy((char *)buffer + read_bytes, (char *)tmp_bh->b_data + offset, @@ -797,11 +779,11 @@ static int ffsReadFile(struct inode *inode, struct file_id_t *fid, void *buffer, *rcount = read_bytes; if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -814,7 +796,7 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, s32 num_clusters, num_alloc, num_alloced = (s32)~0; int ret = 0; u32 clu, last_clu; - sector_t LogSector, sector = 0; + sector_t LogSector; u64 oneblkwrite, write_bytes; struct chain_t new_clu; struct timestamp_t tm; @@ -827,18 +809,18 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, /* check the validity of the given file id */ if (!fid) - return FFS_INVALIDFID; + return -EINVAL; /* check the validity of pointer parameters */ if (!buffer) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check if the given file ID is opened */ if (fid->type != TYPE_FILE) { - ret = FFS_PERMISSIONERR; + ret = -EPERM; goto out; } @@ -848,7 +830,7 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, if (count == 0) { if (wcount) *wcount = 0; - ret = FFS_SUCCESS; + ret = 0; goto out; } @@ -864,7 +846,8 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, while (count > 0) { clu_offset = (s32)(fid->rwoffset >> p_fs->cluster_size_bits); - clu = last_clu = fid->start_clu; + clu = fid->start_clu; + last_clu = fid->start_clu; if (fid->flags == 0x03) { if ((clu_offset > 0) && (clu != CLUSTER_32(~0))) { @@ -885,9 +868,9 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, while ((clu_offset > 0) && (clu != CLUSTER_32(~0))) { last_clu = clu; - /* clu = FAT_read(sb, clu); */ - if (FAT_read(sb, clu, &clu) == -1) { - ret = FFS_MEDIAERR; + /* clu = exfat_fat_read(sb, clu); */ + if (exfat_fat_read(sb, clu, &clu) == -1) { + ret = -EIO; goto out; } clu_offset--; @@ -903,13 +886,13 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, new_clu.flags = fid->flags; /* (1) allocate a chain of clusters */ - num_alloced = p_fs->fs_func->alloc_cluster(sb, - num_alloc, - &new_clu); + num_alloced = exfat_alloc_cluster(sb, + num_alloc, + &new_clu); if (num_alloced == 0) break; if (num_alloced < 0) { - ret = FFS_MEDIAERR; + ret = num_alloced; goto out; } @@ -928,7 +911,7 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, modified = true; } if (new_clu.flags == 0x01) - FAT_write(sb, last_clu, new_clu.dir); + exfat_fat_write(sb, last_clu, new_clu.dir); } num_clusters += num_alloced; @@ -957,12 +940,12 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, if ((offset == 0) && (oneblkwrite == p_bd->sector_size)) { if (sector_read(sb, LogSector, &tmp_bh, 0) != - FFS_SUCCESS) + 0) goto err_out; memcpy((char *)tmp_bh->b_data, (char *)buffer + write_bytes, (s32)oneblkwrite); if (sector_write(sb, LogSector, tmp_bh, 0) != - FFS_SUCCESS) { + 0) { brelse(tmp_bh); goto err_out; } @@ -970,18 +953,18 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, if ((offset > 0) || ((fid->rwoffset + oneblkwrite) < fid->size)) { if (sector_read(sb, LogSector, &tmp_bh, 1) != - FFS_SUCCESS) + 0) goto err_out; } else { if (sector_read(sb, LogSector, &tmp_bh, 0) != - FFS_SUCCESS) + 0) goto err_out; } memcpy((char *)tmp_bh->b_data + offset, (char *)buffer + write_bytes, (s32)oneblkwrite); if (sector_write(sb, LogSector, tmp_bh, 0) != - FFS_SUCCESS) { + 0) { brelse(tmp_bh); goto err_out; } @@ -1002,46 +985,31 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, brelse(tmp_bh); /* (3) update the direcoty entry */ - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &(fid->dir), fid->entry, - ES_ALL_ENTRIES, &ep); - if (!es) - goto err_out; - ep2 = ep + 1; - } else { - ep = get_entry_in_dir(sb, &(fid->dir), fid->entry, §or); - if (!ep) - goto err_out; - ep2 = ep; - } - - p_fs->fs_func->set_entry_time(ep, tm_current(&tm), TM_MODIFY); - p_fs->fs_func->set_entry_attr(ep, fid->attr); + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_ALL_ENTRIES, &ep); + if (!es) + goto err_out; + ep2 = ep + 1; - if (p_fs->vol_type != EXFAT) - buf_modify(sb, sector); + exfat_set_entry_time(ep, tm_current(&tm), TM_MODIFY); + exfat_set_entry_attr(ep, fid->attr); if (modified) { - if (p_fs->fs_func->get_entry_flag(ep2) != fid->flags) - p_fs->fs_func->set_entry_flag(ep2, fid->flags); - - if (p_fs->fs_func->get_entry_size(ep2) != fid->size) - p_fs->fs_func->set_entry_size(ep2, fid->size); + if (exfat_get_entry_flag(ep2) != fid->flags) + exfat_set_entry_flag(ep2, fid->flags); - if (p_fs->fs_func->get_entry_clu0(ep2) != fid->start_clu) - p_fs->fs_func->set_entry_clu0(ep2, fid->start_clu); + if (exfat_get_entry_size(ep2) != fid->size) + exfat_set_entry_size(ep2, fid->size); - if (p_fs->vol_type != EXFAT) - buf_modify(sb, sector); + if (exfat_get_entry_clu0(ep2) != fid->start_clu) + exfat_set_entry_clu0(ep2, fid->start_clu); } - if (p_fs->vol_type == EXFAT) { - update_dir_checksum_with_entry_set(sb, es); - release_entry_set(es); - } + update_dir_checksum_with_entry_set(sb, es); + release_entry_set(es); -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif @@ -1051,14 +1019,14 @@ static int ffsWriteFile(struct inode *inode, struct file_id_t *fid, *wcount = write_bytes; if (num_alloced == 0) - ret = FFS_FULL; + ret = -ENOSPC; else if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1068,7 +1036,6 @@ static int ffsTruncateFile(struct inode *inode, u64 old_size, u64 new_size) s32 num_clusters; u32 last_clu = CLUSTER_32(0); int ret = 0; - sector_t sector = 0; struct chain_t clu; struct timestamp_t tm; struct dentry_t *ep, *ep2; @@ -1081,11 +1048,11 @@ static int ffsTruncateFile(struct inode *inode, u64 old_size, u64 new_size) new_size); /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check if the given file ID is opened */ if (fid->type != TYPE_FILE) { - ret = FFS_PERMISSIONERR; + ret = -EPERM; goto out; } @@ -1095,7 +1062,7 @@ static int ffsTruncateFile(struct inode *inode, u64 old_size, u64 new_size) } if (old_size <= new_size) { - ret = FFS_SUCCESS; + ret = 0; goto out; } @@ -1114,8 +1081,8 @@ static int ffsTruncateFile(struct inode *inode, u64 old_size, u64 new_size) } else { while (num_clusters > 0) { last_clu = clu.dir; - if (FAT_read(sb, clu.dir, &clu.dir) == -1) { - ret = FFS_MEDIAERR; + if (exfat_fat_read(sb, clu.dir, &clu.dir) == -1) { + ret = -EIO; goto out; } num_clusters--; @@ -1133,65 +1100,52 @@ static int ffsTruncateFile(struct inode *inode, u64 old_size, u64 new_size) } /* (1) update the directory entry */ - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, - ES_ALL_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep + 1; - } else { - ep = get_entry_in_dir(sb, &(fid->dir), fid->entry, §or); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_ALL_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } - ep2 = ep; - } + ep2 = ep + 1; - p_fs->fs_func->set_entry_time(ep, tm_current(&tm), TM_MODIFY); - p_fs->fs_func->set_entry_attr(ep, fid->attr); + exfat_set_entry_time(ep, tm_current(&tm), TM_MODIFY); + exfat_set_entry_attr(ep, fid->attr); - p_fs->fs_func->set_entry_size(ep2, new_size); + exfat_set_entry_size(ep2, new_size); if (new_size == 0) { - p_fs->fs_func->set_entry_flag(ep2, 0x01); - p_fs->fs_func->set_entry_clu0(ep2, CLUSTER_32(0)); + exfat_set_entry_flag(ep2, 0x01); + exfat_set_entry_clu0(ep2, CLUSTER_32(0)); } - if (p_fs->vol_type != EXFAT) { - buf_modify(sb, sector); - } else { - update_dir_checksum_with_entry_set(sb, es); - release_entry_set(es); - } + update_dir_checksum_with_entry_set(sb, es); + release_entry_set(es); /* (2) cut off from the FAT chain */ if (last_clu != CLUSTER_32(0)) { if (fid->flags == 0x01) - FAT_write(sb, last_clu, CLUSTER_32(~0)); + exfat_fat_write(sb, last_clu, CLUSTER_32(~0)); } /* (3) free the clusters */ - p_fs->fs_func->free_cluster(sb, &clu, 0); + exfat_free_cluster(sb, &clu, 0); /* hint information */ fid->hint_last_off = -1; if (fid->rwoffset > fid->size) fid->rwoffset = fid->size; -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: pr_debug("%s exited (%d)\n", __func__, ret); /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1232,14 +1186,14 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, /* check the validity of the given file id */ if (!fid) - return FFS_INVALIDFID; + return -EINVAL; /* check the validity of pointer parameters */ if (!new_path || (*new_path == '\0')) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); update_parent_info(fid, old_parent_inode); @@ -1252,19 +1206,19 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, /* check if the old file is "." or ".." */ if (p_fs->vol_type != EXFAT) { if ((olddir.dir != p_fs->root_dir) && (dentry < 2)) { - ret = FFS_PERMISSIONERR; + ret = -EPERM; goto out2; } } ep = get_entry_in_dir(sb, &olddir, dentry, NULL); if (!ep) { - ret = FFS_MEDIAERR; + ret = -ENOENT; goto out2; } - if (p_fs->fs_func->get_entry_attr(ep) & ATTR_READONLY) { - ret = FFS_PERMISSIONERR; + if (exfat_get_entry_attr(ep) & ATTR_READONLY) { + ret = -EPERM; goto out2; } @@ -1272,18 +1226,18 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, if (new_inode) { u32 entry_type; - ret = FFS_MEDIAERR; + ret = -ENOENT; new_fid = &EXFAT_I(new_inode)->fid; update_parent_info(new_fid, new_parent_inode); - p_dir = &(new_fid->dir); + p_dir = &new_fid->dir; new_entry = new_fid->entry; ep = get_entry_in_dir(sb, p_dir, new_entry, NULL); if (!ep) goto out; - entry_type = p_fs->fs_func->get_entry_type(ep); + entry_type = exfat_get_entry_type(ep); if (entry_type == TYPE_DIR) { struct chain_t new_clu; @@ -1294,7 +1248,7 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, new_clu.flags = new_fid->flags; if (!is_dir_empty(sb, &new_clu)) { - ret = FFS_FILEEXIST; + ret = -EEXIST; goto out; } } @@ -1314,30 +1268,30 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, ret = move_file(new_parent_inode, &olddir, dentry, &newdir, &uni_name, fid); - if ((ret == FFS_SUCCESS) && new_inode) { + if ((ret == 0) && new_inode) { /* delete entries of new_dir */ ep = get_entry_in_dir(sb, p_dir, new_entry, NULL); if (!ep) goto out; - num_entries = p_fs->fs_func->count_ext_entries(sb, p_dir, - new_entry, ep); + num_entries = exfat_count_ext_entries(sb, p_dir, + new_entry, ep); if (num_entries < 0) goto out; - p_fs->fs_func->delete_dir_entry(sb, p_dir, new_entry, 0, - num_entries + 1); + exfat_delete_dir_entry(sb, p_dir, new_entry, 0, + num_entries + 1); } out: -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out2: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1345,7 +1299,7 @@ static int ffsMoveFile(struct inode *old_parent_inode, struct file_id_t *fid, static int ffsRemoveFile(struct inode *inode, struct file_id_t *fid) { s32 dentry; - int ret = FFS_SUCCESS; + int ret = 0; struct chain_t dir, clu_to_free; struct dentry_t *ep; struct super_block *sb = inode->i_sb; @@ -1353,10 +1307,10 @@ static int ffsRemoveFile(struct inode *inode, struct file_id_t *fid) /* check the validity of the given file id */ if (!fid) - return FFS_INVALIDFID; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); dir.dir = fid->dir.dir; dir.size = fid->dir.size; @@ -1366,12 +1320,12 @@ static int ffsRemoveFile(struct inode *inode, struct file_id_t *fid) ep = get_entry_in_dir(sb, &dir, dentry, NULL); if (!ep) { - ret = FFS_MEDIAERR; + ret = -ENOENT; goto out; } - if (p_fs->fs_func->get_entry_attr(ep) & ATTR_READONLY) { - ret = FFS_PERMISSIONERR; + if (exfat_get_entry_attr(ep) & ATTR_READONLY) { + ret = -EPERM; goto out; } fs_set_vol_flags(sb, VOL_DIRTY); @@ -1384,22 +1338,22 @@ static int ffsRemoveFile(struct inode *inode, struct file_id_t *fid) clu_to_free.flags = fid->flags; /* (2) free the clusters */ - p_fs->fs_func->free_cluster(sb, &clu_to_free, 0); + exfat_free_cluster(sb, &clu_to_free, 0); fid->size = 0; fid->start_clu = CLUSTER_32(~0); fid->flags = (p_fs->vol_type == EXFAT) ? 0x03 : 0x01; -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1409,7 +1363,7 @@ static int ffsRemoveFile(struct inode *inode, struct file_id_t *fid) static int ffsSetAttr(struct inode *inode, u32 attr) { u32 type; - int ret = FFS_SUCCESS; + int ret = 0; sector_t sector = 0; struct dentry_t *ep; struct super_block *sb = inode->i_sb; @@ -1420,49 +1374,40 @@ static int ffsSetAttr(struct inode *inode, u32 attr) if (fid->attr == attr) { if (p_fs->dev_ejected) - return FFS_MEDIAERR; - return FFS_SUCCESS; + return -EIO; + return 0; } if (is_dir) { if ((fid->dir.dir == p_fs->root_dir) && (fid->entry == -1)) { if (p_fs->dev_ejected) - return FFS_MEDIAERR; - return FFS_SUCCESS; + return -EIO; + return 0; } } /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* get the directory entry of given file */ - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &(fid->dir), fid->entry, - ES_ALL_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - } else { - ep = get_entry_in_dir(sb, &(fid->dir), fid->entry, §or); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_ALL_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } - type = p_fs->fs_func->get_entry_type(ep); + type = exfat_get_entry_type(ep); if (((type == TYPE_FILE) && (attr & ATTR_SUBDIR)) || ((type == TYPE_DIR) && (!(attr & ATTR_SUBDIR)))) { if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; else - ret = FFS_ERROR; + ret = -EINVAL; - if (p_fs->vol_type == EXFAT) - release_entry_set(es); + release_entry_set(es); goto out; } @@ -1470,25 +1415,21 @@ static int ffsSetAttr(struct inode *inode, u32 attr) /* set the file attribute */ fid->attr = attr; - p_fs->fs_func->set_entry_attr(ep, attr); + exfat_set_entry_attr(ep, attr); - if (p_fs->vol_type != EXFAT) { - buf_modify(sb, sector); - } else { - update_dir_checksum_with_entry_set(sb, es); - release_entry_set(es); - } + update_dir_checksum_with_entry_set(sb, es); + release_entry_set(es); -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1496,9 +1437,8 @@ static int ffsSetAttr(struct inode *inode, u32 attr) static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) { - sector_t sector = 0; s32 count; - int ret = FFS_SUCCESS; + int ret = 0; struct chain_t dir; struct uni_name_t uni_name; struct timestamp_t tm; @@ -1512,7 +1452,7 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) pr_debug("%s entered\n", __func__); /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); if (is_dir) { if ((fid->dir.dir == p_fs->root_dir) && @@ -1541,40 +1481,30 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) count = count_dos_name_entries(sb, &dir, TYPE_DIR); if (count < 0) { - ret = FFS_MEDIAERR; + ret = count; /* propagate error upward */ goto out; } info->NumSubdirs = count; if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; goto out; } } /* get the directory entry of given file or directory */ - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &(fid->dir), fid->entry, - ES_2_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep + 1; - } else { - ep = get_entry_in_dir(sb, &(fid->dir), fid->entry, §or); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep; - buf_lock(sb, sector); + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_2_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } + ep2 = ep + 1; /* set FILE_INFO structure using the acquired struct dentry_t */ - info->Attr = p_fs->fs_func->get_entry_attr(ep); + info->Attr = exfat_get_entry_attr(ep); - p_fs->fs_func->get_entry_time(ep, &tm, TM_CREATE); + exfat_get_entry_time(ep, &tm, TM_CREATE); info->CreateTimestamp.Year = tm.year; info->CreateTimestamp.Month = tm.mon; info->CreateTimestamp.Day = tm.day; @@ -1583,7 +1513,7 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) info->CreateTimestamp.Second = tm.sec; info->CreateTimestamp.MilliSecond = 0; - p_fs->fs_func->get_entry_time(ep, &tm, TM_MODIFY); + exfat_get_entry_time(ep, &tm, TM_MODIFY); info->ModifyTimestamp.Year = tm.year; info->ModifyTimestamp.Month = tm.mon; info->ModifyTimestamp.Day = tm.day; @@ -1594,31 +1524,19 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) memset((char *)&info->AccessTimestamp, 0, sizeof(struct date_time_t)); - *(uni_name.name) = 0x0; + *uni_name.name = 0x0; /* XXX this is very bad for exfat cuz name is already included in es. * API should be revised */ - p_fs->fs_func->get_uni_name_from_ext_entry(sb, &(fid->dir), fid->entry, - uni_name.name); - if (*uni_name.name == 0x0 && p_fs->vol_type != EXFAT) - get_uni_name_from_dos_entry(sb, (struct dos_dentry_t *)ep, - &uni_name, 0x1); + exfat_get_uni_name_from_ext_entry(sb, &fid->dir, fid->entry, + uni_name.name); nls_uniname_to_cstring(sb, info->Name, &uni_name); - if (p_fs->vol_type == EXFAT) { - info->NumSubdirs = 2; - } else { - buf_unlock(sb, sector); - get_uni_name_from_dos_entry(sb, (struct dos_dentry_t *)ep, - &uni_name, 0x0); - nls_uniname_to_cstring(sb, info->ShortName, &uni_name); - info->NumSubdirs = 0; - } + info->NumSubdirs = 2; - info->Size = p_fs->fs_func->get_entry_size(ep2); + info->Size = exfat_get_entry_size(ep2); - if (p_fs->vol_type == EXFAT) - release_entry_set(es); + release_entry_set(es); if (is_dir) { dir.dir = fid->start_clu; @@ -1630,18 +1548,18 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) count = count_dos_name_entries(sb, &dir, TYPE_DIR); if (count < 0) { - ret = FFS_MEDIAERR; + ret = count; /* propagate error upward */ goto out; } info->NumSubdirs += count; } if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); pr_debug("%s exited successfully\n", __func__); return ret; @@ -1649,8 +1567,7 @@ static int ffsReadStat(struct inode *inode, struct dir_entry_t *info) static int ffsWriteStat(struct inode *inode, struct dir_entry_t *info) { - sector_t sector = 0; - int ret = FFS_SUCCESS; + int ret = 0; struct timestamp_t tm; struct dentry_t *ep, *ep2; struct entry_set_cache_t *es = NULL; @@ -1662,14 +1579,14 @@ static int ffsWriteStat(struct inode *inode, struct dir_entry_t *info) pr_debug("%s entered (inode %p info %p\n", __func__, inode, info); /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); if (is_dir) { if ((fid->dir.dir == p_fs->root_dir) && (fid->entry == -1)) { if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; - ret = FFS_SUCCESS; + ret = -EIO; + ret = 0; goto out; } } @@ -1677,25 +1594,15 @@ static int ffsWriteStat(struct inode *inode, struct dir_entry_t *info) fs_set_vol_flags(sb, VOL_DIRTY); /* get the directory entry of given file or directory */ - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &(fid->dir), fid->entry, - ES_ALL_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep + 1; - } else { - /* for other than exfat */ - ep = get_entry_in_dir(sb, &(fid->dir), fid->entry, §or); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } - ep2 = ep; + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_ALL_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } + ep2 = ep + 1; - p_fs->fs_func->set_entry_attr(ep, info->Attr); + exfat_set_entry_attr(ep, info->Attr); /* set FILE_INFO structure using the acquired struct dentry_t */ tm.sec = info->CreateTimestamp.Second; @@ -1704,7 +1611,7 @@ static int ffsWriteStat(struct inode *inode, struct dir_entry_t *info) tm.day = info->CreateTimestamp.Day; tm.mon = info->CreateTimestamp.Month; tm.year = info->CreateTimestamp.Year; - p_fs->fs_func->set_entry_time(ep, &tm, TM_CREATE); + exfat_set_entry_time(ep, &tm, TM_CREATE); tm.sec = info->ModifyTimestamp.Second; tm.min = info->ModifyTimestamp.Minute; @@ -1712,23 +1619,19 @@ static int ffsWriteStat(struct inode *inode, struct dir_entry_t *info) tm.day = info->ModifyTimestamp.Day; tm.mon = info->ModifyTimestamp.Month; tm.year = info->ModifyTimestamp.Year; - p_fs->fs_func->set_entry_time(ep, &tm, TM_MODIFY); + exfat_set_entry_time(ep, &tm, TM_MODIFY); - p_fs->fs_func->set_entry_size(ep2, info->Size); + exfat_set_entry_size(ep2, info->Size); - if (p_fs->vol_type != EXFAT) { - buf_modify(sb, sector); - } else { - update_dir_checksum_with_entry_set(sb, es); - release_entry_set(es); - } + update_dir_checksum_with_entry_set(sb, es); + release_entry_set(es); if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); pr_debug("%s exited (%d)\n", __func__, ret); @@ -1740,8 +1643,7 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) s32 num_clusters, num_alloced; bool modified = false; u32 last_clu; - int ret = FFS_SUCCESS; - sector_t sector = 0; + int ret = 0; struct chain_t new_clu; struct dentry_t *ep; struct entry_set_cache_t *es = NULL; @@ -1751,10 +1653,10 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) /* check the validity of pointer parameters */ if (!clu) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); fid->rwoffset = (s64)(clu_offset) << p_fs->cluster_size_bits; @@ -1785,8 +1687,8 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) while ((clu_offset > 0) && (*clu != CLUSTER_32(~0))) { last_clu = *clu; - if (FAT_read(sb, *clu, clu) == -1) { - ret = FFS_MEDIAERR; + if (exfat_fat_read(sb, *clu, clu) == -1) { + ret = -EIO; goto out; } clu_offset--; @@ -1802,12 +1704,12 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) new_clu.flags = fid->flags; /* (1) allocate a cluster */ - num_alloced = p_fs->fs_func->alloc_cluster(sb, 1, &new_clu); + num_alloced = exfat_alloc_cluster(sb, 1, &new_clu); if (num_alloced < 0) { - ret = FFS_MEDIAERR; + ret = -EIO; goto out; } else if (num_alloced == 0) { - ret = FFS_FULL; + ret = -ENOSPC; goto out; } @@ -1825,49 +1727,32 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) modified = true; } if (new_clu.flags == 0x01) - FAT_write(sb, last_clu, new_clu.dir); + exfat_fat_write(sb, last_clu, new_clu.dir); } num_clusters += num_alloced; *clu = new_clu.dir; - if (p_fs->vol_type == EXFAT) { - es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, - ES_ALL_ENTRIES, &ep); - if (!es) { - ret = FFS_MEDIAERR; - goto out; - } - /* get stream entry */ - ep++; + es = get_entry_set_in_dir(sb, &fid->dir, fid->entry, + ES_ALL_ENTRIES, &ep); + if (!es) { + ret = -ENOENT; + goto out; } + /* get stream entry */ + ep++; /* (3) update directory entry */ if (modified) { - if (p_fs->vol_type != EXFAT) { - ep = get_entry_in_dir(sb, &(fid->dir), - fid->entry, §or); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } - } + if (exfat_get_entry_flag(ep) != fid->flags) + exfat_set_entry_flag(ep, fid->flags); - if (p_fs->fs_func->get_entry_flag(ep) != fid->flags) - p_fs->fs_func->set_entry_flag(ep, fid->flags); - - if (p_fs->fs_func->get_entry_clu0(ep) != fid->start_clu) - p_fs->fs_func->set_entry_clu0(ep, - fid->start_clu); - - if (p_fs->vol_type != EXFAT) - buf_modify(sb, sector); + if (exfat_get_entry_clu0(ep) != fid->start_clu) + exfat_set_entry_clu0(ep, fid->start_clu); } - if (p_fs->vol_type == EXFAT) { - update_dir_checksum_with_entry_set(sb, es); - release_entry_set(es); - } + update_dir_checksum_with_entry_set(sb, es); + release_entry_set(es); /* add number of new blocks to inode */ inode->i_blocks += num_alloced << (p_fs->cluster_size_bits - 9); @@ -1878,11 +1763,11 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) fid->hint_last_clu = *clu; if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1893,7 +1778,7 @@ static int ffsMapCluster(struct inode *inode, s32 clu_offset, u32 *clu) static int ffsCreateDir(struct inode *inode, char *path, struct file_id_t *fid) { - int ret = FFS_SUCCESS; + int ret = 0; struct chain_t dir; struct uni_name_t uni_name; struct super_block *sb = inode->i_sb; @@ -1903,10 +1788,10 @@ static int ffsCreateDir(struct inode *inode, char *path, struct file_id_t *fid) /* check the validity of pointer parameters */ if (!fid || !path || (*path == '\0')) - return FFS_ERROR; + return -EINVAL; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); /* check the validity of directory name in the given old pathname */ ret = resolve_path(inode, path, &dir, &uni_name); @@ -1917,16 +1802,16 @@ static int ffsCreateDir(struct inode *inode, char *path, struct file_id_t *fid) ret = create_dir(inode, &dir, &uni_name, fid); -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -1934,7 +1819,7 @@ static int ffsCreateDir(struct inode *inode, char *path, struct file_id_t *fid) static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) { int i, dentry, clu_offset; - int ret = FFS_SUCCESS; + int ret = 0; s32 dentries_per_clu, dentries_per_clu_bits = 0; u32 type; sector_t sector; @@ -1944,19 +1829,18 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) struct dentry_t *ep; struct super_block *sb = inode->i_sb; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); - struct fs_func *fs_func = p_fs->fs_func; struct file_id_t *fid = &(EXFAT_I(inode)->fid); /* check the validity of pointer parameters */ if (!dir_entry) - return FFS_ERROR; + return -EINVAL; /* check if the given file ID is opened */ if (fid->type != TYPE_DIR) - return FFS_PERMISSIONERR; + return -ENOTDIR; /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); if (fid->entry == -1) { dir.dir = p_fs->root_dir; @@ -2001,9 +1885,9 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) } while (clu_offset > 0) { - /* clu.dir = FAT_read(sb, clu.dir); */ - if (FAT_read(sb, clu.dir, &clu.dir) == -1) { - ret = FFS_MEDIAERR; + /* clu.dir = exfat_fat_read(sb, clu.dir); */ + if (exfat_fat_read(sb, clu.dir, &clu.dir) == -1) { + ret = -EIO; goto out; } clu_offset--; @@ -2023,10 +1907,10 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) for ( ; i < dentries_per_clu; i++, dentry++) { ep = get_entry_in_dir(sb, &clu, i, §or); if (!ep) { - ret = FFS_MEDIAERR; + ret = -ENOENT; goto out; } - type = fs_func->get_entry_type(ep); + type = exfat_get_entry_type(ep); if (type == TYPE_UNUSED) break; @@ -2034,10 +1918,10 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) if ((type != TYPE_FILE) && (type != TYPE_DIR)) continue; - buf_lock(sb, sector); - dir_entry->Attr = fs_func->get_entry_attr(ep); + exfat_buf_lock(sb, sector); + dir_entry->Attr = exfat_get_entry_attr(ep); - fs_func->get_entry_time(ep, &tm, TM_CREATE); + exfat_get_entry_time(ep, &tm, TM_CREATE); dir_entry->CreateTimestamp.Year = tm.year; dir_entry->CreateTimestamp.Month = tm.mon; dir_entry->CreateTimestamp.Day = tm.day; @@ -2046,7 +1930,7 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) dir_entry->CreateTimestamp.Second = tm.sec; dir_entry->CreateTimestamp.MilliSecond = 0; - fs_func->get_entry_time(ep, &tm, TM_MODIFY); + exfat_get_entry_time(ep, &tm, TM_MODIFY); dir_entry->ModifyTimestamp.Year = tm.year; dir_entry->ModifyTimestamp.Month = tm.mon; dir_entry->ModifyTimestamp.Day = tm.day; @@ -2058,31 +1942,19 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) memset((char *)&dir_entry->AccessTimestamp, 0, sizeof(struct date_time_t)); - *(uni_name.name) = 0x0; - fs_func->get_uni_name_from_ext_entry(sb, &dir, dentry, - uni_name.name); - if (*uni_name.name == 0x0 && p_fs->vol_type != EXFAT) - get_uni_name_from_dos_entry(sb, - (struct dos_dentry_t *)ep, - &uni_name, 0x1); + *uni_name.name = 0x0; + exfat_get_uni_name_from_ext_entry(sb, &dir, dentry, + uni_name.name); nls_uniname_to_cstring(sb, dir_entry->Name, &uni_name); - buf_unlock(sb, sector); + exfat_buf_unlock(sb, sector); - if (p_fs->vol_type == EXFAT) { - ep = get_entry_in_dir(sb, &clu, i + 1, NULL); - if (!ep) { - ret = FFS_MEDIAERR; - goto out; - } - } else { - get_uni_name_from_dos_entry(sb, - (struct dos_dentry_t *)ep, - &uni_name, 0x0); - nls_uniname_to_cstring(sb, dir_entry->ShortName, - &uni_name); + ep = get_entry_in_dir(sb, &clu, i + 1, NULL); + if (!ep) { + ret = -ENOENT; + goto out; } - dir_entry->Size = fs_func->get_entry_size(ep); + dir_entry->Size = exfat_get_entry_size(ep); /* hint information */ if (dir.dir == CLUSTER_32(0)) { /* FAT16 root_dir */ @@ -2095,7 +1967,7 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) fid->rwoffset = (s64)(++dentry); if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; goto out; } @@ -2108,24 +1980,24 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) else clu.dir = CLUSTER_32(~0); } else { - /* clu.dir = FAT_read(sb, clu.dir); */ - if (FAT_read(sb, clu.dir, &clu.dir) == -1) { - ret = FFS_MEDIAERR; + /* clu.dir = exfat_fat_read(sb, clu.dir); */ + if (exfat_fat_read(sb, clu.dir, &clu.dir) == -1) { + ret = -EIO; goto out; } } } - *(dir_entry->Name) = '\0'; + *dir_entry->Name = '\0'; fid->rwoffset = (s64)(++dentry); if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -2133,14 +2005,14 @@ static int ffsReadDir(struct inode *inode, struct dir_entry_t *dir_entry) static int ffsRemoveDir(struct inode *inode, struct file_id_t *fid) { s32 dentry; - int ret = FFS_SUCCESS; + int ret = 0; struct chain_t dir, clu_to_free; struct super_block *sb = inode->i_sb; struct fs_info_t *p_fs = &(EXFAT_SB(sb)->fs_info); /* check the validity of the given file id */ if (!fid) - return FFS_INVALIDFID; + return -EINVAL; dir.dir = fid->dir.dir; dir.size = fid->dir.size; @@ -2151,18 +2023,18 @@ static int ffsRemoveDir(struct inode *inode, struct file_id_t *fid) /* check if the file is "." or ".." */ if (p_fs->vol_type != EXFAT) { if ((dir.dir != p_fs->root_dir) && (dentry < 2)) - return FFS_PERMISSIONERR; + return -EPERM; } /* acquire the lock for file system critical section */ - down(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); clu_to_free.dir = fid->start_clu; clu_to_free.size = (s32)((fid->size - 1) >> p_fs->cluster_size_bits) + 1; clu_to_free.flags = fid->flags; if (!is_dir_empty(sb, &clu_to_free)) { - ret = FFS_FILEEXIST; + ret = -ENOTEMPTY; goto out; } @@ -2172,23 +2044,23 @@ static int ffsRemoveDir(struct inode *inode, struct file_id_t *fid) remove_file(inode, &dir, dentry); /* (2) free the clusters */ - p_fs->fs_func->free_cluster(sb, &clu_to_free, 1); + exfat_free_cluster(sb, &clu_to_free, 1); fid->size = 0; fid->start_clu = CLUSTER_32(~0); fid->flags = (p_fs->vol_type == EXFAT) ? 0x03 : 0x01; -#ifdef CONFIG_EXFAT_DELAYED_SYNC - fs_sync(sb, false); +#ifndef CONFIG_STAGING_EXFAT_DELAYED_SYNC + fs_sync(sb, true); fs_set_vol_flags(sb, VOL_CLEAN); #endif if (p_fs->dev_ejected) - ret = FFS_MEDIAERR; + ret = -EIO; out: /* release the lock for file system critical section */ - up(&p_fs->v_sem); + mutex_unlock(&p_fs->v_mutex); return ret; } @@ -2202,7 +2074,7 @@ static int exfat_readdir(struct file *filp, struct dir_context *ctx) struct inode *inode = file_inode(filp); struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct fs_info_t *p_fs = &(sbi->fs_info); + struct fs_info_t *p_fs = &sbi->fs_info; struct bd_info_t *p_bd = &(EXFAT_SB(sb)->bd_info); struct dir_entry_t de; unsigned long inum; @@ -2244,12 +2116,11 @@ static int exfat_readdir(struct file *filp, struct dir_context *ctx) /* at least we tried to read a sector * move cpos to next sector position (should be aligned) */ - if (err == FFS_MEDIAERR) { + if (err == -EIO) { cpos += 1 << p_bd->sector_size_bits; cpos &= ~((1 << p_bd->sector_size_bits) - 1); } - err = -EIO; goto end_of_dir; } @@ -2293,7 +2164,7 @@ static int exfat_ioctl_volume_id(struct inode *dir) { struct super_block *sb = dir->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct fs_info_t *p_fs = &(sbi->fs_info); + struct fs_info_t *p_fs = &sbi->fs_info; return p_fs->vol_id; } @@ -2301,15 +2172,15 @@ static int exfat_ioctl_volume_id(struct inode *dir) static long exfat_generic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { -struct inode *inode = filp->f_path.dentry->d_inode; -#ifdef CONFIG_EXFAT_KERNEL_DEBUG + struct inode *inode = filp->f_path.dentry->d_inode; +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG unsigned int flags; -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ switch (cmd) { case EXFAT_IOCTL_GET_VOLUME_ID: return exfat_ioctl_volume_id(inode); -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG case EXFAT_IOC_GET_DEBUGFLAGS: { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -2333,7 +2204,7 @@ struct inode *inode = filp->f_path.dentry->d_inode; return 0; } -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ default: return -ENOTTY; /* Inappropriate ioctl for device */ } @@ -2351,6 +2222,7 @@ static int exfat_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) { struct super_block *sb = dir->i_sb; + struct timespec64 curtime; struct inode *inode; struct file_id_t fid; loff_t i_pos; @@ -2361,21 +2233,14 @@ static int exfat_create(struct inode *dir, struct dentry *dentry, umode_t mode, pr_debug("%s entered\n", __func__); err = ffsCreateFile(dir, (u8 *)dentry->d_name.name, FM_REGULAR, &fid); - if (err) { - if (err == FFS_INVALIDPATH) - err = -EINVAL; - else if (err == FFS_FILEEXIST) - err = -EEXIST; - else if (err == FFS_FULL) - err = -ENOSPC; - else if (err == FFS_NAMETOOLONG) - err = -ENAMETOOLONG; - else - err = -EIO; + if (err) goto out; - } + INC_IVERSION(dir); - dir->i_ctime = dir->i_mtime = dir->i_atime = current_time(dir); + curtime = current_time(dir); + dir->i_ctime = curtime; + dir->i_mtime = curtime; + dir->i_atime = curtime; if (IS_DIRSYNC(dir)) (void)exfat_sync_inode(dir); else @@ -2389,7 +2254,10 @@ static int exfat_create(struct inode *dir, struct dentry *dentry, umode_t mode, goto out; } INC_IVERSION(inode); - inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; + inode->i_ctime = curtime; /* * timestamp is already written, so mark_inode_dirty() is unnecessary. */ @@ -2522,6 +2390,7 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry) { struct inode *inode = dentry->d_inode; struct super_block *sb = dir->i_sb; + struct timespec64 curtime; int err; __lock_super(sb); @@ -2531,22 +2400,22 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry) EXFAT_I(inode)->fid.size = i_size_read(inode); err = ffsRemoveFile(dir, &(EXFAT_I(inode)->fid)); - if (err) { - if (err == FFS_PERMISSIONERR) - err = -EPERM; - else - err = -EIO; + if (err) goto out; - } + INC_IVERSION(dir); - dir->i_mtime = dir->i_atime = current_time(dir); + curtime = current_time(dir); + dir->i_mtime = curtime; + dir->i_atime = curtime; if (IS_DIRSYNC(dir)) (void)exfat_sync_inode(dir); else mark_inode_dirty(dir); clear_nlink(inode); - inode->i_mtime = inode->i_atime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; exfat_detach(inode); remove_inode_hash(inode); @@ -2560,6 +2429,7 @@ static int exfat_symlink(struct inode *dir, struct dentry *dentry, const char *target) { struct super_block *sb = dir->i_sb; + struct timespec64 curtime; struct inode *inode; struct file_id_t fid; loff_t i_pos; @@ -2572,32 +2442,22 @@ static int exfat_symlink(struct inode *dir, struct dentry *dentry, pr_debug("%s entered\n", __func__); err = ffsCreateFile(dir, (u8 *)dentry->d_name.name, FM_SYMLINK, &fid); - if (err) { - if (err == FFS_INVALIDPATH) - err = -EINVAL; - else if (err == FFS_FILEEXIST) - err = -EEXIST; - else if (err == FFS_FULL) - err = -ENOSPC; - else - err = -EIO; + if (err) goto out; - } + err = ffsWriteFile(dir, &fid, (char *)target, len, &ret); if (err) { ffsRemoveFile(dir, &fid); - - if (err == FFS_FULL) - err = -ENOSPC; - else - err = -EIO; goto out; } INC_IVERSION(dir); - dir->i_ctime = dir->i_mtime = dir->i_atime = current_time(dir); + curtime = current_time(dir); + dir->i_ctime = curtime; + dir->i_mtime = curtime; + dir->i_atime = curtime; if (IS_DIRSYNC(dir)) (void)exfat_sync_inode(dir); else @@ -2611,7 +2471,10 @@ static int exfat_symlink(struct inode *dir, struct dentry *dentry, goto out; } INC_IVERSION(inode); - inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; + inode->i_ctime = curtime; /* timestamp is already written, so mark_inode_dirty() is unneeded. */ EXFAT_I(inode)->target = kmemdup(target, len + 1, GFP_KERNEL); @@ -2632,6 +2495,7 @@ static int exfat_symlink(struct inode *dir, struct dentry *dentry, static int exfat_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { struct super_block *sb = dir->i_sb; + struct timespec64 curtime; struct inode *inode; struct file_id_t fid; loff_t i_pos; @@ -2642,21 +2506,14 @@ static int exfat_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) pr_debug("%s entered\n", __func__); err = ffsCreateDir(dir, (u8 *)dentry->d_name.name, &fid); - if (err) { - if (err == FFS_INVALIDPATH) - err = -EINVAL; - else if (err == FFS_FILEEXIST) - err = -EEXIST; - else if (err == FFS_FULL) - err = -ENOSPC; - else if (err == FFS_NAMETOOLONG) - err = -ENAMETOOLONG; - else - err = -EIO; + if (err) goto out; - } + INC_IVERSION(dir); - dir->i_ctime = dir->i_mtime = dir->i_atime = current_time(dir); + curtime = current_time(dir); + dir->i_ctime = curtime; + dir->i_mtime = curtime; + dir->i_atime = curtime; if (IS_DIRSYNC(dir)) (void)exfat_sync_inode(dir); else @@ -2671,7 +2528,10 @@ static int exfat_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) goto out; } INC_IVERSION(inode); - inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; + inode->i_ctime = curtime; /* timestamp is already written, so mark_inode_dirty() is unneeded. */ dentry->d_time = GET_IVERSION(dentry->d_parent->d_inode); @@ -2687,6 +2547,7 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) { struct inode *inode = dentry->d_inode; struct super_block *sb = dir->i_sb; + struct timespec64 curtime; int err; __lock_super(sb); @@ -2696,21 +2557,13 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) EXFAT_I(inode)->fid.size = i_size_read(inode); err = ffsRemoveDir(dir, &(EXFAT_I(inode)->fid)); - if (err) { - if (err == FFS_INVALIDPATH) - err = -EINVAL; - else if (err == FFS_FILEEXIST) - err = -ENOTEMPTY; - else if (err == FFS_NOTFOUND) - err = -ENOENT; - else if (err == FFS_DIRBUSY) - err = -EBUSY; - else - err = -EIO; + if (err) goto out; - } + INC_IVERSION(dir); - dir->i_mtime = dir->i_atime = current_time(dir); + curtime = current_time(dir); + dir->i_mtime = curtime; + dir->i_atime = curtime; if (IS_DIRSYNC(dir)) (void)exfat_sync_inode(dir); else @@ -2718,7 +2571,9 @@ static int exfat_rmdir(struct inode *dir, struct dentry *dentry) drop_nlink(dir); clear_nlink(inode); - inode->i_mtime = inode->i_atime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; exfat_detach(inode); remove_inode_hash(inode); @@ -2734,6 +2589,7 @@ static int exfat_rename(struct inode *old_dir, struct dentry *old_dentry, { struct inode *old_inode, *new_inode; struct super_block *sb = old_dir->i_sb; + struct timespec64 curtime; loff_t i_pos; int err; @@ -2751,24 +2607,15 @@ static int exfat_rename(struct inode *old_dir, struct dentry *old_dentry, err = ffsMoveFile(old_dir, &(EXFAT_I(old_inode)->fid), new_dir, new_dentry); - if (err) { - if (err == FFS_PERMISSIONERR) - err = -EPERM; - else if (err == FFS_INVALIDPATH) - err = -EINVAL; - else if (err == FFS_FILEEXIST) - err = -EEXIST; - else if (err == FFS_NOTFOUND) - err = -ENOENT; - else if (err == FFS_FULL) - err = -ENOSPC; - else - err = -EIO; + if (err) goto out; - } + INC_IVERSION(new_dir); - new_dir->i_ctime = new_dir->i_mtime = new_dir->i_atime = - current_time(new_dir); + curtime = current_time(new_dir); + new_dir->i_ctime = curtime; + new_dir->i_mtime = curtime; + new_dir->i_atime = curtime; + if (IS_DIRSYNC(new_dir)) (void)exfat_sync_inode(new_dir); else @@ -2790,7 +2637,9 @@ static int exfat_rename(struct inode *old_dir, struct dentry *old_dentry, inc_nlink(new_dir); } INC_IVERSION(old_dir); - old_dir->i_ctime = old_dir->i_mtime = current_time(old_dir); + curtime = current_time(old_dir); + old_dir->i_ctime = curtime; + old_dir->i_mtime = curtime; if (IS_DIRSYNC(old_dir)) (void)exfat_sync_inode(old_dir); else @@ -2814,13 +2663,16 @@ static int exfat_cont_expand(struct inode *inode, loff_t size) { struct address_space *mapping = inode->i_mapping; loff_t start = i_size_read(inode), count = size - i_size_read(inode); + struct timespec64 curtime; int err, err2; err = generic_cont_expand_simple(inode, size); if (err != 0) return err; - inode->i_ctime = inode->i_mtime = current_time(inode); + curtime = current_time(inode); + inode->i_ctime = curtime; + inode->i_mtime = curtime; mark_inode_dirty(inode); if (IS_SYNC(inode)) { @@ -2895,7 +2747,8 @@ static void exfat_truncate(struct inode *inode, loff_t old_size) { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct fs_info_t *p_fs = &(sbi->fs_info); + struct fs_info_t *p_fs = &sbi->fs_info; + struct timespec64 curtime; int err; __lock_super(sb); @@ -2914,7 +2767,9 @@ static void exfat_truncate(struct inode *inode, loff_t old_size) if (err) goto out; - inode->i_ctime = inode->i_mtime = current_time(inode); + curtime = current_time(inode); + inode->i_ctime = curtime; + inode->i_mtime = curtime; if (IS_DIRSYNC(inode)) (void)exfat_sync_inode(inode); else @@ -2936,8 +2791,8 @@ static int exfat_setattr(struct dentry *dentry, struct iattr *attr) pr_debug("%s entered\n", __func__); - if ((attr->ia_valid & ATTR_SIZE) - && (attr->ia_size > i_size_read(inode))) { + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size > i_size_read(inode)) { error = exfat_cont_expand(inode, attr->ia_size); if (error || attr->ia_valid == ATTR_SIZE) return error; @@ -2946,8 +2801,8 @@ static int exfat_setattr(struct dentry *dentry, struct iattr *attr) ia_valid = attr->ia_valid; - if ((ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) - && exfat_allow_set_time(sbi, inode)) { + if ((ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) && + exfat_allow_set_time(sbi, inode)) { attr->ia_valid &= ~(ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET); @@ -3073,8 +2928,7 @@ static int exfat_bmap(struct inode *inode, sector_t sector, sector_t *phys, { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct fs_info_t *p_fs = &(sbi->fs_info); - struct bd_info_t *p_bd = &(sbi->bd_info); + struct fs_info_t *p_fs = &sbi->fs_info; const unsigned long blocksize = sb->s_blocksize; const unsigned char blocksize_bits = sb->s_blocksize_bits; sector_t last_block; @@ -3084,18 +2938,6 @@ static int exfat_bmap(struct inode *inode, sector_t sector, sector_t *phys, *phys = 0; *mapped_blocks = 0; - if ((p_fs->vol_type == FAT12) || (p_fs->vol_type == FAT16)) { - if (inode->i_ino == EXFAT_ROOT_INO) { - if (sector < - (p_fs->dentries_in_root >> - (p_bd->sector_size_bits - DENTRY_SIZE_BITS))) { - *phys = sector + p_fs->root_start_sector; - *mapped_blocks = 1; - } - return 0; - } - } - last_block = (i_size_read(inode) + (blocksize - 1)) >> blocksize_bits; if (sector >= last_block) { if (*create == 0) @@ -3114,12 +2956,7 @@ static int exfat_bmap(struct inode *inode, sector_t sector, sector_t *phys, err = ffsMapCluster(inode, clu_offset, &cluster); - if (err) { - if (err == FFS_FULL) - return -ENOSPC; - else - return -EIO; - } else if (cluster != CLUSTER_32(~0)) { + if (!err && (cluster != CLUSTER_32(~0))) { *phys = START_SECTOR(cluster) + sec_offset; *mapped_blocks = p_fs->sectors_per_clu - sec_offset; } @@ -3215,6 +3052,7 @@ static int exfat_write_end(struct file *file, struct address_space *mapping, { struct inode *inode = mapping->host; struct file_id_t *fid = &(EXFAT_I(inode)->fid); + struct timespec64 curtime; int err; err = generic_write_end(file, mapping, pos, len, copied, pagep, fsdata); @@ -3223,7 +3061,9 @@ static int exfat_write_end(struct file *file, struct address_space *mapping, exfat_write_failed(mapping, pos + len); if (!(err < 0) && !(fid->attr & ATTR_ARCHIVE)) { - inode->i_mtime = inode->i_ctime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_ctime = curtime; fid->attr |= ATTR_ARCHIVE; mark_inode_dirty(inode); } @@ -3302,7 +3142,7 @@ static struct inode *exfat_iget(struct super_block *sb, loff_t i_pos) static int exfat_fill_inode(struct inode *inode, struct file_id_t *fid) { struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); - struct fs_info_t *p_fs = &(sbi->fs_info); + struct fs_info_t *p_fs = &sbi->fs_info; struct dir_entry_t info; memcpy(&(EXFAT_I(inode)->fid), fid, sizeof(struct file_id_t)); @@ -3314,7 +3154,7 @@ static int exfat_fill_inode(struct inode *inode, struct file_id_t *fid) inode->i_uid = sbi->options.fs_uid; inode->i_gid = sbi->options.fs_gid; INC_IVERSION(inode); - inode->i_generation = get_seconds(); + inode->i_generation = prandom_u32(); if (info.Attr & ATTR_SUBDIR) { /* directory */ inode->i_generation &= ~1; @@ -3501,7 +3341,7 @@ static int exfat_statfs(struct dentry *dentry, struct kstatfs *buf) struct vol_info_t info; if (p_fs->used_clusters == UINT_MAX) { - if (ffsGetVolInfo(sb, &info) == FFS_MEDIAERR) + if (ffsGetVolInfo(sb, &info) == -EIO) return -EIO; } else { @@ -3557,7 +3397,7 @@ static int exfat_show_options(struct seq_file *m, struct dentry *root) seq_puts(m, ",errors=panic"); else seq_puts(m, ",errors=remount-ro"); -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD if (opts->discard) seq_puts(m, ",discard"); #endif @@ -3638,7 +3478,7 @@ enum { Opt_err_ro, Opt_utf8_hack, Opt_err, -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD Opt_discard, #endif /* EXFAT_CONFIG_DISCARD */ }; @@ -3658,9 +3498,9 @@ static const match_table_t exfat_tokens = { {Opt_err_panic, "errors=panic"}, {Opt_err_ro, "errors=remount-ro"}, {Opt_utf8_hack, "utf8"}, -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD {Opt_discard, "discard"}, -#endif /* CONFIG_EXFAT_DISCARD */ +#endif /* CONFIG_STAGING_EXFAT_DISCARD */ {Opt_err, NULL} }; @@ -3674,13 +3514,14 @@ static int parse_options(char *options, int silent, int *debug, opts->fs_uid = current_uid(); opts->fs_gid = current_gid(); - opts->fs_fmask = opts->fs_dmask = current->fs->umask; + opts->fs_fmask = current->fs->umask; + opts->fs_dmask = current->fs->umask; opts->allow_utime = U16_MAX; opts->codepage = exfat_default_codepage; opts->iocharset = exfat_default_iocharset; opts->casesensitive = 0; opts->errors = EXFAT_ERRORS_RO; -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD opts->discard = 0; #endif *debug = 0; @@ -3751,11 +3592,11 @@ static int parse_options(char *options, int silent, int *debug, case Opt_debug: *debug = 1; break; -#ifdef CONFIG_EXFAT_DISCARD +#ifdef CONFIG_STAGING_EXFAT_DISCARD case Opt_discard: opts->discard = 1; break; -#endif /* CONFIG_EXFAT_DISCARD */ +#endif /* CONFIG_STAGING_EXFAT_DISCARD */ case Opt_utf8_hack: break; default: @@ -3787,7 +3628,8 @@ static int exfat_read_root(struct inode *inode) { struct super_block *sb = inode->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); - struct fs_info_t *p_fs = &(sbi->fs_info); + struct fs_info_t *p_fs = &sbi->fs_info; + struct timespec64 curtime; struct dir_entry_t info; EXFAT_I(inode)->fid.dir.dir = p_fs->root_dir; @@ -3818,7 +3660,10 @@ static int exfat_read_root(struct inode *inode) EXFAT_I(inode)->mmu_private = i_size_read(inode); exfat_save_attr(inode, ATTR_SUBDIR); - inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); + curtime = current_time(inode); + inode->i_mtime = curtime; + inode->i_atime = curtime; + inode->i_ctime = curtime; set_nlink(inode, info.NumSubdirs + 2); return 0; @@ -3838,7 +3683,6 @@ static int exfat_fill_super(struct super_block *sb, void *data, int silent) struct exfat_sb_info *sbi; int debug, ret; long error; - char buf[50]; /* * GFP_KERNEL is ok here, because while we do hold the @@ -3885,17 +3729,6 @@ static int exfat_fill_super(struct super_block *sb, void *data, int silent) * if (FAT_FIRST_ENT(sb, media) != first) */ - /* codepage is not meaningful in exfat */ - if (sbi->fs_info.vol_type != EXFAT) { - error = -EINVAL; - sprintf(buf, "cp%d", sbi->options.codepage); - sbi->nls_disk = load_nls(buf); - if (!sbi->nls_disk) { - pr_err("[EXFAT] Codepage %s not found\n", buf); - goto out_fail2; - } - } - sbi->nls_io = load_nls(sbi->options.iocharset); error = -ENOMEM; @@ -3967,7 +3800,7 @@ static void __exit exfat_destroy_inodecache(void) kmem_cache_destroy(exfat_inode_cachep); } -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG static void exfat_debug_kill_sb(struct super_block *sb) { struct exfat_sb_info *sbi = EXFAT_SB(sb); @@ -3984,10 +3817,10 @@ static void exfat_debug_kill_sb(struct super_block *sb) * invalidate_bdev drops all device cache include * dirty. We use this to simulate device removal. */ - down(&p_fs->v_sem); - FAT_release_all(sb); - buf_release_all(sb); - up(&p_fs->v_sem); + mutex_lock(&p_fs->v_mutex); + exfat_fat_release_all(sb); + exfat_buf_release_all(sb); + mutex_unlock(&p_fs->v_mutex); invalidate_bdev(bdev); } @@ -3995,17 +3828,17 @@ static void exfat_debug_kill_sb(struct super_block *sb) kill_block_super(sb); } -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ static struct file_system_type exfat_fs_type = { .owner = THIS_MODULE, .name = "exfat", .mount = exfat_fs_mount, -#ifdef CONFIG_EXFAT_KERNEL_DEBUG +#ifdef CONFIG_STAGING_EXFAT_KERNEL_DEBUG .kill_sb = exfat_debug_kill_sb, #else .kill_sb = kill_block_super, -#endif /* CONFIG_EXFAT_KERNEL_DEBUG */ +#endif /* CONFIG_STAGING_EXFAT_KERNEL_DEBUG */ .fs_flags = FS_REQUIRES_DEV, }; From 4001f03afe6cea71af225678f8cbd6ace6fb6bf7 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:05:15 -0600 Subject: [PATCH 008/135] backports: typec: from: linux.git Reference: v5.5.19 Signed-off-by: Robert Nelson --- drivers/usb/typec/Kconfig | 11 + drivers/usb/typec/Makefile | 1 + drivers/usb/typec/bus.c | 5 +- drivers/usb/typec/class.c | 42 +- drivers/usb/typec/hd3ss3220.c | 269 +++++++++++ drivers/usb/typec/tcpm/tcpci_rt1711h.c | 31 +- drivers/usb/typec/tcpm/tcpm.c | 161 +------ drivers/usb/typec/tps6598x.c | 49 +- drivers/usb/typec/ucsi/displayport.c | 40 +- drivers/usb/typec/ucsi/trace.c | 11 - drivers/usb/typec/ucsi/trace.h | 79 +--- drivers/usb/typec/ucsi/ucsi.c | 631 +++++++++++++------------ drivers/usb/typec/ucsi/ucsi.h | 417 ++++++---------- drivers/usb/typec/ucsi/ucsi_acpi.c | 95 +++- drivers/usb/typec/ucsi/ucsi_ccg.c | 168 ++++--- include/linux/usb/typec.h | 41 +- 16 files changed, 1062 insertions(+), 989 deletions(-) create mode 100644 drivers/usb/typec/hd3ss3220.c diff --git a/drivers/usb/typec/Kconfig b/drivers/usb/typec/Kconfig index 895e2418de53b..b4f2aac7ae8a9 100644 --- a/drivers/usb/typec/Kconfig +++ b/drivers/usb/typec/Kconfig @@ -50,6 +50,17 @@ source "drivers/usb/typec/tcpm/Kconfig" source "drivers/usb/typec/ucsi/Kconfig" +config TYPEC_HD3SS3220 + tristate "TI HD3SS3220 Type-C DRP Port controller driver" + depends on I2C + depends on USB_ROLE_SWITCH + help + Say Y or M here if your system has TI HD3SS3220 Type-C DRP Port + controller driver. + + If you choose to build this driver as a dynamically linked module, the + module will be called hd3ss3220.ko. + config TYPEC_TPS6598X tristate "TI TPS6598x USB Power Delivery controller driver" depends on I2C diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 6696b7263d61a..7753a5c3cd462 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -4,5 +4,6 @@ typec-y := class.o mux.o bus.o obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ +obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o obj-$(CONFIG_TYPEC_TPS6598X) += tps6598x.o obj-$(CONFIG_TYPEC) += mux/ diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c index c950171556d8c..74cb3c2ecb347 100644 --- a/drivers/usb/typec/bus.c +++ b/drivers/usb/typec/bus.c @@ -192,10 +192,7 @@ EXPORT_SYMBOL_GPL(typec_altmode_vdm); const struct typec_altmode * typec_altmode_get_partner(struct typec_altmode *adev) { - if (!adev || !to_altmode(adev)->partner) - return NULL; - - return &to_altmode(adev)->partner->adev; + return adev ? &to_altmode(adev)->partner->adev : NULL; } EXPORT_SYMBOL_GPL(typec_altmode_get_partner); diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index a400b65cf17bb..91d62276b56fe 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -53,6 +53,7 @@ struct typec_port { struct typec_mux *mux; const struct typec_capability *cap; + const struct typec_operations *ops; }; #define to_typec_port(_dev_) container_of(_dev_, struct typec_port, dev) @@ -955,7 +956,7 @@ preferred_role_store(struct device *dev, struct device_attribute *attr, return -EOPNOTSUPP; } - if (!port->cap->try_role) { + if (!port->ops || !port->ops->try_role) { dev_dbg(dev, "Setting preferred role not supported\n"); return -EOPNOTSUPP; } @@ -968,7 +969,7 @@ preferred_role_store(struct device *dev, struct device_attribute *attr, return -EINVAL; } - ret = port->cap->try_role(port->cap, role); + ret = port->ops->try_role(port, role); if (ret) return ret; @@ -999,7 +1000,7 @@ static ssize_t data_role_store(struct device *dev, struct typec_port *port = to_typec_port(dev); int ret; - if (!port->cap->dr_set) { + if (!port->ops || !port->ops->dr_set) { dev_dbg(dev, "data role swapping not supported\n"); return -EOPNOTSUPP; } @@ -1014,7 +1015,7 @@ static ssize_t data_role_store(struct device *dev, goto unlock_and_ret; } - ret = port->cap->dr_set(port->cap, ret); + ret = port->ops->dr_set(port, ret); if (ret) goto unlock_and_ret; @@ -1049,7 +1050,7 @@ static ssize_t power_role_store(struct device *dev, return -EOPNOTSUPP; } - if (!port->cap->pr_set) { + if (!port->ops || !port->ops->pr_set) { dev_dbg(dev, "power role swapping not supported\n"); return -EOPNOTSUPP; } @@ -1071,7 +1072,7 @@ static ssize_t power_role_store(struct device *dev, goto unlock_and_ret; } - ret = port->cap->pr_set(port->cap, ret); + ret = port->ops->pr_set(port, ret); if (ret) goto unlock_and_ret; @@ -1102,7 +1103,8 @@ port_type_store(struct device *dev, struct device_attribute *attr, int ret; enum typec_port_type type; - if (!port->cap->port_type_set || port->cap->type != TYPEC_PORT_DRP) { + if (port->cap->type != TYPEC_PORT_DRP || + !port->ops || !port->ops->port_type_set) { dev_dbg(dev, "changing port type not supported\n"); return -EOPNOTSUPP; } @@ -1119,7 +1121,7 @@ port_type_store(struct device *dev, struct device_attribute *attr, goto unlock_and_ret; } - ret = port->cap->port_type_set(port->cap, type); + ret = port->ops->port_type_set(port, type); if (ret) goto unlock_and_ret; @@ -1175,7 +1177,7 @@ static ssize_t vconn_source_store(struct device *dev, return -EOPNOTSUPP; } - if (!port->cap->vconn_set) { + if (!port->ops || !port->ops->vconn_set) { dev_dbg(dev, "VCONN swapping not supported\n"); return -EOPNOTSUPP; } @@ -1184,7 +1186,7 @@ static ssize_t vconn_source_store(struct device *dev, if (ret) return ret; - ret = port->cap->vconn_set(port->cap, (enum typec_role)source); + ret = port->ops->vconn_set(port, (enum typec_role)source); if (ret) return ret; @@ -1278,6 +1280,7 @@ static void typec_release(struct device *dev) ida_destroy(&port->mode_ids); typec_switch_put(port->sw); typec_mux_put(port->mux); + kfree(port->cap); kfree(port); } @@ -1486,6 +1489,16 @@ EXPORT_SYMBOL_GPL(typec_set_mode); /* --------------------------------------- */ +/** + * typec_get_drvdata - Return private driver data pointer + * @port: USB Type-C port + */ +void *typec_get_drvdata(struct typec_port *port) +{ + return dev_get_drvdata(&port->dev); +} +EXPORT_SYMBOL_GPL(typec_get_drvdata); + /** * typec_port_register_altmode - Register USB Type-C Port Alternate Mode * @port: USB Type-C Port that supports the alternate mode @@ -1579,7 +1592,7 @@ struct typec_port *typec_register_port(struct device *parent, mutex_init(&port->port_type_lock); port->id = id; - port->cap = cap; + port->ops = cap->ops; port->port_type = cap->type; port->prefer_role = cap->prefer_role; @@ -1589,6 +1602,13 @@ struct typec_port *typec_register_port(struct device *parent, port->dev.fwnode = cap->fwnode; port->dev.type = &typec_port_dev_type; dev_set_name(&port->dev, "port%d", id); + dev_set_drvdata(&port->dev, cap->driver_data); + + port->cap = kmemdup(cap, sizeof(*cap), GFP_KERNEL); + if (!port->cap) { + put_device(&port->dev); + return ERR_PTR(-ENOMEM); + } port->sw = typec_switch_get(&port->dev); if (IS_ERR(port->sw)) { diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c new file mode 100644 index 0000000000000..323dfa8160ab0 --- /dev/null +++ b/drivers/usb/typec/hd3ss3220.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * TI HD3SS3220 Type-C DRP Port Controller Driver + * + * Copyright (C) 2019 Renesas Electronics Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HD3SS3220_REG_CN_STAT_CTRL 0x09 +#define HD3SS3220_REG_GEN_CTRL 0x0A +#define HD3SS3220_REG_DEV_REV 0xA0 + +/* Register HD3SS3220_REG_CN_STAT_CTRL*/ +#define HD3SS3220_REG_CN_STAT_CTRL_ATTACHED_STATE_MASK (BIT(7) | BIT(6)) +#define HD3SS3220_REG_CN_STAT_CTRL_AS_DFP BIT(6) +#define HD3SS3220_REG_CN_STAT_CTRL_AS_UFP BIT(7) +#define HD3SS3220_REG_CN_STAT_CTRL_TO_ACCESSORY (BIT(7) | BIT(6)) +#define HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS BIT(4) + +/* Register HD3SS3220_REG_GEN_CTRL*/ +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_MASK (BIT(2) | BIT(1)) +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT 0x00 +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SNK BIT(1) +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SRC (BIT(2) | BIT(1)) + +struct hd3ss3220 { + struct device *dev; + struct regmap *regmap; + struct usb_role_switch *role_sw; + struct typec_port *port; +}; + +static int hd3ss3220_set_source_pref(struct hd3ss3220 *hd3ss3220, int src_pref) +{ + return regmap_update_bits(hd3ss3220->regmap, HD3SS3220_REG_GEN_CTRL, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_MASK, + src_pref); +} + +static enum usb_role hd3ss3220_get_attached_state(struct hd3ss3220 *hd3ss3220) +{ + unsigned int reg_val; + enum usb_role attached_state; + int ret; + + ret = regmap_read(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, + ®_val); + if (ret < 0) + return ret; + + switch (reg_val & HD3SS3220_REG_CN_STAT_CTRL_ATTACHED_STATE_MASK) { + case HD3SS3220_REG_CN_STAT_CTRL_AS_DFP: + attached_state = USB_ROLE_HOST; + break; + case HD3SS3220_REG_CN_STAT_CTRL_AS_UFP: + attached_state = USB_ROLE_DEVICE; + break; + default: + attached_state = USB_ROLE_NONE; + break; + } + + return attached_state; +} + +static int hd3ss3220_dr_set(struct typec_port *port, enum typec_data_role role) +{ + struct hd3ss3220 *hd3ss3220 = typec_get_drvdata(port); + enum usb_role role_val; + int pref, ret = 0; + + if (role == TYPEC_HOST) { + role_val = USB_ROLE_HOST; + pref = HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SRC; + } else { + role_val = USB_ROLE_DEVICE; + pref = HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SNK; + } + + ret = hd3ss3220_set_source_pref(hd3ss3220, pref); + usleep_range(10, 100); + + usb_role_switch_set_role(hd3ss3220->role_sw, role_val); + typec_set_data_role(hd3ss3220->port, role); + + return ret; +} + +static const struct typec_operations hd3ss3220_ops = { + .dr_set = hd3ss3220_dr_set +}; + +static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) +{ + enum usb_role role_state = hd3ss3220_get_attached_state(hd3ss3220); + + usb_role_switch_set_role(hd3ss3220->role_sw, role_state); + if (role_state == USB_ROLE_NONE) + hd3ss3220_set_source_pref(hd3ss3220, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT); + + switch (role_state) { + case USB_ROLE_HOST: + typec_set_data_role(hd3ss3220->port, TYPEC_HOST); + break; + case USB_ROLE_DEVICE: + typec_set_data_role(hd3ss3220->port, TYPEC_DEVICE); + break; + default: + break; + } +} + +static irqreturn_t hd3ss3220_irq(struct hd3ss3220 *hd3ss3220) +{ + int err; + + hd3ss3220_set_role(hd3ss3220); + err = regmap_update_bits_base(hd3ss3220->regmap, + HD3SS3220_REG_CN_STAT_CTRL, + HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS, + HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS, + NULL, false, true); + if (err < 0) + return IRQ_NONE; + + return IRQ_HANDLED; +} + +static irqreturn_t hd3ss3220_irq_handler(int irq, void *data) +{ + struct i2c_client *client = to_i2c_client(data); + struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client); + + return hd3ss3220_irq(hd3ss3220); +} + +static const struct regmap_config config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = 0x0A, +}; + +static int hd3ss3220_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct typec_capability typec_cap = { }; + struct hd3ss3220 *hd3ss3220; + struct fwnode_handle *connector; + int ret; + unsigned int data; + + hd3ss3220 = devm_kzalloc(&client->dev, sizeof(struct hd3ss3220), + GFP_KERNEL); + if (!hd3ss3220) + return -ENOMEM; + + i2c_set_clientdata(client, hd3ss3220); + + hd3ss3220->dev = &client->dev; + hd3ss3220->regmap = devm_regmap_init_i2c(client, &config); + if (IS_ERR(hd3ss3220->regmap)) + return PTR_ERR(hd3ss3220->regmap); + + hd3ss3220_set_source_pref(hd3ss3220, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT); + connector = device_get_named_child_node(hd3ss3220->dev, "connector"); + if (!connector) + return -ENODEV; + + hd3ss3220->role_sw = fwnode_usb_role_switch_get(connector); + if (IS_ERR(hd3ss3220->role_sw)) { + ret = PTR_ERR(hd3ss3220->role_sw); + goto err_put_fwnode; + } + + typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; + typec_cap.driver_data = hd3ss3220; + typec_cap.type = TYPEC_PORT_DRP; + typec_cap.data = TYPEC_PORT_DRD; + typec_cap.ops = &hd3ss3220_ops; + typec_cap.fwnode = connector; + + hd3ss3220->port = typec_register_port(&client->dev, &typec_cap); + if (IS_ERR(hd3ss3220->port)) { + ret = PTR_ERR(hd3ss3220->port); + goto err_put_role; + } + + hd3ss3220_set_role(hd3ss3220); + ret = regmap_read(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, &data); + if (ret < 0) + goto err_unreg_port; + + if (data & HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS) { + ret = regmap_write(hd3ss3220->regmap, + HD3SS3220_REG_CN_STAT_CTRL, + data | HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS); + if (ret < 0) + goto err_unreg_port; + } + + if (client->irq > 0) { + ret = devm_request_threaded_irq(&client->dev, client->irq, NULL, + hd3ss3220_irq_handler, + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + "hd3ss3220", &client->dev); + if (ret) + goto err_unreg_port; + } + + ret = i2c_smbus_read_byte_data(client, HD3SS3220_REG_DEV_REV); + if (ret < 0) + goto err_unreg_port; + + fwnode_handle_put(connector); + + dev_info(&client->dev, "probed revision=0x%x\n", ret); + + return 0; +err_unreg_port: + typec_unregister_port(hd3ss3220->port); +err_put_role: + usb_role_switch_put(hd3ss3220->role_sw); +err_put_fwnode: + fwnode_handle_put(connector); + + return ret; +} + +static int hd3ss3220_remove(struct i2c_client *client) +{ + struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client); + + typec_unregister_port(hd3ss3220->port); + usb_role_switch_put(hd3ss3220->role_sw); + + return 0; +} + +static const struct of_device_id dev_ids[] = { + { .compatible = "ti,hd3ss3220"}, + {} +}; +MODULE_DEVICE_TABLE(of, dev_ids); + +static struct i2c_driver hd3ss3220_driver = { + .driver = { + .name = "hd3ss3220", + .of_match_table = of_match_ptr(dev_ids), + }, + .probe = hd3ss3220_probe, + .remove = hd3ss3220_remove, +}; + +module_i2c_driver(hd3ss3220_driver); + +MODULE_AUTHOR("Biju Das "); +MODULE_DESCRIPTION("TI HD3SS3220 DRP Port Controller Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c index b56a0880a0441..017389021b96a 100644 --- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c +++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c @@ -179,6 +179,26 @@ static irqreturn_t rt1711h_irq(int irq, void *dev_id) return tcpci_irq(chip->tcpci); } +static int rt1711h_init_alert(struct rt1711h_chip *chip, + struct i2c_client *client) +{ + int ret; + + /* Disable chip interrupts before requesting irq */ + ret = rt1711h_write16(chip, TCPC_ALERT_MASK, 0); + if (ret < 0) + return ret; + + ret = devm_request_threaded_irq(chip->dev, client->irq, NULL, + rt1711h_irq, + IRQF_ONESHOT | IRQF_TRIGGER_LOW, + dev_name(chip->dev), chip); + if (ret < 0) + return ret; + enable_irq_wake(client->irq); + return 0; +} + static int rt1711h_sw_reset(struct rt1711h_chip *chip) { int ret; @@ -240,8 +260,7 @@ static int rt1711h_probe(struct i2c_client *client, if (ret < 0) return ret; - /* Disable chip interrupts before requesting irq */ - ret = rt1711h_write16(chip, TCPC_ALERT_MASK, 0); + ret = rt1711h_init_alert(chip, client); if (ret < 0) return ret; @@ -252,14 +271,6 @@ static int rt1711h_probe(struct i2c_client *client, if (IS_ERR_OR_NULL(chip->tcpci)) return PTR_ERR(chip->tcpci); - ret = devm_request_threaded_irq(chip->dev, client->irq, NULL, - rt1711h_irq, - IRQF_ONESHOT | IRQF_TRIGGER_LOW, - dev_name(chip->dev), chip); - if (ret < 0) - return ret; - enable_irq_wake(client->irq); - return 0; } diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 355a2c7fac0b4..56fc356bc55c6 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -380,9 +380,6 @@ static enum tcpm_state tcpm_default_state(struct tcpm_port *port) return SNK_UNATTACHED; else if (port->try_role == TYPEC_SOURCE) return SRC_UNATTACHED; - else if (port->tcpc->config && - port->tcpc->config->default_role == TYPEC_SINK) - return SNK_UNATTACHED; /* Fall through to return SRC_UNATTACHED */ } else if (port->port_type == TYPEC_PORT_SNK) { return SNK_UNATTACHED; @@ -390,12 +387,6 @@ static enum tcpm_state tcpm_default_state(struct tcpm_port *port) return SRC_UNATTACHED; } -static inline -struct tcpm_port *typec_cap_to_tcpm(const struct typec_capability *cap) -{ - return container_of(cap, struct tcpm_port, typec_caps); -} - static bool tcpm_port_is_disconnected(struct tcpm_port *port) { return (!port->attached && port->cc1 == TYPEC_CC_OPEN && @@ -3768,14 +3759,6 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, */ break; - case PORT_RESET: - case PORT_RESET_WAIT_OFF: - /* - * State set back to default mode once the timer completes. - * Ignore CC changes here. - */ - break; - default: if (tcpm_port_is_disconnected(port)) tcpm_set_state(port, unattached_state(port), 0); @@ -3837,15 +3820,6 @@ static void _tcpm_pd_vbus_on(struct tcpm_port *port) case SRC_TRY_DEBOUNCE: /* Do nothing, waiting for sink detection */ break; - - case PORT_RESET: - case PORT_RESET_WAIT_OFF: - /* - * State set back to default mode once the timer completes. - * Ignore vbus changes here. - */ - break; - default: break; } @@ -3899,19 +3873,10 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port) case PORT_RESET_WAIT_OFF: tcpm_set_state(port, tcpm_default_state(port), 0); break; - case SRC_TRY_WAIT: case SRC_TRY_DEBOUNCE: /* Do nothing, waiting for sink detection */ break; - - case PORT_RESET: - /* - * State set back to default mode once the timer completes. - * Ignore vbus changes here. - */ - break; - default: if (port->pwr_role == TYPEC_SINK && port->attached) @@ -3996,10 +3961,9 @@ void tcpm_pd_hard_reset(struct tcpm_port *port) } EXPORT_SYMBOL_GPL(tcpm_pd_hard_reset); -static int tcpm_dr_set(const struct typec_capability *cap, - enum typec_data_role data) +static int tcpm_dr_set(struct typec_port *p, enum typec_data_role data) { - struct tcpm_port *port = typec_cap_to_tcpm(cap); + struct tcpm_port *port = typec_get_drvdata(p); int ret; mutex_lock(&port->swap_lock); @@ -4064,10 +4028,9 @@ static int tcpm_dr_set(const struct typec_capability *cap, return ret; } -static int tcpm_pr_set(const struct typec_capability *cap, - enum typec_role role) +static int tcpm_pr_set(struct typec_port *p, enum typec_role role) { - struct tcpm_port *port = typec_cap_to_tcpm(cap); + struct tcpm_port *port = typec_get_drvdata(p); int ret; mutex_lock(&port->swap_lock); @@ -4108,10 +4071,9 @@ static int tcpm_pr_set(const struct typec_capability *cap, return ret; } -static int tcpm_vconn_set(const struct typec_capability *cap, - enum typec_role role) +static int tcpm_vconn_set(struct typec_port *p, enum typec_role role) { - struct tcpm_port *port = typec_cap_to_tcpm(cap); + struct tcpm_port *port = typec_get_drvdata(p); int ret; mutex_lock(&port->swap_lock); @@ -4148,16 +4110,16 @@ static int tcpm_vconn_set(const struct typec_capability *cap, return ret; } -static int tcpm_try_role(const struct typec_capability *cap, int role) +static int tcpm_try_role(struct typec_port *p, int role) { - struct tcpm_port *port = typec_cap_to_tcpm(cap); + struct tcpm_port *port = typec_get_drvdata(p); struct tcpc_dev *tcpc = port->tcpc; int ret = 0; mutex_lock(&port->lock); if (tcpc->try_role) ret = tcpc->try_role(tcpc, role); - if (!ret && (!tcpc->config || !tcpc->config->try_role_hw)) + if (!ret) port->try_role = role; port->try_src_count = 0; port->try_snk_count = 0; @@ -4357,10 +4319,9 @@ static void tcpm_init(struct tcpm_port *port) tcpm_set_state(port, PORT_RESET, 0); } -static int tcpm_port_type_set(const struct typec_capability *cap, - enum typec_port_type type) +static int tcpm_port_type_set(struct typec_port *p, enum typec_port_type type) { - struct tcpm_port *port = typec_cap_to_tcpm(cap); + struct tcpm_port *port = typec_get_drvdata(p); mutex_lock(&port->lock); if (type == port->port_type) @@ -4385,6 +4346,14 @@ static int tcpm_port_type_set(const struct typec_capability *cap, return 0; } +static const struct typec_operations tcpm_ops = { + .try_role = tcpm_try_role, + .dr_set = tcpm_dr_set, + .pr_set = tcpm_pr_set, + .vconn_set = tcpm_vconn_set, + .port_type_set = tcpm_port_type_set +}; + void tcpm_tcpc_reset(struct tcpm_port *port) { mutex_lock(&port->lock); @@ -4394,34 +4363,6 @@ void tcpm_tcpc_reset(struct tcpm_port *port) } EXPORT_SYMBOL_GPL(tcpm_tcpc_reset); -static int tcpm_copy_pdos(u32 *dest_pdo, const u32 *src_pdo, - unsigned int nr_pdo) -{ - unsigned int i; - - if (nr_pdo > PDO_MAX_OBJECTS) - nr_pdo = PDO_MAX_OBJECTS; - - for (i = 0; i < nr_pdo; i++) - dest_pdo[i] = src_pdo[i]; - - return nr_pdo; -} - -static int tcpm_copy_vdos(u32 *dest_vdo, const u32 *src_vdo, - unsigned int nr_vdo) -{ - unsigned int i; - - if (nr_vdo > VDO_MAX_OBJECTS) - nr_vdo = VDO_MAX_OBJECTS; - - for (i = 0; i < nr_vdo; i++) - dest_vdo[i] = src_vdo[i]; - - return nr_vdo; -} - static int tcpm_fw_get_caps(struct tcpm_port *port, struct fwnode_handle *fwnode) { @@ -4724,35 +4665,10 @@ static int devm_tcpm_psy_register(struct tcpm_port *port) return PTR_ERR_OR_ZERO(port->psy); } -static int tcpm_copy_caps(struct tcpm_port *port, - const struct tcpc_config *tcfg) -{ - if (tcpm_validate_caps(port, tcfg->src_pdo, tcfg->nr_src_pdo) || - tcpm_validate_caps(port, tcfg->snk_pdo, tcfg->nr_snk_pdo)) - return -EINVAL; - - port->nr_src_pdo = tcpm_copy_pdos(port->src_pdo, tcfg->src_pdo, - tcfg->nr_src_pdo); - port->nr_snk_pdo = tcpm_copy_pdos(port->snk_pdo, tcfg->snk_pdo, - tcfg->nr_snk_pdo); - - port->nr_snk_vdo = tcpm_copy_vdos(port->snk_vdo, tcfg->snk_vdo, - tcfg->nr_snk_vdo); - - port->operating_snk_mw = tcfg->operating_snk_mw; - - port->typec_caps.prefer_role = tcfg->default_role; - port->typec_caps.type = tcfg->type; - port->typec_caps.data = tcfg->data; - port->self_powered = tcfg->self_powered; - - return 0; -} - struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) { struct tcpm_port *port; - int i, err; + int err; if (!dev || !tcpc || !tcpc->get_vbus || !tcpc->set_cc || !tcpc->get_cc || @@ -4785,24 +4701,16 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) tcpm_debugfs_init(port); err = tcpm_fw_get_caps(port, tcpc->fwnode); - if ((err < 0) && tcpc->config) - err = tcpm_copy_caps(port, tcpc->config); if (err < 0) goto out_destroy_wq; - if (!tcpc->config || !tcpc->config->try_role_hw) - port->try_role = port->typec_caps.prefer_role; - else - port->try_role = TYPEC_NO_PREFERRED_ROLE; + port->try_role = port->typec_caps.prefer_role; port->typec_caps.fwnode = tcpc->fwnode; port->typec_caps.revision = 0x0120; /* Type-C spec release 1.2 */ port->typec_caps.pd_revision = 0x0300; /* USB-PD spec release 3.0 */ - port->typec_caps.dr_set = tcpm_dr_set; - port->typec_caps.pr_set = tcpm_pr_set; - port->typec_caps.vconn_set = tcpm_vconn_set; - port->typec_caps.try_role = tcpm_try_role; - port->typec_caps.port_type_set = tcpm_port_type_set; + port->typec_caps.driver_data = port; + port->typec_caps.ops = &tcpm_ops; port->partner_desc.identity = &port->partner_ident; port->port_type = port->typec_caps.type; @@ -4823,29 +4731,6 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) goto out_role_sw_put; } - if (tcpc->config && tcpc->config->alt_modes) { - const struct typec_altmode_desc *paltmode = tcpc->config->alt_modes; - - i = 0; - while (paltmode->svid && i < ARRAY_SIZE(port->port_altmode)) { - struct typec_altmode *alt; - - alt = typec_port_register_altmode(port->typec_port, - paltmode); - if (IS_ERR(alt)) { - tcpm_log(port, - "%s: failed to register port alternate mode 0x%x", - dev_name(dev), paltmode->svid); - break; - } - typec_altmode_set_drvdata(alt, port); - alt->ops = &tcpm_altmode_ops; - port->port_altmode[i] = alt; - i++; - paltmode++; - } - } - mutex_lock(&port->lock); tcpm_init(port); mutex_unlock(&port->lock); diff --git a/drivers/usb/typec/tps6598x.c b/drivers/usb/typec/tps6598x.c index a38d1409f15b7..0698addd1185b 100644 --- a/drivers/usb/typec/tps6598x.c +++ b/drivers/usb/typec/tps6598x.c @@ -94,7 +94,6 @@ struct tps6598x { struct typec_port *port; struct typec_partner *partner; struct usb_pd_identity partner_identity; - struct typec_capability typec_cap; }; /* @@ -307,11 +306,10 @@ static int tps6598x_exec_cmd(struct tps6598x *tps, const char *cmd, return 0; } -static int -tps6598x_dr_set(const struct typec_capability *cap, enum typec_data_role role) +static int tps6598x_dr_set(struct typec_port *port, enum typec_data_role role) { - struct tps6598x *tps = container_of(cap, struct tps6598x, typec_cap); const char *cmd = (role == TYPEC_DEVICE) ? "SWUF" : "SWDF"; + struct tps6598x *tps = typec_get_drvdata(port); u32 status; int ret; @@ -338,11 +336,10 @@ tps6598x_dr_set(const struct typec_capability *cap, enum typec_data_role role) return ret; } -static int -tps6598x_pr_set(const struct typec_capability *cap, enum typec_role role) +static int tps6598x_pr_set(struct typec_port *port, enum typec_role role) { - struct tps6598x *tps = container_of(cap, struct tps6598x, typec_cap); const char *cmd = (role == TYPEC_SINK) ? "SWSk" : "SWSr"; + struct tps6598x *tps = typec_get_drvdata(port); u32 status; int ret; @@ -369,6 +366,11 @@ tps6598x_pr_set(const struct typec_capability *cap, enum typec_role role) return ret; } +static const struct typec_operations tps6598x_ops = { + .dr_set = tps6598x_dr_set, + .pr_set = tps6598x_pr_set, +}; + static irqreturn_t tps6598x_interrupt(int irq, void *data) { struct tps6598x *tps = data; @@ -448,6 +450,7 @@ static const struct regmap_config tps6598x_regmap_config = { static int tps6598x_probe(struct i2c_client *client) { + struct typec_capability typec_cap = { }; struct tps6598x *tps; u32 status; u32 conf; @@ -492,40 +495,40 @@ static int tps6598x_probe(struct i2c_client *client) if (ret < 0) return ret; - tps->typec_cap.revision = USB_TYPEC_REV_1_2; - tps->typec_cap.pd_revision = 0x200; - tps->typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; - tps->typec_cap.pr_set = tps6598x_pr_set; - tps->typec_cap.dr_set = tps6598x_dr_set; + typec_cap.revision = USB_TYPEC_REV_1_2; + typec_cap.pd_revision = 0x200; + typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; + typec_cap.driver_data = tps; + typec_cap.ops = &tps6598x_ops; switch (TPS_SYSCONF_PORTINFO(conf)) { case TPS_PORTINFO_SINK_ACCESSORY: case TPS_PORTINFO_SINK: - tps->typec_cap.type = TYPEC_PORT_SNK; - tps->typec_cap.data = TYPEC_PORT_UFP; + typec_cap.type = TYPEC_PORT_SNK; + typec_cap.data = TYPEC_PORT_UFP; break; case TPS_PORTINFO_DRP_UFP_DRD: case TPS_PORTINFO_DRP_DFP_DRD: - tps->typec_cap.type = TYPEC_PORT_DRP; - tps->typec_cap.data = TYPEC_PORT_DRD; + typec_cap.type = TYPEC_PORT_DRP; + typec_cap.data = TYPEC_PORT_DRD; break; case TPS_PORTINFO_DRP_UFP: - tps->typec_cap.type = TYPEC_PORT_DRP; - tps->typec_cap.data = TYPEC_PORT_UFP; + typec_cap.type = TYPEC_PORT_DRP; + typec_cap.data = TYPEC_PORT_UFP; break; case TPS_PORTINFO_DRP_DFP: - tps->typec_cap.type = TYPEC_PORT_DRP; - tps->typec_cap.data = TYPEC_PORT_DFP; + typec_cap.type = TYPEC_PORT_DRP; + typec_cap.data = TYPEC_PORT_DFP; break; case TPS_PORTINFO_SOURCE: - tps->typec_cap.type = TYPEC_PORT_SRC; - tps->typec_cap.data = TYPEC_PORT_DFP; + typec_cap.type = TYPEC_PORT_SRC; + typec_cap.data = TYPEC_PORT_DFP; break; default: return -ENODEV; } - tps->port = typec_register_port(&client->dev, &tps->typec_cap); + tps->port = typec_register_port(&client->dev, &typec_cap); if (IS_ERR(tps->port)) return PTR_ERR(tps->port); diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 166c2aabb512b..77655d9ce5498 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -48,7 +48,8 @@ struct ucsi_dp { static int ucsi_displayport_enter(struct typec_altmode *alt) { struct ucsi_dp *dp = typec_altmode_get_drvdata(alt); - struct ucsi_control ctrl; + struct ucsi *ucsi = dp->con->ucsi; + u64 command; u8 cur = 0; int ret; @@ -59,25 +60,21 @@ static int ucsi_displayport_enter(struct typec_altmode *alt) dev_warn(&p->dev, "firmware doesn't support alternate mode overriding\n"); - mutex_unlock(&dp->con->lock); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto err_unlock; } - UCSI_CMD_GET_CURRENT_CAM(ctrl, dp->con->num); - ret = ucsi_send_command(dp->con->ucsi, &ctrl, &cur, sizeof(cur)); + command = UCSI_GET_CURRENT_CAM | UCSI_CONNECTOR_NUMBER(dp->con->num); + ret = ucsi_send_command(ucsi, command, &cur, sizeof(cur)); if (ret < 0) { - if (dp->con->ucsi->ppm->data->version > 0x0100) { - mutex_unlock(&dp->con->lock); - return ret; - } + if (ucsi->version > 0x0100) + goto err_unlock; cur = 0xff; } if (cur != 0xff) { - mutex_unlock(&dp->con->lock); - if (dp->con->port_altmode[cur] == alt) - return 0; - return -EBUSY; + ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY; + goto err_unlock; } /* @@ -94,16 +91,17 @@ static int ucsi_displayport_enter(struct typec_altmode *alt) dp->vdo_size = 1; schedule_work(&dp->work); - + ret = 0; +err_unlock: mutex_unlock(&dp->con->lock); - return 0; + return ret; } static int ucsi_displayport_exit(struct typec_altmode *alt) { struct ucsi_dp *dp = typec_altmode_get_drvdata(alt); - struct ucsi_control ctrl; + u64 command; int ret = 0; mutex_lock(&dp->con->lock); @@ -117,8 +115,8 @@ static int ucsi_displayport_exit(struct typec_altmode *alt) goto out_unlock; } - ctrl.raw_cmd = UCSI_CMD_SET_NEW_CAM(dp->con->num, 0, dp->offset, 0); - ret = ucsi_send_command(dp->con->ucsi, &ctrl, NULL, 0); + command = UCSI_CMD_SET_NEW_CAM(dp->con->num, 0, dp->offset, 0); + ret = ucsi_send_command(dp->con->ucsi, command, NULL, 0); if (ret < 0) goto out_unlock; @@ -172,14 +170,14 @@ static int ucsi_displayport_status_update(struct ucsi_dp *dp) static int ucsi_displayport_configure(struct ucsi_dp *dp) { u32 pins = DP_CONF_GET_PIN_ASSIGN(dp->data.conf); - struct ucsi_control ctrl; + u64 command; if (!dp->override) return 0; - ctrl.raw_cmd = UCSI_CMD_SET_NEW_CAM(dp->con->num, 1, dp->offset, pins); + command = UCSI_CMD_SET_NEW_CAM(dp->con->num, 1, dp->offset, pins); - return ucsi_send_command(dp->con->ucsi, &ctrl, NULL, 0); + return ucsi_send_command(dp->con->ucsi, command, NULL, 0); } static int ucsi_displayport_vdm(struct typec_altmode *alt, diff --git a/drivers/usb/typec/ucsi/trace.c b/drivers/usb/typec/ucsi/trace.c index 1dabafb743205..48ad1dc1b1b2e 100644 --- a/drivers/usb/typec/ucsi/trace.c +++ b/drivers/usb/typec/ucsi/trace.c @@ -33,17 +33,6 @@ const char *ucsi_cmd_str(u64 raw_cmd) return ucsi_cmd_strs[(cmd >= ARRAY_SIZE(ucsi_cmd_strs)) ? 0 : cmd]; } -static const char * const ucsi_ack_strs[] = { - [0] = "", - [UCSI_ACK_EVENT] = "event", - [UCSI_ACK_CMD] = "command", -}; - -const char *ucsi_ack_str(u8 ack) -{ - return ucsi_ack_strs[(ack >= ARRAY_SIZE(ucsi_ack_strs)) ? 0 : ack]; -} - const char *ucsi_cci_str(u32 cci) { if (cci & GENMASK(7, 0)) { diff --git a/drivers/usb/typec/ucsi/trace.h b/drivers/usb/typec/ucsi/trace.h index 783ec9c720559..a0d3a934d3d92 100644 --- a/drivers/usb/typec/ucsi/trace.h +++ b/drivers/usb/typec/ucsi/trace.h @@ -10,54 +10,18 @@ #include const char *ucsi_cmd_str(u64 raw_cmd); -const char *ucsi_ack_str(u8 ack); const char *ucsi_cci_str(u32 cci); const char *ucsi_recipient_str(u8 recipient); -DECLARE_EVENT_CLASS(ucsi_log_ack, - TP_PROTO(u8 ack), - TP_ARGS(ack), - TP_STRUCT__entry( - __field(u8, ack) - ), - TP_fast_assign( - __entry->ack = ack; - ), - TP_printk("ACK %s", ucsi_ack_str(__entry->ack)) -); - -DEFINE_EVENT(ucsi_log_ack, ucsi_ack, - TP_PROTO(u8 ack), - TP_ARGS(ack) -); - -DECLARE_EVENT_CLASS(ucsi_log_control, - TP_PROTO(struct ucsi_control *ctrl), - TP_ARGS(ctrl), - TP_STRUCT__entry( - __field(u64, ctrl) - ), - TP_fast_assign( - __entry->ctrl = ctrl->raw_cmd; - ), - TP_printk("control=%08llx (%s)", __entry->ctrl, - ucsi_cmd_str(__entry->ctrl)) -); - -DEFINE_EVENT(ucsi_log_control, ucsi_command, - TP_PROTO(struct ucsi_control *ctrl), - TP_ARGS(ctrl) -); - DECLARE_EVENT_CLASS(ucsi_log_command, - TP_PROTO(struct ucsi_control *ctrl, int ret), - TP_ARGS(ctrl, ret), + TP_PROTO(u64 command, int ret), + TP_ARGS(command, ret), TP_STRUCT__entry( __field(u64, ctrl) __field(int, ret) ), TP_fast_assign( - __entry->ctrl = ctrl->raw_cmd; + __entry->ctrl = command; __entry->ret = ret; ), TP_printk("%s -> %s (err=%d)", ucsi_cmd_str(__entry->ctrl), @@ -66,30 +30,13 @@ DECLARE_EVENT_CLASS(ucsi_log_command, ); DEFINE_EVENT(ucsi_log_command, ucsi_run_command, - TP_PROTO(struct ucsi_control *ctrl, int ret), - TP_ARGS(ctrl, ret) + TP_PROTO(u64 command, int ret), + TP_ARGS(command, ret) ); DEFINE_EVENT(ucsi_log_command, ucsi_reset_ppm, - TP_PROTO(struct ucsi_control *ctrl, int ret), - TP_ARGS(ctrl, ret) -); - -DECLARE_EVENT_CLASS(ucsi_log_cci, - TP_PROTO(u32 cci), - TP_ARGS(cci), - TP_STRUCT__entry( - __field(u32, cci) - ), - TP_fast_assign( - __entry->cci = cci; - ), - TP_printk("CCI=%08x %s", __entry->cci, ucsi_cci_str(__entry->cci)) -); - -DEFINE_EVENT(ucsi_log_cci, ucsi_notify, - TP_PROTO(u32 cci), - TP_ARGS(cci) + TP_PROTO(u64 command, int ret), + TP_ARGS(command, ret) ); DECLARE_EVENT_CLASS(ucsi_log_connector_status, @@ -109,13 +56,13 @@ DECLARE_EVENT_CLASS(ucsi_log_connector_status, TP_fast_assign( __entry->port = port - 1; __entry->change = status->change; - __entry->opmode = status->pwr_op_mode; - __entry->connected = status->connected; - __entry->pwr_dir = status->pwr_dir; - __entry->partner_flags = status->partner_flags; - __entry->partner_type = status->partner_type; + __entry->opmode = UCSI_CONSTAT_PWR_OPMODE(status->flags); + __entry->connected = !!(status->flags & UCSI_CONSTAT_CONNECTED); + __entry->pwr_dir = !!(status->flags & UCSI_CONSTAT_PWR_DIR); + __entry->partner_flags = UCSI_CONSTAT_PARTNER_FLAGS(status->flags); + __entry->partner_type = UCSI_CONSTAT_PARTNER_TYPE(status->flags); __entry->request_data_obj = status->request_data_obj; - __entry->bc_status = status->bc_status; + __entry->bc_status = UCSI_CONSTAT_BC_STATUS(status->pwr_status); ), TP_printk("port%d status: change=%04x, opmode=%x, connected=%d, " "sourcing=%d, partner_flags=%x, partner_type=%x, " diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 4d56408ac623d..4459bc68aa331 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -17,9 +17,6 @@ #include "ucsi.h" #include "trace.h" -#define to_ucsi_connector(_cap_) container_of(_cap_, struct ucsi_connector, \ - typec_cap) - /* * UCSI_TIMEOUT_MS - PPM communication timeout * @@ -39,169 +36,148 @@ */ #define UCSI_SWAP_TIMEOUT_MS 5000 -static inline int ucsi_sync(struct ucsi *ucsi) +static int ucsi_acknowledge_command(struct ucsi *ucsi) { - if (ucsi->ppm && ucsi->ppm->sync) - return ucsi->ppm->sync(ucsi->ppm); - return 0; + u64 ctrl; + + ctrl = UCSI_ACK_CC_CI; + ctrl |= UCSI_ACK_COMMAND_COMPLETE; + + return ucsi->ops->sync_write(ucsi, UCSI_CONTROL, &ctrl, sizeof(ctrl)); } -static int ucsi_command(struct ucsi *ucsi, struct ucsi_control *ctrl) +static int ucsi_acknowledge_connector_change(struct ucsi *ucsi) { + u64 ctrl; + + ctrl = UCSI_ACK_CC_CI; + ctrl |= UCSI_ACK_CONNECTOR_CHANGE; + + return ucsi->ops->async_write(ucsi, UCSI_CONTROL, &ctrl, sizeof(ctrl)); +} + +static int ucsi_exec_command(struct ucsi *ucsi, u64 command); + +static int ucsi_read_error(struct ucsi *ucsi) +{ + u16 error; int ret; - trace_ucsi_command(ctrl); + /* Acknowlege the command that failed */ + ret = ucsi_acknowledge_command(ucsi); + if (ret) + return ret; - set_bit(COMMAND_PENDING, &ucsi->flags); + ret = ucsi_exec_command(ucsi, UCSI_GET_ERROR_STATUS); + if (ret < 0) + return ret; - ret = ucsi->ppm->cmd(ucsi->ppm, ctrl); + ret = ucsi->ops->read(ucsi, UCSI_MESSAGE_IN, &error, sizeof(error)); if (ret) - goto err_clear_flag; + return ret; - if (!wait_for_completion_timeout(&ucsi->complete, - msecs_to_jiffies(UCSI_TIMEOUT_MS))) { - dev_warn(ucsi->dev, "PPM NOT RESPONDING\n"); - ret = -ETIMEDOUT; + switch (error) { + case UCSI_ERROR_INCOMPATIBLE_PARTNER: + return -EOPNOTSUPP; + case UCSI_ERROR_CC_COMMUNICATION_ERR: + return -ECOMM; + case UCSI_ERROR_CONTRACT_NEGOTIATION_FAIL: + return -EPROTO; + case UCSI_ERROR_DEAD_BATTERY: + dev_warn(ucsi->dev, "Dead battery condition!\n"); + return -EPERM; + case UCSI_ERROR_INVALID_CON_NUM: + case UCSI_ERROR_UNREGONIZED_CMD: + case UCSI_ERROR_INVALID_CMD_ARGUMENT: + dev_err(ucsi->dev, "possible UCSI driver bug %u\n", error); + return -EINVAL; + case UCSI_ERROR_OVERCURRENT: + dev_warn(ucsi->dev, "Overcurrent condition\n"); + break; + case UCSI_ERROR_PARTNER_REJECTED_SWAP: + dev_warn(ucsi->dev, "Partner rejected swap\n"); + break; + case UCSI_ERROR_HARD_RESET: + dev_warn(ucsi->dev, "Hard reset occurred\n"); + break; + case UCSI_ERROR_PPM_POLICY_CONFLICT: + dev_warn(ucsi->dev, "PPM Policy conflict\n"); + break; + case UCSI_ERROR_SWAP_REJECTED: + dev_warn(ucsi->dev, "Swap rejected\n"); + break; + case UCSI_ERROR_UNDEFINED: + default: + dev_err(ucsi->dev, "unknown error %u\n", error); + break; } -err_clear_flag: - clear_bit(COMMAND_PENDING, &ucsi->flags); - - return ret; + return -EIO; } -static int ucsi_ack(struct ucsi *ucsi, u8 ack) +static int ucsi_exec_command(struct ucsi *ucsi, u64 cmd) { - struct ucsi_control ctrl; + u32 cci; int ret; - trace_ucsi_ack(ack); - - set_bit(ACK_PENDING, &ucsi->flags); + ret = ucsi->ops->sync_write(ucsi, UCSI_CONTROL, &cmd, sizeof(cmd)); + if (ret) + return ret; - UCSI_CMD_ACK(ctrl, ack); - ret = ucsi->ppm->cmd(ucsi->ppm, &ctrl); + ret = ucsi->ops->read(ucsi, UCSI_CCI, &cci, sizeof(cci)); if (ret) - goto out_clear_bit; + return ret; - /* Waiting for ACK with ACK CMD, but not with EVENT for now */ - if (ack == UCSI_ACK_EVENT) - goto out_clear_bit; + if (cci & UCSI_CCI_BUSY) + return -EBUSY; - if (!wait_for_completion_timeout(&ucsi->complete, - msecs_to_jiffies(UCSI_TIMEOUT_MS))) - ret = -ETIMEDOUT; + if (!(cci & UCSI_CCI_COMMAND_COMPLETE)) + return -EIO; -out_clear_bit: - clear_bit(ACK_PENDING, &ucsi->flags); + if (cci & UCSI_CCI_NOT_SUPPORTED) + return -EOPNOTSUPP; - if (ret) - dev_err(ucsi->dev, "%s: failed\n", __func__); + if (cci & UCSI_CCI_ERROR) { + if (cmd == UCSI_GET_ERROR_STATUS) + return -EIO; + return ucsi_read_error(ucsi); + } - return ret; + return UCSI_CCI_LENGTH(cci); } -static int ucsi_run_command(struct ucsi *ucsi, struct ucsi_control *ctrl, +static int ucsi_run_command(struct ucsi *ucsi, u64 command, void *data, size_t size) { - struct ucsi_control _ctrl; - u8 data_length; - u16 error; + u8 length; int ret; - ret = ucsi_command(ucsi, ctrl); - if (ret) - goto err; - - switch (ucsi->status) { - case UCSI_IDLE: - ret = ucsi_sync(ucsi); - if (ret) - dev_warn(ucsi->dev, "%s: sync failed\n", __func__); - - if (data) - memcpy(data, ucsi->ppm->data->message_in, size); - - data_length = ucsi->ppm->data->cci.data_length; - - ret = ucsi_ack(ucsi, UCSI_ACK_CMD); - if (!ret) - ret = data_length; - break; - case UCSI_BUSY: - /* The caller decides whether to cancel or not */ - ret = -EBUSY; - break; - case UCSI_ERROR: - ret = ucsi_ack(ucsi, UCSI_ACK_CMD); - if (ret) - break; - - _ctrl.raw_cmd = 0; - _ctrl.cmd.cmd = UCSI_GET_ERROR_STATUS; - ret = ucsi_command(ucsi, &_ctrl); - if (ret) { - dev_err(ucsi->dev, "reading error failed!\n"); - break; - } - - memcpy(&error, ucsi->ppm->data->message_in, sizeof(error)); + ret = ucsi_exec_command(ucsi, command); + if (ret < 0) + return ret; - /* Something has really gone wrong */ - if (WARN_ON(ucsi->status == UCSI_ERROR)) { - ret = -ENODEV; - break; - } + length = ret; - ret = ucsi_ack(ucsi, UCSI_ACK_CMD); + if (data) { + ret = ucsi->ops->read(ucsi, UCSI_MESSAGE_IN, data, size); if (ret) - break; - - switch (error) { - case UCSI_ERROR_INCOMPATIBLE_PARTNER: - ret = -EOPNOTSUPP; - break; - case UCSI_ERROR_CC_COMMUNICATION_ERR: - ret = -ECOMM; - break; - case UCSI_ERROR_CONTRACT_NEGOTIATION_FAIL: - ret = -EPROTO; - break; - case UCSI_ERROR_DEAD_BATTERY: - dev_warn(ucsi->dev, "Dead battery condition!\n"); - ret = -EPERM; - break; - /* The following mean a bug in this driver */ - case UCSI_ERROR_INVALID_CON_NUM: - case UCSI_ERROR_UNREGONIZED_CMD: - case UCSI_ERROR_INVALID_CMD_ARGUMENT: - dev_warn(ucsi->dev, - "%s: possible UCSI driver bug - error 0x%x\n", - __func__, error); - ret = -EINVAL; - break; - default: - dev_warn(ucsi->dev, - "%s: error without status\n", __func__); - ret = -EIO; - break; - } - break; + return ret; } -err: - trace_ucsi_run_command(ctrl, ret); + ret = ucsi_acknowledge_command(ucsi); + if (ret) + return ret; - return ret; + return length; } -int ucsi_send_command(struct ucsi *ucsi, struct ucsi_control *ctrl, +int ucsi_send_command(struct ucsi *ucsi, u64 command, void *retval, size_t size) { int ret; mutex_lock(&ucsi->ppm_lock); - ret = ucsi_run_command(ucsi, ctrl, retval, size); + ret = ucsi_run_command(ucsi, command, retval, size); mutex_unlock(&ucsi->ppm_lock); return ret; @@ -210,11 +186,12 @@ EXPORT_SYMBOL_GPL(ucsi_send_command); int ucsi_resume(struct ucsi *ucsi) { - struct ucsi_control ctrl; + u64 command; /* Restore UCSI notification enable mask after system resume */ - UCSI_CMD_SET_NTFY_ENABLE(ctrl, UCSI_ENABLE_NTFY_ALL); - return ucsi_send_command(ucsi, &ctrl, NULL, 0); + command = UCSI_SET_NOTIFICATION_ENABLE | UCSI_ENABLE_NTFY_ALL; + + return ucsi_send_command(ucsi, command, NULL, 0); } EXPORT_SYMBOL_GPL(ucsi_resume); /* -------------------------------------------------------------------------- */ @@ -222,15 +199,15 @@ EXPORT_SYMBOL_GPL(ucsi_resume); void ucsi_altmode_update_active(struct ucsi_connector *con) { const struct typec_altmode *altmode = NULL; - struct ucsi_control ctrl; + u64 command; int ret; u8 cur; int i; - UCSI_CMD_GET_CURRENT_CAM(ctrl, con->num); - ret = ucsi_run_command(con->ucsi, &ctrl, &cur, sizeof(cur)); + command = UCSI_GET_CURRENT_CAM | UCSI_CONNECTOR_NUMBER(con->num); + ret = ucsi_run_command(con->ucsi, command, &cur, sizeof(cur)); if (ret < 0) { - if (con->ucsi->ppm->data->version > 0x0100) { + if (con->ucsi->version > 0x0100) { dev_err(con->ucsi->dev, "GET_CURRENT_CAM command failed\n"); return; @@ -246,18 +223,14 @@ void ucsi_altmode_update_active(struct ucsi_connector *con) con->partner_altmode[i] == altmode); } -static int ucsi_altmode_next_mode(struct typec_altmode **alt, u16 svid) +static u8 ucsi_altmode_next_mode(struct typec_altmode **alt, u16 svid) { u8 mode = 1; int i; - for (i = 0; alt[i]; i++) { - if (i > MODE_DISCOVERY_MAX) - return -ERANGE; - + for (i = 0; alt[i]; i++) if (alt[i]->svid == svid) mode++; - } return mode; } @@ -292,11 +265,8 @@ static int ucsi_register_altmode(struct ucsi_connector *con, goto err; } - ret = ucsi_altmode_next_mode(con->port_altmode, desc->svid); - if (ret < 0) - return ret; - - desc->mode = ret; + desc->mode = ucsi_altmode_next_mode(con->port_altmode, + desc->svid); switch (desc->svid) { case USB_TYPEC_DP_SID: @@ -322,11 +292,8 @@ static int ucsi_register_altmode(struct ucsi_connector *con, goto err; } - ret = ucsi_altmode_next_mode(con->partner_altmode, desc->svid); - if (ret < 0) - return ret; - - desc->mode = ret; + desc->mode = ucsi_altmode_next_mode(con->partner_altmode, + desc->svid); alt = typec_partner_register_altmode(con->partner, desc); if (IS_ERR(alt)) { @@ -356,7 +323,7 @@ static int ucsi_register_altmodes(struct ucsi_connector *con, u8 recipient) int max_altmodes = UCSI_MAX_ALTMODES; struct typec_altmode_desc desc; struct ucsi_altmode alt[2]; - struct ucsi_control ctrl; + u64 command; int num = 1; int ret; int len; @@ -374,8 +341,11 @@ static int ucsi_register_altmodes(struct ucsi_connector *con, u8 recipient) for (i = 0; i < max_altmodes;) { memset(alt, 0, sizeof(alt)); - UCSI_CMD_GET_ALTERNATE_MODES(ctrl, recipient, con->num, i, 1); - len = ucsi_run_command(con->ucsi, &ctrl, alt, sizeof(alt)); + command = UCSI_GET_ALTERNATE_MODES; + command |= UCSI_GET_ALTMODE_RECIPIENT(recipient); + command |= UCSI_GET_ALTMODE_CONNECTOR_NUMBER(con->num); + command |= UCSI_GET_ALTMODE_OFFSET(i); + len = ucsi_run_command(con->ucsi, command, alt, sizeof(alt)); if (len <= 0) return len; @@ -437,7 +407,7 @@ static void ucsi_unregister_altmodes(struct ucsi_connector *con, u8 recipient) static void ucsi_pwr_opmode_change(struct ucsi_connector *con) { - switch (con->status.pwr_op_mode) { + switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) { case UCSI_CONSTAT_PWR_OPMODE_PD: typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_PD); break; @@ -455,6 +425,7 @@ static void ucsi_pwr_opmode_change(struct ucsi_connector *con) static int ucsi_register_partner(struct ucsi_connector *con) { + u8 pwr_opmode = UCSI_CONSTAT_PWR_OPMODE(con->status.flags); struct typec_partner_desc desc; struct typec_partner *partner; @@ -463,7 +434,7 @@ static int ucsi_register_partner(struct ucsi_connector *con) memset(&desc, 0, sizeof(desc)); - switch (con->status.partner_type) { + switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) { case UCSI_CONSTAT_PARTNER_TYPE_DEBUG: desc.accessory = TYPEC_ACCESSORY_DEBUG; break; @@ -474,7 +445,7 @@ static int ucsi_register_partner(struct ucsi_connector *con) break; } - desc.usb_pd = con->status.pwr_op_mode == UCSI_CONSTAT_PWR_OPMODE_PD; + desc.usb_pd = pwr_opmode == UCSI_CONSTAT_PWR_OPMODE_PD; partner = typec_register_partner(con->port, &desc); if (IS_ERR(partner)) { @@ -506,7 +477,7 @@ static void ucsi_partner_change(struct ucsi_connector *con) if (!con->partner) return; - switch (con->status.partner_type) { + switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) { case UCSI_CONSTAT_PARTNER_TYPE_UFP: typec_set_data_role(con->port, TYPEC_HOST); break; @@ -531,29 +502,33 @@ static void ucsi_partner_change(struct ucsi_connector *con) ucsi_altmode_update_active(con); } -static void ucsi_connector_change(struct work_struct *work) +static void ucsi_handle_connector_change(struct work_struct *work) { struct ucsi_connector *con = container_of(work, struct ucsi_connector, work); struct ucsi *ucsi = con->ucsi; - struct ucsi_control ctrl; + enum typec_role role; + u64 command; int ret; mutex_lock(&con->lock); - UCSI_CMD_GET_CONNECTOR_STATUS(ctrl, con->num); - ret = ucsi_send_command(ucsi, &ctrl, &con->status, sizeof(con->status)); + command = UCSI_GET_CONNECTOR_STATUS | UCSI_CONNECTOR_NUMBER(con->num); + ret = ucsi_send_command(ucsi, command, &con->status, + sizeof(con->status)); if (ret < 0) { dev_err(ucsi->dev, "%s: GET_CONNECTOR_STATUS failed (%d)\n", __func__, ret); goto out_unlock; } + role = !!(con->status.flags & UCSI_CONSTAT_PWR_DIR); + if (con->status.change & UCSI_CONSTAT_POWER_OPMODE_CHANGE) ucsi_pwr_opmode_change(con); if (con->status.change & UCSI_CONSTAT_POWER_DIR_CHANGE) { - typec_set_pwr_role(con->port, con->status.pwr_dir); + typec_set_pwr_role(con->port, role); /* Complete pending power role swap */ if (!completion_done(&con->complete)) @@ -561,9 +536,9 @@ static void ucsi_connector_change(struct work_struct *work) } if (con->status.change & UCSI_CONSTAT_CONNECT_CHANGE) { - typec_set_pwr_role(con->port, con->status.pwr_dir); + typec_set_pwr_role(con->port, role); - switch (con->status.partner_type) { + switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) { case UCSI_CONSTAT_PARTNER_TYPE_UFP: typec_set_data_role(con->port, TYPEC_HOST); break; @@ -574,7 +549,7 @@ static void ucsi_connector_change(struct work_struct *work) break; } - if (con->status.connected) + if (con->status.flags & UCSI_CONSTAT_CONNECTED) ucsi_register_partner(con); else ucsi_unregister_partner(con); @@ -586,14 +561,15 @@ static void ucsi_connector_change(struct work_struct *work) * Running GET_CAM_SUPPORTED command just to make sure the PPM * does not get stuck in case it assumes we do so. */ - UCSI_CMD_GET_CAM_SUPPORTED(ctrl, con->num); - ucsi_run_command(con->ucsi, &ctrl, NULL, 0); + command = UCSI_GET_CAM_SUPPORTED; + command |= UCSI_CONNECTOR_NUMBER(con->num); + ucsi_run_command(con->ucsi, command, NULL, 0); } if (con->status.change & UCSI_CONSTAT_PARTNER_CHANGE) ucsi_partner_change(con); - ret = ucsi_ack(ucsi, UCSI_ACK_EVENT); + ret = ucsi_acknowledge_connector_change(ucsi); if (ret) dev_err(ucsi->dev, "%s: ACK failed (%d)", __func__, ret); @@ -605,117 +581,83 @@ static void ucsi_connector_change(struct work_struct *work) } /** - * ucsi_notify - PPM notification handler - * @ucsi: Source UCSI Interface for the notifications - * - * Handle notifications from PPM of @ucsi. + * ucsi_connector_change - Process Connector Change Event + * @ucsi: UCSI Interface + * @num: Connector number */ -void ucsi_notify(struct ucsi *ucsi) +void ucsi_connector_change(struct ucsi *ucsi, u8 num) { - struct ucsi_cci *cci; - - /* There is no requirement to sync here, but no harm either. */ - ucsi_sync(ucsi); + struct ucsi_connector *con = &ucsi->connector[num - 1]; - cci = &ucsi->ppm->data->cci; - - if (cci->error) - ucsi->status = UCSI_ERROR; - else if (cci->busy) - ucsi->status = UCSI_BUSY; - else - ucsi->status = UCSI_IDLE; - - if (cci->cmd_complete && test_bit(COMMAND_PENDING, &ucsi->flags)) { - complete(&ucsi->complete); - } else if (cci->ack_complete && test_bit(ACK_PENDING, &ucsi->flags)) { - complete(&ucsi->complete); - } else if (cci->connector_change) { - struct ucsi_connector *con; - - con = &ucsi->connector[cci->connector_change - 1]; - - if (!test_and_set_bit(EVENT_PENDING, &ucsi->flags)) - schedule_work(&con->work); - } - - trace_ucsi_notify(ucsi->ppm->data->raw_cci); + if (!test_and_set_bit(EVENT_PENDING, &ucsi->flags)) + schedule_work(&con->work); } -EXPORT_SYMBOL_GPL(ucsi_notify); +EXPORT_SYMBOL_GPL(ucsi_connector_change); /* -------------------------------------------------------------------------- */ static int ucsi_reset_connector(struct ucsi_connector *con, bool hard) { - struct ucsi_control ctrl; + u64 command; - UCSI_CMD_CONNECTOR_RESET(ctrl, con, hard); + command = UCSI_CONNECTOR_RESET | UCSI_CONNECTOR_NUMBER(con->num); + command |= hard ? UCSI_CONNECTOR_RESET_HARD : 0; - return ucsi_send_command(con->ucsi, &ctrl, NULL, 0); + return ucsi_send_command(con->ucsi, command, NULL, 0); } static int ucsi_reset_ppm(struct ucsi *ucsi) { - struct ucsi_control ctrl; + u64 command = UCSI_PPM_RESET; unsigned long tmo; + u32 cci; int ret; - ctrl.raw_cmd = 0; - ctrl.cmd.cmd = UCSI_PPM_RESET; - trace_ucsi_command(&ctrl); - ret = ucsi->ppm->cmd(ucsi->ppm, &ctrl); - if (ret) - goto err; + ret = ucsi->ops->async_write(ucsi, UCSI_CONTROL, &command, + sizeof(command)); + if (ret < 0) + return ret; tmo = jiffies + msecs_to_jiffies(UCSI_TIMEOUT_MS); do { - /* Here sync is critical. */ - ret = ucsi_sync(ucsi); - if (ret) - goto err; + if (time_is_before_jiffies(tmo)) + return -ETIMEDOUT; - if (ucsi->ppm->data->cci.reset_complete) - break; + ret = ucsi->ops->read(ucsi, UCSI_CCI, &cci, sizeof(cci)); + if (ret) + return ret; /* If the PPM is still doing something else, reset it again. */ - if (ucsi->ppm->data->raw_cci) { - dev_warn_ratelimited(ucsi->dev, - "Failed to reset PPM! Trying again..\n"); - - trace_ucsi_command(&ctrl); - ret = ucsi->ppm->cmd(ucsi->ppm, &ctrl); - if (ret) - goto err; + if (cci & ~UCSI_CCI_RESET_COMPLETE) { + ret = ucsi->ops->async_write(ucsi, UCSI_CONTROL, + &command, + sizeof(command)); + if (ret < 0) + return ret; } - /* Letting the PPM settle down. */ msleep(20); + } while (!(cci & UCSI_CCI_RESET_COMPLETE)); - ret = -ETIMEDOUT; - } while (time_is_after_jiffies(tmo)); - -err: - trace_ucsi_reset_ppm(&ctrl, ret); - - return ret; + return 0; } -static int ucsi_role_cmd(struct ucsi_connector *con, struct ucsi_control *ctrl) +static int ucsi_role_cmd(struct ucsi_connector *con, u64 command) { int ret; - ret = ucsi_send_command(con->ucsi, ctrl, NULL, 0); + ret = ucsi_send_command(con->ucsi, command, NULL, 0); if (ret == -ETIMEDOUT) { - struct ucsi_control c; + u64 c; /* PPM most likely stopped responding. Resetting everything. */ mutex_lock(&con->ucsi->ppm_lock); ucsi_reset_ppm(con->ucsi); mutex_unlock(&con->ucsi->ppm_lock); - UCSI_CMD_SET_NTFY_ENABLE(c, UCSI_ENABLE_NTFY_ALL); - ucsi_send_command(con->ucsi, &c, NULL, 0); + c = UCSI_SET_NOTIFICATION_ENABLE | UCSI_ENABLE_NTFY_ALL; + ucsi_send_command(con->ucsi, c, NULL, 0); ucsi_reset_connector(con, true); } @@ -723,11 +665,11 @@ static int ucsi_role_cmd(struct ucsi_connector *con, struct ucsi_control *ctrl) return ret; } -static int -ucsi_dr_swap(const struct typec_capability *cap, enum typec_data_role role) +static int ucsi_dr_swap(struct typec_port *port, enum typec_data_role role) { - struct ucsi_connector *con = to_ucsi_connector(cap); - struct ucsi_control ctrl; + struct ucsi_connector *con = typec_get_drvdata(port); + u8 partner_type; + u64 command; int ret = 0; mutex_lock(&con->lock); @@ -737,14 +679,17 @@ ucsi_dr_swap(const struct typec_capability *cap, enum typec_data_role role) goto out_unlock; } - if ((con->status.partner_type == UCSI_CONSTAT_PARTNER_TYPE_DFP && + partner_type = UCSI_CONSTAT_PARTNER_TYPE(con->status.flags); + if ((partner_type == UCSI_CONSTAT_PARTNER_TYPE_DFP && role == TYPEC_DEVICE) || - (con->status.partner_type == UCSI_CONSTAT_PARTNER_TYPE_UFP && + (partner_type == UCSI_CONSTAT_PARTNER_TYPE_UFP && role == TYPEC_HOST)) goto out_unlock; - UCSI_CMD_SET_UOR(ctrl, con, role); - ret = ucsi_role_cmd(con, &ctrl); + command = UCSI_SET_UOR | UCSI_CONNECTOR_NUMBER(con->num); + command |= UCSI_SET_UOR_ROLE(role); + command |= UCSI_SET_UOR_ACCEPT_ROLE_SWAPS; + ret = ucsi_role_cmd(con, command); if (ret < 0) goto out_unlock; @@ -758,11 +703,11 @@ ucsi_dr_swap(const struct typec_capability *cap, enum typec_data_role role) return ret < 0 ? ret : 0; } -static int -ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role) +static int ucsi_pr_swap(struct typec_port *port, enum typec_role role) { - struct ucsi_connector *con = to_ucsi_connector(cap); - struct ucsi_control ctrl; + struct ucsi_connector *con = typec_get_drvdata(port); + enum typec_role cur_role; + u64 command; int ret = 0; mutex_lock(&con->lock); @@ -772,11 +717,15 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role) goto out_unlock; } - if (con->status.pwr_dir == role) + cur_role = !!(con->status.flags & UCSI_CONSTAT_PWR_DIR); + + if (cur_role == role) goto out_unlock; - UCSI_CMD_SET_PDR(ctrl, con, role); - ret = ucsi_role_cmd(con, &ctrl); + command = UCSI_SET_PDR | UCSI_CONNECTOR_NUMBER(con->num); + command |= UCSI_SET_PDR_ROLE(role); + command |= UCSI_SET_PDR_ACCEPT_ROLE_SWAPS; + ret = ucsi_role_cmd(con, command); if (ret < 0) goto out_unlock; @@ -787,7 +736,8 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role) } /* Something has gone wrong while swapping the role */ - if (con->status.pwr_op_mode != UCSI_CONSTAT_PWR_OPMODE_PD) { + if (UCSI_CONSTAT_PWR_OPMODE(con->status.flags) != + UCSI_CONSTAT_PWR_OPMODE_PD) { ucsi_reset_connector(con, true); ret = -EPROTO; } @@ -798,6 +748,11 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role) return ret; } +static const struct typec_operations ucsi_ops = { + .dr_set = ucsi_dr_swap, + .pr_set = ucsi_pr_swap +}; + static struct fwnode_handle *ucsi_find_fwnode(struct ucsi_connector *con) { struct fwnode_handle *fwnode; @@ -814,18 +769,19 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) struct ucsi_connector *con = &ucsi->connector[index]; struct typec_capability *cap = &con->typec_cap; enum typec_accessory *accessory = cap->accessory; - struct ucsi_control ctrl; + u64 command; int ret; - INIT_WORK(&con->work, ucsi_connector_change); + INIT_WORK(&con->work, ucsi_handle_connector_change); init_completion(&con->complete); mutex_init(&con->lock); con->num = index + 1; con->ucsi = ucsi; /* Get connector capability */ - UCSI_CMD_GET_CONNECTOR_CAPABILITY(ctrl, con->num); - ret = ucsi_run_command(ucsi, &ctrl, &con->cap, sizeof(con->cap)); + command = UCSI_GET_CONNECTOR_CAPABILITY; + command |= UCSI_CONNECTOR_NUMBER(con->num); + ret = ucsi_run_command(ucsi, command, &con->cap, sizeof(con->cap)); if (ret < 0) return ret; @@ -836,11 +792,12 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) else if (con->cap.op_mode & UCSI_CONCAP_OPMODE_UFP) cap->data = TYPEC_PORT_UFP; - if (con->cap.provider && con->cap.consumer) + if ((con->cap.flags & UCSI_CONCAP_FLAG_PROVIDER) && + (con->cap.flags & UCSI_CONCAP_FLAG_CONSUMER)) cap->type = TYPEC_PORT_DRP; - else if (con->cap.provider) + else if (con->cap.flags & UCSI_CONCAP_FLAG_PROVIDER) cap->type = TYPEC_PORT_SRC; - else if (con->cap.consumer) + else if (con->cap.flags & UCSI_CONCAP_FLAG_CONSUMER) cap->type = TYPEC_PORT_SNK; cap->revision = ucsi->cap.typec_version; @@ -853,8 +810,8 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) *accessory = TYPEC_ACCESSORY_DEBUG; cap->fwnode = ucsi_find_fwnode(con); - cap->dr_set = ucsi_dr_swap; - cap->pr_set = ucsi_pr_swap; + cap->driver_data = con; + cap->ops = &ucsi_ops; /* Register the connector */ con->port = typec_register_port(ucsi->dev, cap); @@ -868,17 +825,15 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) con->num); /* Get the status */ - UCSI_CMD_GET_CONNECTOR_STATUS(ctrl, con->num); - ret = ucsi_run_command(ucsi, &ctrl, &con->status, sizeof(con->status)); + command = UCSI_GET_CONNECTOR_STATUS | UCSI_CONNECTOR_NUMBER(con->num); + ret = ucsi_run_command(ucsi, command, &con->status, + sizeof(con->status)); if (ret < 0) { dev_err(ucsi->dev, "con%d: failed to get status\n", con->num); return 0; } - ucsi_pwr_opmode_change(con); - typec_set_pwr_role(con->port, con->status.pwr_dir); - - switch (con->status.partner_type) { + switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) { case UCSI_CONSTAT_PARTNER_TYPE_UFP: typec_set_data_role(con->port, TYPEC_HOST); break; @@ -890,8 +845,12 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) } /* Check if there is already something connected */ - if (con->status.connected) + if (con->status.flags & UCSI_CONSTAT_CONNECTED) { + typec_set_pwr_role(con->port, + !!(con->status.flags & UCSI_CONSTAT_PWR_DIR)); + ucsi_pwr_opmode_change(con); ucsi_register_partner(con); + } if (con->partner) { ret = ucsi_register_altmodes(con, UCSI_RECIPIENT_SOP); @@ -908,11 +867,16 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) return 0; } -static void ucsi_init(struct work_struct *work) +/** + * ucsi_init - Initialize UCSI interface + * @ucsi: UCSI to be initialized + * + * Registers all ports @ucsi has and enables all notification events. + */ +int ucsi_init(struct ucsi *ucsi) { - struct ucsi *ucsi = container_of(work, struct ucsi, work); struct ucsi_connector *con; - struct ucsi_control ctrl; + u64 command; int ret; int i; @@ -926,15 +890,15 @@ static void ucsi_init(struct work_struct *work) } /* Enable basic notifications */ - UCSI_CMD_SET_NTFY_ENABLE(ctrl, UCSI_ENABLE_NTFY_CMD_COMPLETE | - UCSI_ENABLE_NTFY_ERROR); - ret = ucsi_run_command(ucsi, &ctrl, NULL, 0); + command = UCSI_SET_NOTIFICATION_ENABLE; + command |= UCSI_ENABLE_NTFY_CMD_COMPLETE | UCSI_ENABLE_NTFY_ERROR; + ret = ucsi_run_command(ucsi, command, NULL, 0); if (ret < 0) goto err_reset; /* Get PPM capabilities */ - UCSI_CMD_GET_CAPABILITY(ctrl); - ret = ucsi_run_command(ucsi, &ctrl, &ucsi->cap, sizeof(ucsi->cap)); + command = UCSI_GET_CAPABILITY; + ret = ucsi_run_command(ucsi, command, &ucsi->cap, sizeof(ucsi->cap)); if (ret < 0) goto err_reset; @@ -959,14 +923,14 @@ static void ucsi_init(struct work_struct *work) } /* Enable all notifications */ - UCSI_CMD_SET_NTFY_ENABLE(ctrl, UCSI_ENABLE_NTFY_ALL); - ret = ucsi_run_command(ucsi, &ctrl, NULL, 0); + command = UCSI_SET_NOTIFICATION_ENABLE | UCSI_ENABLE_NTFY_ALL; + ret = ucsi_run_command(ucsi, command, NULL, 0); if (ret < 0) goto err_unregister; mutex_unlock(&ucsi->ppm_lock); - return; + return 0; err_unregister: for (con = ucsi->connector; con->port; con++) { @@ -980,59 +944,115 @@ static void ucsi_init(struct work_struct *work) ucsi_reset_ppm(ucsi); err: mutex_unlock(&ucsi->ppm_lock); - dev_err(ucsi->dev, "PPM init failed (%d)\n", ret); + + return ret; +} +EXPORT_SYMBOL_GPL(ucsi_init); + +static void ucsi_init_work(struct work_struct *work) +{ + struct ucsi *ucsi = container_of(work, struct ucsi, work); + int ret; + + ret = ucsi_init(ucsi); + if (ret) + dev_err(ucsi->dev, "PPM init failed (%d)\n", ret); } /** - * ucsi_register_ppm - Register UCSI PPM Interface - * @dev: Device interface to the PPM - * @ppm: The PPM interface - * - * Allocates UCSI instance, associates it with @ppm and returns it to the - * caller, and schedules initialization of the interface. + * ucsi_get_drvdata - Return private driver data pointer + * @ucsi: UCSI interface + */ +void *ucsi_get_drvdata(struct ucsi *ucsi) +{ + return ucsi->driver_data; +} +EXPORT_SYMBOL_GPL(ucsi_get_drvdata); + +/** + * ucsi_get_drvdata - Assign private driver data pointer + * @ucsi: UCSI interface + * @data: Private data pointer */ -struct ucsi *ucsi_register_ppm(struct device *dev, struct ucsi_ppm *ppm) +void ucsi_set_drvdata(struct ucsi *ucsi, void *data) +{ + ucsi->driver_data = data; +} +EXPORT_SYMBOL_GPL(ucsi_set_drvdata); + +/** + * ucsi_create - Allocate UCSI instance + * @dev: Device interface to the PPM (Platform Policy Manager) + * @ops: I/O routines + */ +struct ucsi *ucsi_create(struct device *dev, const struct ucsi_operations *ops) { struct ucsi *ucsi; + if (!ops || !ops->read || !ops->sync_write || !ops->async_write) + return ERR_PTR(-EINVAL); + ucsi = kzalloc(sizeof(*ucsi), GFP_KERNEL); if (!ucsi) return ERR_PTR(-ENOMEM); - INIT_WORK(&ucsi->work, ucsi_init); - init_completion(&ucsi->complete); + INIT_WORK(&ucsi->work, ucsi_init_work); mutex_init(&ucsi->ppm_lock); - ucsi->dev = dev; - ucsi->ppm = ppm; + ucsi->ops = ops; + + return ucsi; +} +EXPORT_SYMBOL_GPL(ucsi_create); + +/** + * ucsi_destroy - Free UCSI instance + * @ucsi: UCSI instance to be freed + */ +void ucsi_destroy(struct ucsi *ucsi) +{ + kfree(ucsi); +} +EXPORT_SYMBOL_GPL(ucsi_destroy); + +/** + * ucsi_register - Register UCSI interface + * @ucsi: UCSI instance + */ +int ucsi_register(struct ucsi *ucsi) +{ + int ret; + + ret = ucsi->ops->read(ucsi, UCSI_VERSION, &ucsi->version, + sizeof(ucsi->version)); + if (ret) + return ret; + + if (!ucsi->version) + return -ENODEV; - /* - * Communication with the PPM takes a lot of time. It is not reasonable - * to initialize the driver here. Using a work for now. - */ queue_work(system_long_wq, &ucsi->work); - return ucsi; + return 0; } -EXPORT_SYMBOL_GPL(ucsi_register_ppm); +EXPORT_SYMBOL_GPL(ucsi_register); /** - * ucsi_unregister_ppm - Unregister UCSI PPM Interface - * @ucsi: struct ucsi associated with the PPM + * ucsi_unregister - Unregister UCSI interface + * @ucsi: UCSI interface to be unregistered * - * Unregister UCSI PPM that was created with ucsi_register(). + * Unregister UCSI interface that was created with ucsi_register(). */ -void ucsi_unregister_ppm(struct ucsi *ucsi) +void ucsi_unregister(struct ucsi *ucsi) { - struct ucsi_control ctrl; + u64 cmd = UCSI_SET_NOTIFICATION_ENABLE; int i; /* Make sure that we are not in the middle of driver initialization */ cancel_work_sync(&ucsi->work); - /* Disable everything except command complete notification */ - UCSI_CMD_SET_NTFY_ENABLE(ctrl, UCSI_ENABLE_NTFY_CMD_COMPLETE) - ucsi_send_command(ucsi, &ctrl, NULL, 0); + /* Disable notifications */ + ucsi->ops->async_write(ucsi, UCSI_CONTROL, &cmd, sizeof(cmd)); for (i = 0; i < ucsi->cap.num_connectors; i++) { cancel_work_sync(&ucsi->connector[i].work); @@ -1042,12 +1062,9 @@ void ucsi_unregister_ppm(struct ucsi *ucsi) typec_unregister_port(ucsi->connector[i].port); } - ucsi_reset_ppm(ucsi); - kfree(ucsi->connector); - kfree(ucsi); } -EXPORT_SYMBOL_GPL(ucsi_unregister_ppm); +EXPORT_SYMBOL_GPL(ucsi_unregister); MODULE_AUTHOR("Heikki Krogerus "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index de87d0b8319d0..831c9470bdc1f 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -10,177 +10,55 @@ /* -------------------------------------------------------------------------- */ -/* Command Status and Connector Change Indication (CCI) data structure */ -struct ucsi_cci { - u8:1; /* reserved */ - u8 connector_change:7; - u8 data_length; - u16:9; /* reserved */ - u16 not_supported:1; - u16 cancel_complete:1; - u16 reset_complete:1; - u16 busy:1; - u16 ack_complete:1; - u16 error:1; - u16 cmd_complete:1; -} __packed; - -/* Default fields in CONTROL data structure */ -struct ucsi_command { - u8 cmd; - u8 length; - u64 data:48; -} __packed; - -/* ACK Command structure */ -struct ucsi_ack_cmd { - u8 cmd; - u8 length; - u8 cci_ack:1; - u8 cmd_ack:1; - u8:6; /* reserved */ -} __packed; - -/* Connector Reset Command structure */ -struct ucsi_con_rst { - u8 cmd; - u8 length; - u8 con_num:7; - u8 hard_reset:1; -} __packed; - -/* Set USB Operation Mode Command structure */ -struct ucsi_uor_cmd { - u8 cmd; - u8 length; - u16 con_num:7; - u16 role:3; -#define UCSI_UOR_ROLE_DFP BIT(0) -#define UCSI_UOR_ROLE_UFP BIT(1) -#define UCSI_UOR_ROLE_DRP BIT(2) - u16:6; /* reserved */ -} __packed; - -/* Get Alternate Modes Command structure */ -struct ucsi_altmode_cmd { - u8 cmd; - u8 length; - u8 recipient; -#define UCSI_RECIPIENT_CON 0 -#define UCSI_RECIPIENT_SOP 1 -#define UCSI_RECIPIENT_SOP_P 2 -#define UCSI_RECIPIENT_SOP_PP 3 - u8 con_num; - u8 offset; - u8 num_altmodes; -} __packed; +struct ucsi; -struct ucsi_control { - union { - u64 raw_cmd; - struct ucsi_command cmd; - struct ucsi_uor_cmd uor; - struct ucsi_ack_cmd ack; - struct ucsi_con_rst con_rst; - struct ucsi_altmode_cmd alt; - }; +/* UCSI offsets (Bytes) */ +#define UCSI_VERSION 0 +#define UCSI_CCI 4 +#define UCSI_CONTROL 8 +#define UCSI_MESSAGE_IN 16 +#define UCSI_MESSAGE_OUT 32 + +/* Command Status and Connector Change Indication (CCI) bits */ +#define UCSI_CCI_CONNECTOR(_c_) (((_c_) & GENMASK(7, 0)) >> 1) +#define UCSI_CCI_LENGTH(_c_) (((_c_) & GENMASK(15, 8)) >> 8) +#define UCSI_CCI_NOT_SUPPORTED BIT(25) +#define UCSI_CCI_CANCEL_COMPLETE BIT(26) +#define UCSI_CCI_RESET_COMPLETE BIT(27) +#define UCSI_CCI_BUSY BIT(28) +#define UCSI_CCI_ACK_COMPLETE BIT(29) +#define UCSI_CCI_ERROR BIT(30) +#define UCSI_CCI_COMMAND_COMPLETE BIT(31) + +/** + * struct ucsi_operations - UCSI I/O operations + * @read: Read operation + * @sync_write: Blocking write operation + * @async_write: Non-blocking write operation + * + * Read and write routines for UCSI interface. @sync_write must wait for the + * Command Completion Event from the PPM before returning, and @async_write must + * return immediately after sending the data to the PPM. + */ +struct ucsi_operations { + int (*read)(struct ucsi *ucsi, unsigned int offset, + void *val, size_t val_len); + int (*sync_write)(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len); + int (*async_write)(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len); }; -#define __UCSI_CMD(_ctrl_, _cmd_) \ -{ \ - (_ctrl_).raw_cmd = 0; \ - (_ctrl_).cmd.cmd = _cmd_; \ -} - -/* Helper for preparing ucsi_control for CONNECTOR_RESET command. */ -#define UCSI_CMD_CONNECTOR_RESET(_ctrl_, _con_, _hard_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_CONNECTOR_RESET) \ - (_ctrl_).con_rst.con_num = (_con_)->num; \ - (_ctrl_).con_rst.hard_reset = _hard_; \ -} - -/* Helper for preparing ucsi_control for ACK_CC_CI command. */ -#define UCSI_CMD_ACK(_ctrl_, _ack_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_ACK_CC_CI) \ - (_ctrl_).ack.cci_ack = ((_ack_) == UCSI_ACK_EVENT); \ - (_ctrl_).ack.cmd_ack = ((_ack_) == UCSI_ACK_CMD); \ -} - -/* Helper for preparing ucsi_control for SET_NOTIFY_ENABLE command. */ -#define UCSI_CMD_SET_NTFY_ENABLE(_ctrl_, _ntfys_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_SET_NOTIFICATION_ENABLE) \ - (_ctrl_).cmd.data = _ntfys_; \ -} - -/* Helper for preparing ucsi_control for GET_CAPABILITY command. */ -#define UCSI_CMD_GET_CAPABILITY(_ctrl_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_GET_CAPABILITY) \ -} - -/* Helper for preparing ucsi_control for GET_CONNECTOR_CAPABILITY command. */ -#define UCSI_CMD_GET_CONNECTOR_CAPABILITY(_ctrl_, _con_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_GET_CONNECTOR_CAPABILITY) \ - (_ctrl_).cmd.data = _con_; \ -} +struct ucsi *ucsi_create(struct device *dev, const struct ucsi_operations *ops); +void ucsi_destroy(struct ucsi *ucsi); +int ucsi_register(struct ucsi *ucsi); +void ucsi_unregister(struct ucsi *ucsi); +void *ucsi_get_drvdata(struct ucsi *ucsi); +void ucsi_set_drvdata(struct ucsi *ucsi, void *data); -/* Helper for preparing ucsi_control for GET_ALTERNATE_MODES command. */ -#define UCSI_CMD_GET_ALTERNATE_MODES(_ctrl_, _r_, _con_num_, _o_, _num_)\ -{ \ - __UCSI_CMD((_ctrl_), UCSI_GET_ALTERNATE_MODES) \ - _ctrl_.alt.recipient = (_r_); \ - _ctrl_.alt.con_num = (_con_num_); \ - _ctrl_.alt.offset = (_o_); \ - _ctrl_.alt.num_altmodes = (_num_) - 1; \ -} +void ucsi_connector_change(struct ucsi *ucsi, u8 num); -/* Helper for preparing ucsi_control for GET_CAM_SUPPORTED command. */ -#define UCSI_CMD_GET_CAM_SUPPORTED(_ctrl_, _con_) \ -{ \ - __UCSI_CMD((_ctrl_), UCSI_GET_CAM_SUPPORTED) \ - _ctrl_.cmd.data = (_con_); \ -} - -/* Helper for preparing ucsi_control for GET_CAM_SUPPORTED command. */ -#define UCSI_CMD_GET_CURRENT_CAM(_ctrl_, _con_) \ -{ \ - __UCSI_CMD((_ctrl_), UCSI_GET_CURRENT_CAM) \ - _ctrl_.cmd.data = (_con_); \ -} - -/* Helper for preparing ucsi_control for GET_CONNECTOR_STATUS command. */ -#define UCSI_CMD_GET_CONNECTOR_STATUS(_ctrl_, _con_) \ -{ \ - __UCSI_CMD(_ctrl_, UCSI_GET_CONNECTOR_STATUS) \ - (_ctrl_).cmd.data = _con_; \ -} - -#define __UCSI_ROLE(_ctrl_, _cmd_, _con_num_) \ -{ \ - __UCSI_CMD(_ctrl_, _cmd_) \ - (_ctrl_).uor.con_num = _con_num_; \ - (_ctrl_).uor.role = UCSI_UOR_ROLE_DRP; \ -} - -/* Helper for preparing ucsi_control for SET_UOR command. */ -#define UCSI_CMD_SET_UOR(_ctrl_, _con_, _role_) \ -{ \ - __UCSI_ROLE(_ctrl_, UCSI_SET_UOR, (_con_)->num) \ - (_ctrl_).uor.role |= (_role_) == TYPEC_HOST ? UCSI_UOR_ROLE_DFP : \ - UCSI_UOR_ROLE_UFP; \ -} - -/* Helper for preparing ucsi_control for SET_PDR command. */ -#define UCSI_CMD_SET_PDR(_ctrl_, _con_, _role_) \ -{ \ - __UCSI_ROLE(_ctrl_, UCSI_SET_PDR, (_con_)->num) \ - (_ctrl_).uor.role |= (_role_) == TYPEC_SOURCE ? UCSI_UOR_ROLE_DFP : \ - UCSI_UOR_ROLE_UFP; \ -} +/* -------------------------------------------------------------------------- */ /* Commands */ #define UCSI_PPM_RESET 0x01 @@ -203,24 +81,49 @@ struct ucsi_control { #define UCSI_GET_CONNECTOR_STATUS 0x12 #define UCSI_GET_ERROR_STATUS 0x13 -/* ACK_CC_CI commands */ -#define UCSI_ACK_EVENT 1 -#define UCSI_ACK_CMD 2 - -/* Bits for SET_NOTIFICATION_ENABLE command */ -#define UCSI_ENABLE_NTFY_CMD_COMPLETE BIT(0) -#define UCSI_ENABLE_NTFY_EXT_PWR_SRC_CHANGE BIT(1) -#define UCSI_ENABLE_NTFY_PWR_OPMODE_CHANGE BIT(2) -#define UCSI_ENABLE_NTFY_CAP_CHANGE BIT(5) -#define UCSI_ENABLE_NTFY_PWR_LEVEL_CHANGE BIT(6) -#define UCSI_ENABLE_NTFY_PD_RESET_COMPLETE BIT(7) -#define UCSI_ENABLE_NTFY_CAM_CHANGE BIT(8) -#define UCSI_ENABLE_NTFY_BAT_STATUS_CHANGE BIT(9) -#define UCSI_ENABLE_NTFY_PARTNER_CHANGE BIT(11) -#define UCSI_ENABLE_NTFY_PWR_DIR_CHANGE BIT(12) -#define UCSI_ENABLE_NTFY_CONNECTOR_CHANGE BIT(14) -#define UCSI_ENABLE_NTFY_ERROR BIT(15) -#define UCSI_ENABLE_NTFY_ALL 0xdbe7 +#define UCSI_CONNECTOR_NUMBER(_num_) ((u64)(_num_) << 16) + +/* CONNECTOR_RESET command bits */ +#define UCSI_CONNECTOR_RESET_HARD BIT(23) /* Deprecated in v1.1 */ + +/* ACK_CC_CI bits */ +#define UCSI_ACK_CONNECTOR_CHANGE BIT(16) +#define UCSI_ACK_COMMAND_COMPLETE BIT(17) + +/* SET_NOTIFICATION_ENABLE command bits */ +#define UCSI_ENABLE_NTFY_CMD_COMPLETE BIT(16) +#define UCSI_ENABLE_NTFY_EXT_PWR_SRC_CHANGE BIT(17) +#define UCSI_ENABLE_NTFY_PWR_OPMODE_CHANGE BIT(18) +#define UCSI_ENABLE_NTFY_CAP_CHANGE BIT(21) +#define UCSI_ENABLE_NTFY_PWR_LEVEL_CHANGE BIT(22) +#define UCSI_ENABLE_NTFY_PD_RESET_COMPLETE BIT(23) +#define UCSI_ENABLE_NTFY_CAM_CHANGE BIT(24) +#define UCSI_ENABLE_NTFY_BAT_STATUS_CHANGE BIT(25) +#define UCSI_ENABLE_NTFY_PARTNER_CHANGE BIT(27) +#define UCSI_ENABLE_NTFY_PWR_DIR_CHANGE BIT(28) +#define UCSI_ENABLE_NTFY_CONNECTOR_CHANGE BIT(30) +#define UCSI_ENABLE_NTFY_ERROR BIT(31) +#define UCSI_ENABLE_NTFY_ALL 0xdbe70000 + +/* SET_UOR command bits */ +#define UCSI_SET_UOR_ROLE(_r_) (((_r_) == TYPEC_HOST ? 1 : 2) << 23) +#define UCSI_SET_UOR_ACCEPT_ROLE_SWAPS BIT(25) + +/* SET_PDF command bits */ +#define UCSI_SET_PDR_ROLE(_r_) (((_r_) == TYPEC_SOURCE ? 1 : 2) << 23) +#define UCSI_SET_PDR_ACCEPT_ROLE_SWAPS BIT(25) + +/* GET_ALTERNATE_MODES command bits */ +#define UCSI_GET_ALTMODE_RECIPIENT(_r_) ((u64)(_r_) << 16) +#define UCSI_RECIPIENT_CON 0 +#define UCSI_RECIPIENT_SOP 1 +#define UCSI_RECIPIENT_SOP_P 2 +#define UCSI_RECIPIENT_SOP_PP 3 +#define UCSI_GET_ALTMODE_CONNECTOR_NUMBER(_r_) ((u64)(_r_) << 24) +#define UCSI_GET_ALTMODE_OFFSET(_r_) ((u64)(_r_) << 32) +#define UCSI_GET_ALTMODE_NUM_ALTMODES(_r_) ((u64)(_r_) << 40) + +/* -------------------------------------------------------------------------- */ /* Error information returned by PPM in response to GET_ERROR_STATUS command. */ #define UCSI_ERROR_UNREGONIZED_CMD BIT(0) @@ -230,6 +133,12 @@ struct ucsi_control { #define UCSI_ERROR_CC_COMMUNICATION_ERR BIT(4) #define UCSI_ERROR_DEAD_BATTERY BIT(5) #define UCSI_ERROR_CONTRACT_NEGOTIATION_FAIL BIT(6) +#define UCSI_ERROR_OVERCURRENT BIT(7) +#define UCSI_ERROR_UNDEFINED BIT(8) +#define UCSI_ERROR_PARTNER_REJECTED_SWAP BIT(9) +#define UCSI_ERROR_HARD_RESET BIT(10) +#define UCSI_ERROR_PPM_POLICY_CONFLICT BIT(11) +#define UCSI_ERROR_SWAP_REJECTED BIT(12) /* Data structure filled by PPM in response to GET_CAPABILITY command. */ struct ucsi_capability { @@ -241,8 +150,8 @@ struct ucsi_capability { #define UCSI_CAP_ATTR_POWER_AC_SUPPLY BIT(8) #define UCSI_CAP_ATTR_POWER_OTHER BIT(10) #define UCSI_CAP_ATTR_POWER_VBUS BIT(14) - u32 num_connectors:8; - u32 features:24; + u8 num_connectors; + u8 features; #define UCSI_CAP_SET_UOM BIT(0) #define UCSI_CAP_SET_PDM BIT(1) #define UCSI_CAP_ALT_MODE_DETAILS BIT(2) @@ -251,8 +160,9 @@ struct ucsi_capability { #define UCSI_CAP_CABLE_DETAILS BIT(5) #define UCSI_CAP_EXT_SUPPLY_NOTIFICATIONS BIT(6) #define UCSI_CAP_PD_RESET BIT(7) + u16 reserved_1; u8 num_alt_modes; - u8 reserved; + u8 reserved_2; u16 bc_version; u16 pd_version; u16 typec_version; @@ -269,9 +179,9 @@ struct ucsi_connector_capability { #define UCSI_CONCAP_OPMODE_USB2 BIT(5) #define UCSI_CONCAP_OPMODE_USB3 BIT(6) #define UCSI_CONCAP_OPMODE_ALT_MODE BIT(7) - u8 provider:1; - u8 consumer:1; - u8:6; /* reserved */ + u8 flags; +#define UCSI_CONCAP_FLAG_PROVIDER BIT(0) +#define UCSI_CONCAP_FLAG_CONSUMER BIT(1) } __packed; struct ucsi_altmode { @@ -283,18 +193,17 @@ struct ucsi_altmode { struct ucsi_cable_property { u16 speed_supported; u8 current_capability; - u8 vbus_in_cable:1; - u8 active_cable:1; - u8 directionality:1; - u8 plug_type:2; -#define UCSI_CABLE_PROPERTY_PLUG_TYPE_A 0 -#define UCSI_CABLE_PROPERTY_PLUG_TYPE_B 1 -#define UCSI_CABLE_PROPERTY_PLUG_TYPE_C 2 -#define UCSI_CABLE_PROPERTY_PLUG_OTHER 3 - u8 mode_support:1; - u8:2; /* reserved */ - u8 latency:4; - u8:4; /* reserved */ + u8 flags; +#define UCSI_CABLE_PROP_FLAG_VBUS_IN_CABLE BIT(0) +#define UCSI_CABLE_PROP_FLAG_ACTIVE_CABLE BIT(1) +#define UCSI_CABLE_PROP_FLAG_DIRECTIONALITY BIT(2) +#define UCSI_CABLE_PROP_FLAG_PLUG_TYPE(_f_) ((_f_) & GENMASK(3, 0)) +#define UCSI_CABLE_PROPERTY_PLUG_TYPE_A 0 +#define UCSI_CABLE_PROPERTY_PLUG_TYPE_B 1 +#define UCSI_CABLE_PROPERTY_PLUG_TYPE_C 2 +#define UCSI_CABLE_PROPERTY_PLUG_OTHER 3 +#define UCSI_CABLE_PROP_MODE_SUPPORT BIT(5) + u8 latency; } __packed; /* Data structure filled by PPM in response to GET_CONNECTOR_STATUS command. */ @@ -311,83 +220,47 @@ struct ucsi_connector_status { #define UCSI_CONSTAT_POWER_DIR_CHANGE BIT(12) #define UCSI_CONSTAT_CONNECT_CHANGE BIT(14) #define UCSI_CONSTAT_ERROR BIT(15) - u16 pwr_op_mode:3; -#define UCSI_CONSTAT_PWR_OPMODE_NONE 0 -#define UCSI_CONSTAT_PWR_OPMODE_DEFAULT 1 -#define UCSI_CONSTAT_PWR_OPMODE_BC 2 -#define UCSI_CONSTAT_PWR_OPMODE_PD 3 -#define UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5 4 -#define UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0 5 - u16 connected:1; - u16 pwr_dir:1; - u16 partner_flags:8; -#define UCSI_CONSTAT_PARTNER_FLAG_USB BIT(0) -#define UCSI_CONSTAT_PARTNER_FLAG_ALT_MODE BIT(1) - u16 partner_type:3; -#define UCSI_CONSTAT_PARTNER_TYPE_DFP 1 -#define UCSI_CONSTAT_PARTNER_TYPE_UFP 2 -#define UCSI_CONSTAT_PARTNER_TYPE_CABLE 3 /* Powered Cable */ -#define UCSI_CONSTAT_PARTNER_TYPE_CABLE_AND_UFP 4 /* Powered Cable */ -#define UCSI_CONSTAT_PARTNER_TYPE_DEBUG 5 -#define UCSI_CONSTAT_PARTNER_TYPE_AUDIO 6 + u16 flags; +#define UCSI_CONSTAT_PWR_OPMODE(_f_) ((_f_) & GENMASK(2, 0)) +#define UCSI_CONSTAT_PWR_OPMODE_NONE 0 +#define UCSI_CONSTAT_PWR_OPMODE_DEFAULT 1 +#define UCSI_CONSTAT_PWR_OPMODE_BC 2 +#define UCSI_CONSTAT_PWR_OPMODE_PD 3 +#define UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5 4 +#define UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0 5 +#define UCSI_CONSTAT_CONNECTED BIT(3) +#define UCSI_CONSTAT_PWR_DIR BIT(4) +#define UCSI_CONSTAT_PARTNER_FLAGS(_f_) (((_f_) & GENMASK(12, 5)) >> 5) +#define UCSI_CONSTAT_PARTNER_FLAG_USB 1 +#define UCSI_CONSTAT_PARTNER_FLAG_ALT_MODE 2 +#define UCSI_CONSTAT_PARTNER_TYPE(_f_) (((_f_) & GENMASK(15, 13)) >> 13) +#define UCSI_CONSTAT_PARTNER_TYPE_DFP 1 +#define UCSI_CONSTAT_PARTNER_TYPE_UFP 2 +#define UCSI_CONSTAT_PARTNER_TYPE_CABLE 3 /* Powered Cable */ +#define UCSI_CONSTAT_PARTNER_TYPE_CABLE_AND_UFP 4 /* Powered Cable */ +#define UCSI_CONSTAT_PARTNER_TYPE_DEBUG 5 +#define UCSI_CONSTAT_PARTNER_TYPE_AUDIO 6 u32 request_data_obj; - u8 bc_status:2; -#define UCSI_CONSTAT_BC_NOT_CHARGING 0 -#define UCSI_CONSTAT_BC_NOMINAL_CHARGING 1 -#define UCSI_CONSTAT_BC_SLOW_CHARGING 2 -#define UCSI_CONSTAT_BC_TRICKLE_CHARGING 3 - u8 provider_cap_limit_reason:4; -#define UCSI_CONSTAT_CAP_PWR_LOWERED 0 -#define UCSI_CONSTAT_CAP_PWR_BUDGET_LIMIT 1 - u8:2; /* reserved */ + u8 pwr_status; +#define UCSI_CONSTAT_BC_STATUS(_p_) ((_p_) & GENMASK(2, 0)) +#define UCSI_CONSTAT_BC_NOT_CHARGING 0 +#define UCSI_CONSTAT_BC_NOMINAL_CHARGING 1 +#define UCSI_CONSTAT_BC_SLOW_CHARGING 2 +#define UCSI_CONSTAT_BC_TRICKLE_CHARGING 3 +#define UCSI_CONSTAT_PROVIDER_CAP_LIMIT(_p_) (((_p_) & GENMASK(6, 3)) >> 3) +#define UCSI_CONSTAT_CAP_PWR_LOWERED 0 +#define UCSI_CONSTAT_CAP_PWR_BUDGET_LIMIT 1 } __packed; /* -------------------------------------------------------------------------- */ -struct ucsi; - -struct ucsi_data { - u16 version; - u16 reserved; - union { - u32 raw_cci; - struct ucsi_cci cci; - }; - struct ucsi_control ctrl; - u32 message_in[4]; - u32 message_out[4]; -} __packed; - -/* - * struct ucsi_ppm - Interface to UCSI Platform Policy Manager - * @data: memory location to the UCSI data structures - * @cmd: UCSI command execution routine - * @sync: Refresh UCSI mailbox (the data structures) - */ -struct ucsi_ppm { - struct ucsi_data *data; - int (*cmd)(struct ucsi_ppm *, struct ucsi_control *); - int (*sync)(struct ucsi_ppm *); -}; - -struct ucsi *ucsi_register_ppm(struct device *dev, struct ucsi_ppm *ppm); -void ucsi_unregister_ppm(struct ucsi *ucsi); -void ucsi_notify(struct ucsi *ucsi); - -/* -------------------------------------------------------------------------- */ - -enum ucsi_status { - UCSI_IDLE = 0, - UCSI_BUSY, - UCSI_ERROR, -}; - struct ucsi { + u16 version; struct device *dev; - struct ucsi_ppm *ppm; + struct driver_data *driver_data; + + const struct ucsi_operations *ops; - enum ucsi_status status; - struct completion complete; struct ucsi_capability cap; struct ucsi_connector *connector; @@ -426,7 +299,7 @@ struct ucsi_connector { struct ucsi_connector_capability cap; }; -int ucsi_send_command(struct ucsi *ucsi, struct ucsi_control *ctrl, +int ucsi_send_command(struct ucsi *ucsi, u64 command, void *retval, size_t size); void ucsi_altmode_update_active(struct ucsi_connector *con); diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c index dda8bd39c9186..3f17861700985 100644 --- a/drivers/usb/typec/ucsi/ucsi_acpi.c +++ b/drivers/usb/typec/ucsi/ucsi_acpi.c @@ -19,7 +19,9 @@ struct ucsi_acpi { struct device *dev; struct ucsi *ucsi; - struct ucsi_ppm ppm; + void __iomem *base; + struct completion complete; + unsigned long flags; guid_t guid; }; @@ -39,40 +41,82 @@ static int ucsi_acpi_dsm(struct ucsi_acpi *ua, int func) return 0; } -static int ucsi_acpi_cmd(struct ucsi_ppm *ppm, struct ucsi_control *ctrl) +static int ucsi_acpi_read(struct ucsi *ucsi, unsigned int offset, + void *val, size_t val_len) { - struct ucsi_acpi *ua = container_of(ppm, struct ucsi_acpi, ppm); + struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi); + int ret; + + ret = ucsi_acpi_dsm(ua, UCSI_DSM_FUNC_READ); + if (ret) + return ret; - ppm->data->ctrl.raw_cmd = ctrl->raw_cmd; + memcpy(val, (const void __force *)(ua->base + offset), val_len); + + return 0; +} + +static int ucsi_acpi_async_write(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len) +{ + struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi); + + memcpy((void __force *)(ua->base + offset), val, val_len); return ucsi_acpi_dsm(ua, UCSI_DSM_FUNC_WRITE); } -static int ucsi_acpi_sync(struct ucsi_ppm *ppm) +static int ucsi_acpi_sync_write(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len) { - struct ucsi_acpi *ua = container_of(ppm, struct ucsi_acpi, ppm); + struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi); + int ret; + + set_bit(COMMAND_PENDING, &ua->flags); + + ret = ucsi_acpi_async_write(ucsi, offset, val, val_len); + if (ret) + goto out_clear_bit; + + if (!wait_for_completion_timeout(&ua->complete, msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; - return ucsi_acpi_dsm(ua, UCSI_DSM_FUNC_READ); +out_clear_bit: + clear_bit(COMMAND_PENDING, &ua->flags); + + return ret; } +static const struct ucsi_operations ucsi_acpi_ops = { + .read = ucsi_acpi_read, + .sync_write = ucsi_acpi_sync_write, + .async_write = ucsi_acpi_async_write +}; + static void ucsi_acpi_notify(acpi_handle handle, u32 event, void *data) { struct ucsi_acpi *ua = data; + u32 cci; + int ret; - ucsi_notify(ua->ucsi); + ret = ucsi_acpi_read(ua->ucsi, UCSI_CCI, &cci, sizeof(cci)); + if (ret) + return; + + if (test_bit(COMMAND_PENDING, &ua->flags) && + cci & (UCSI_CCI_ACK_COMPLETE | UCSI_CCI_COMMAND_COMPLETE)) + complete(&ua->complete); + else if (UCSI_CCI_CONNECTOR(cci)) + ucsi_connector_change(ua->ucsi, UCSI_CCI_CONNECTOR(cci)); } static int ucsi_acpi_probe(struct platform_device *pdev) { - struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); struct ucsi_acpi *ua; struct resource *res; acpi_status status; int ret; - if (adev->dep_unmet) - return -EPROBE_DEFER; - ua = devm_kzalloc(&pdev->dev, sizeof(*ua), GFP_KERNEL); if (!ua) return -ENOMEM; @@ -94,35 +138,39 @@ static int ucsi_acpi_probe(struct platform_device *pdev) * it can not be requested here, and we can not use * devm_ioremap_resource(). */ - ua->ppm.data = devm_ioremap(&pdev->dev, res->start, resource_size(res)); - if (!ua->ppm.data) + ua->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!ua->base) return -ENOMEM; - if (!ua->ppm.data->version) - return -ENODEV; - ret = guid_parse(UCSI_DSM_UUID, &ua->guid); if (ret) return ret; - ua->ppm.cmd = ucsi_acpi_cmd; - ua->ppm.sync = ucsi_acpi_sync; + init_completion(&ua->complete); ua->dev = &pdev->dev; + ua->ucsi = ucsi_create(&pdev->dev, &ucsi_acpi_ops); + if (IS_ERR(ua->ucsi)) + return PTR_ERR(ua->ucsi); + + ucsi_set_drvdata(ua->ucsi, ua); + status = acpi_install_notify_handler(ACPI_HANDLE(&pdev->dev), ACPI_DEVICE_NOTIFY, ucsi_acpi_notify, ua); if (ACPI_FAILURE(status)) { dev_err(&pdev->dev, "failed to install notify handler\n"); + ucsi_destroy(ua->ucsi); return -ENODEV; } - ua->ucsi = ucsi_register_ppm(&pdev->dev, &ua->ppm); - if (IS_ERR(ua->ucsi)) { + ret = ucsi_register(ua->ucsi); + if (ret) { acpi_remove_notify_handler(ACPI_HANDLE(&pdev->dev), ACPI_DEVICE_NOTIFY, ucsi_acpi_notify); - return PTR_ERR(ua->ucsi); + ucsi_destroy(ua->ucsi); + return ret; } platform_set_drvdata(pdev, ua); @@ -134,7 +182,8 @@ static int ucsi_acpi_remove(struct platform_device *pdev) { struct ucsi_acpi *ua = platform_get_drvdata(pdev); - ucsi_unregister_ppm(ua->ucsi); + ucsi_unregister(ua->ucsi); + ucsi_destroy(ua->ucsi); acpi_remove_notify_handler(ACPI_HANDLE(&pdev->dev), ACPI_DEVICE_NOTIFY, ucsi_acpi_notify); diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c index d772fce519057..bd374cea3ba6c 100644 --- a/drivers/usb/typec/ucsi/ucsi_ccg.c +++ b/drivers/usb/typec/ucsi/ucsi_ccg.c @@ -176,8 +176,8 @@ struct ccg_resp { struct ucsi_ccg { struct device *dev; struct ucsi *ucsi; - struct ucsi_ppm ppm; struct i2c_client *client; + struct ccg_dev_info info; /* version info for boot, primary and secondary */ struct version_info version[FW2 + 1]; @@ -196,6 +196,8 @@ struct ucsi_ccg { /* fw build with vendor information */ u16 fw_build; struct work_struct pm_work; + + struct completion complete; }; static int ccg_read(struct ucsi_ccg *uc, u16 rab, u8 *data, u32 len) @@ -243,7 +245,7 @@ static int ccg_read(struct ucsi_ccg *uc, u16 rab, u8 *data, u32 len) return 0; } -static int ccg_write(struct ucsi_ccg *uc, u16 rab, u8 *data, u32 len) +static int ccg_write(struct ucsi_ccg *uc, u16 rab, const u8 *data, u32 len) { struct i2c_client *client = uc->client; unsigned char *buf; @@ -317,88 +319,85 @@ static int ucsi_ccg_init(struct ucsi_ccg *uc) return -ETIMEDOUT; } -static int ucsi_ccg_send_data(struct ucsi_ccg *uc) +static int ucsi_ccg_read(struct ucsi *ucsi, unsigned int offset, + void *val, size_t val_len) { - u8 *ppm = (u8 *)uc->ppm.data; - int status; - u16 rab; + u16 reg = CCGX_RAB_UCSI_DATA_BLOCK(offset); - rab = CCGX_RAB_UCSI_DATA_BLOCK(offsetof(struct ucsi_data, message_out)); - status = ccg_write(uc, rab, ppm + - offsetof(struct ucsi_data, message_out), - sizeof(uc->ppm.data->message_out)); - if (status < 0) - return status; - - rab = CCGX_RAB_UCSI_DATA_BLOCK(offsetof(struct ucsi_data, ctrl)); - return ccg_write(uc, rab, ppm + offsetof(struct ucsi_data, ctrl), - sizeof(uc->ppm.data->ctrl)); + return ccg_read(ucsi_get_drvdata(ucsi), reg, val, val_len); } -static int ucsi_ccg_recv_data(struct ucsi_ccg *uc) +static int ucsi_ccg_async_write(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len) { - u8 *ppm = (u8 *)uc->ppm.data; - int status; - u16 rab; + u16 reg = CCGX_RAB_UCSI_DATA_BLOCK(offset); - rab = CCGX_RAB_UCSI_DATA_BLOCK(offsetof(struct ucsi_data, cci)); - status = ccg_read(uc, rab, ppm + offsetof(struct ucsi_data, cci), - sizeof(uc->ppm.data->cci)); - if (status < 0) - return status; - - rab = CCGX_RAB_UCSI_DATA_BLOCK(offsetof(struct ucsi_data, message_in)); - return ccg_read(uc, rab, ppm + offsetof(struct ucsi_data, message_in), - sizeof(uc->ppm.data->message_in)); + return ccg_write(ucsi_get_drvdata(ucsi), reg, val, val_len); } -static int ucsi_ccg_ack_interrupt(struct ucsi_ccg *uc) +static int ucsi_ccg_sync_write(struct ucsi *ucsi, unsigned int offset, + const void *val, size_t val_len) { - int status; - unsigned char data; + struct ucsi_ccg *uc = ucsi_get_drvdata(ucsi); + int ret; - status = ccg_read(uc, CCGX_RAB_INTR_REG, &data, sizeof(data)); - if (status < 0) - return status; + mutex_lock(&uc->lock); + pm_runtime_get_sync(uc->dev); + set_bit(DEV_CMD_PENDING, &uc->flags); - return ccg_write(uc, CCGX_RAB_INTR_REG, &data, sizeof(data)); -} + ret = ucsi_ccg_async_write(ucsi, offset, val, val_len); + if (ret) + goto err_clear_bit; -static int ucsi_ccg_sync(struct ucsi_ppm *ppm) -{ - struct ucsi_ccg *uc = container_of(ppm, struct ucsi_ccg, ppm); - int status; + if (!wait_for_completion_timeout(&uc->complete, msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; - status = ucsi_ccg_recv_data(uc); - if (status < 0) - return status; +err_clear_bit: + clear_bit(DEV_CMD_PENDING, &uc->flags); + pm_runtime_put_sync(uc->dev); + mutex_unlock(&uc->lock); - /* ack interrupt to allow next command to run */ - return ucsi_ccg_ack_interrupt(uc); + return ret; } -static int ucsi_ccg_cmd(struct ucsi_ppm *ppm, struct ucsi_control *ctrl) -{ - struct ucsi_ccg *uc = container_of(ppm, struct ucsi_ccg, ppm); - - ppm->data->ctrl.raw_cmd = ctrl->raw_cmd; - return ucsi_ccg_send_data(uc); -} +static const struct ucsi_operations ucsi_ccg_ops = { + .read = ucsi_ccg_read, + .sync_write = ucsi_ccg_sync_write, + .async_write = ucsi_ccg_async_write +}; static irqreturn_t ccg_irq_handler(int irq, void *data) { + u16 reg = CCGX_RAB_UCSI_DATA_BLOCK(UCSI_CCI); struct ucsi_ccg *uc = data; + u8 intr_reg; + u32 cci; + int ret; + + ret = ccg_read(uc, CCGX_RAB_INTR_REG, &intr_reg, sizeof(intr_reg)); + if (ret) + return ret; + + ret = ccg_read(uc, reg, (void *)&cci, sizeof(cci)); + if (ret) + goto err_clear_irq; + + if (UCSI_CCI_CONNECTOR(cci)) + ucsi_connector_change(uc->ucsi, UCSI_CCI_CONNECTOR(cci)); + + if (test_bit(DEV_CMD_PENDING, &uc->flags) && + cci & (UCSI_CCI_ACK_COMPLETE | UCSI_CCI_COMMAND_COMPLETE)) + complete(&uc->complete); - ucsi_notify(uc->ucsi); +err_clear_irq: + ccg_write(uc, CCGX_RAB_INTR_REG, &intr_reg, sizeof(intr_reg)); return IRQ_HANDLED; } static void ccg_pm_workaround_work(struct work_struct *pm_work) { - struct ucsi_ccg *uc = container_of(pm_work, struct ucsi_ccg, pm_work); - - ucsi_notify(uc->ucsi); + ccg_irq_handler(0, container_of(pm_work, struct ucsi_ccg, pm_work)); } static int get_fw_info(struct ucsi_ccg *uc) @@ -1027,12 +1026,13 @@ static int ccg_restart(struct ucsi_ccg *uc) return status; } - uc->ucsi = ucsi_register_ppm(dev, &uc->ppm); - if (IS_ERR(uc->ucsi)) { - dev_err(uc->dev, "ucsi_register_ppm failed\n"); - return PTR_ERR(uc->ucsi); + status = ucsi_register(uc->ucsi); + if (status) { + dev_err(uc->dev, "failed to register the interface\n"); + return status; } + pm_runtime_enable(uc->dev); return 0; } @@ -1047,7 +1047,8 @@ static void ccg_update_firmware(struct work_struct *work) return; if (flash_mode != FLASH_NOT_NEEDED) { - ucsi_unregister_ppm(uc->ucsi); + ucsi_unregister(uc->ucsi); + pm_runtime_disable(uc->dev); free_irq(uc->irq, uc); ccg_fw_update(uc, flash_mode); @@ -1091,21 +1092,15 @@ static int ucsi_ccg_probe(struct i2c_client *client, struct device *dev = &client->dev; struct ucsi_ccg *uc; int status; - u16 rab; uc = devm_kzalloc(dev, sizeof(*uc), GFP_KERNEL); if (!uc) return -ENOMEM; - uc->ppm.data = devm_kzalloc(dev, sizeof(struct ucsi_data), GFP_KERNEL); - if (!uc->ppm.data) - return -ENOMEM; - - uc->ppm.cmd = ucsi_ccg_cmd; - uc->ppm.sync = ucsi_ccg_sync; uc->dev = dev; uc->client = client; mutex_init(&uc->lock); + init_completion(&uc->complete); INIT_WORK(&uc->work, ccg_update_firmware); INIT_WORK(&uc->pm_work, ccg_pm_workaround_work); @@ -1133,30 +1128,25 @@ static int ucsi_ccg_probe(struct i2c_client *client, if (uc->info.mode & CCG_DEVINFO_PDPORTS_MASK) uc->port_num++; + uc->ucsi = ucsi_create(dev, &ucsi_ccg_ops); + if (IS_ERR(uc->ucsi)) + return PTR_ERR(uc->ucsi); + + ucsi_set_drvdata(uc->ucsi, uc); + status = request_threaded_irq(client->irq, NULL, ccg_irq_handler, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, dev_name(dev), uc); if (status < 0) { dev_err(uc->dev, "request_threaded_irq failed - %d\n", status); - return status; + goto out_ucsi_destroy; } uc->irq = client->irq; - uc->ucsi = ucsi_register_ppm(dev, &uc->ppm); - if (IS_ERR(uc->ucsi)) { - dev_err(uc->dev, "ucsi_register_ppm failed\n"); - return PTR_ERR(uc->ucsi); - } - - rab = CCGX_RAB_UCSI_DATA_BLOCK(offsetof(struct ucsi_data, version)); - status = ccg_read(uc, rab, (u8 *)(uc->ppm.data) + - offsetof(struct ucsi_data, version), - sizeof(uc->ppm.data->version)); - if (status < 0) { - ucsi_unregister_ppm(uc->ucsi); - return status; - } + status = ucsi_register(uc->ucsi); + if (status) + goto out_free_irq; i2c_set_clientdata(client, uc); @@ -1167,6 +1157,13 @@ static int ucsi_ccg_probe(struct i2c_client *client, pm_runtime_idle(uc->dev); return 0; + +out_free_irq: + free_irq(uc->irq, uc); +out_ucsi_destroy: + ucsi_destroy(uc->ucsi); + + return status; } static int ucsi_ccg_remove(struct i2c_client *client) @@ -1175,8 +1172,9 @@ static int ucsi_ccg_remove(struct i2c_client *client) cancel_work_sync(&uc->pm_work); cancel_work_sync(&uc->work); - ucsi_unregister_ppm(uc->ucsi); pm_runtime_disable(uc->dev); + ucsi_unregister(uc->ucsi); + ucsi_destroy(uc->ucsi); free_irq(uc->irq, uc); return 0; diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 7df4ecabc78a2..0f52723a11bd1 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -168,6 +168,23 @@ struct typec_partner_desc { struct usb_pd_identity *identity; }; +/** + * struct typec_operations - USB Type-C Port Operations + * @try_role: Set data role preference for DRP port + * @dr_set: Set Data Role + * @pr_set: Set Power Role + * @vconn_set: Source VCONN + * @port_type_set: Set port type + */ +struct typec_operations { + int (*try_role)(struct typec_port *port, int role); + int (*dr_set)(struct typec_port *port, enum typec_data_role role); + int (*pr_set)(struct typec_port *port, enum typec_role role); + int (*vconn_set)(struct typec_port *port, enum typec_role role); + int (*port_type_set)(struct typec_port *port, + enum typec_port_type type); +}; + /* * struct typec_capability - USB Type-C Port Capabilities * @type: Supported power role of the port @@ -179,11 +196,8 @@ struct typec_partner_desc { * @sw: Cable plug orientation switch * @mux: Multiplexer switch for Alternate/Accessory Modes * @fwnode: Optional fwnode of the port - * @try_role: Set data role preference for DRP port - * @dr_set: Set Data Role - * @pr_set: Set Power Role - * @vconn_set: Set VCONN Role - * @port_type_set: Set port type + * @driver_data: Private pointer for driver specific info + * @ops: Port operations vector * * Static capabilities of a single USB Type-C port. */ @@ -195,21 +209,10 @@ struct typec_capability { int prefer_role; enum typec_accessory accessory[TYPEC_MAX_ACCESSORY]; - struct typec_switch *sw; - struct typec_mux *mux; struct fwnode_handle *fwnode; + void *driver_data; - int (*try_role)(const struct typec_capability *, - int role); - - int (*dr_set)(const struct typec_capability *, - enum typec_data_role); - int (*pr_set)(const struct typec_capability *, - enum typec_role); - int (*vconn_set)(const struct typec_capability *, - enum typec_role); - int (*port_type_set)(const struct typec_capability *, - enum typec_port_type); + const struct typec_operations *ops; }; /* Specific to try_role(). Indicates the user want's to clear the preference. */ @@ -241,6 +244,8 @@ int typec_set_orientation(struct typec_port *port, enum typec_orientation typec_get_orientation(struct typec_port *port); int typec_set_mode(struct typec_port *port, int mode); +void *typec_get_drvdata(struct typec_port *port); + int typec_find_port_power_role(const char *name); int typec_find_power_role(const char *name); int typec_find_port_data_role(const char *name); From 9ca81d19ed92c1d95cd17b39e1c2a78373c96439 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:06:02 -0600 Subject: [PATCH 009/135] backports: brcm80211: from: linux.git Reference: v5.4.18 Signed-off-by: Robert Nelson --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 8 ------- .../broadcom/brcm80211/brcmfmac/feature.c | 3 +-- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 7 ++----- .../broadcom/brcm80211/brcmfmac/p2p.c | 3 +-- .../broadcom/brcm80211/brcmfmac/sdio.c | 21 +++++++------------ .../broadcom/brcm80211/brcmfmac/usb.c | 1 - 7 files changed, 12 insertions(+), 33 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 4ca50353538ef..e3ebb7abbdaed 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -82,8 +82,6 @@ #define BRCMF_ND_INFO_TIMEOUT msecs_to_jiffies(2000) -#define BRCMF_PS_MAX_TIMEOUT_MS 2000 - #define BRCMF_ASSOC_PARAMS_FIXED_SIZE \ (sizeof(struct brcmf_assoc_params_le) - sizeof(u16)) @@ -2791,12 +2789,6 @@ brcmf_cfg80211_set_power_mgmt(struct wiphy *wiphy, struct net_device *ndev, else bphy_err(drvr, "error (%d)\n", err); } - - err = brcmf_fil_iovar_int_set(ifp, "pm2_sleep_ret", - min_t(u32, timeout, BRCMF_PS_MAX_TIMEOUT_MS)); - if (err) - bphy_err(drvr, "Unable to set pm timeout, (%d)\n", err); - done: brcmf_dbg(TRACE, "Exit\n"); return err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 545015610cf82..2c3526aeca6fc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -283,14 +283,13 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) if (!err) ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC); - brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); - if (drvr->settings->feature_disable) { brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n", ifp->drvr->feat_flags, drvr->settings->feature_disable); ifp->drvr->feat_flags &= ~drvr->settings->feature_disable; } + brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); brcmf_feat_firmware_overrides(drvr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index ce18433aaefb5..37c512036e0e3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -19,7 +19,7 @@ #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 #define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ -#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 +#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0002 #define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ #define BRCMF_STA_WME 0x00000002 /* WMM association */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 3d36b6ee158bb..2bd892df83cc5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -643,7 +643,6 @@ static inline int brcmf_fws_hanger_poppkt(struct brcmf_fws_hanger *h, static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, int ifidx) { - struct brcmf_fws_hanger_item *hi; bool (*matchfn)(struct sk_buff *, void *) = NULL; struct sk_buff *skb; int prec; @@ -655,9 +654,6 @@ static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); while (skb) { hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); - hi = &fws->hanger.items[hslot]; - WARN_ON(skb != hi->pkt); - hi->state = BRCMF_FWS_HANGER_ITEM_STATE_FREE; brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, true); brcmu_pkt_buf_free_skb(skb); @@ -2149,7 +2145,8 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) brcmf_fws_enq(fws, BRCMF_FWS_SKBSTATE_DELAYED, fifo, skb); brcmf_fws_schedule_deq(fws); } else { - bphy_err(drvr, "no hanger slot available\n"); + bphy_err(drvr, "drop skb: no hanger slot\n"); + brcmf_txfinalize(ifp, skb, false); rc = -ENOMEM; } brcmf_fws_unlock(fws); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 1f5deea5a288e..7ba9f6a686459 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -2092,8 +2092,7 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, /* firmware requires unique mac address for p2pdev interface */ if (addr && ether_addr_equal(addr, pri_ifp->mac_addr)) { bphy_err(drvr, "discovery vif must be different from primary interface\n"); - err = -EINVAL; - goto fail; + return ERR_PTR(-EINVAL); } brcmf_p2p_generate_bss_mac(p2p, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 38e6809f16c75..264ad63232f87 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -1934,10 +1934,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) if (brcmf_sdio_hdparse(bus, bus->rxhdr, &rd_new, BRCMF_SDIO_FT_NORMAL)) { rd->len = 0; - brcmf_sdio_rxfail(bus, true, true); - sdio_release_host(bus->sdiodev->func1); brcmu_pkt_buf_free_skb(pkt); - continue; } bus->sdcnt.rx_readahead_cnt++; if (rd->len != roundup(rd_new.len, 16)) { @@ -3685,11 +3682,7 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) if (bus->idlecount > bus->idletime) { brcmf_dbg(SDIO, "idle\n"); sdio_claim_host(bus->sdiodev->func1); -#ifdef DEBUG - if (!BRCMF_FWCON_ON() || - bus->console_interval == 0) -#endif - brcmf_sdio_wd_timer(bus, false); + brcmf_sdio_wd_timer(bus, false); bus->idlecount = 0; brcmf_sdio_bus_sleep(bus, true, false); sdio_release_host(bus->sdiodev->func1); @@ -4232,12 +4225,6 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, } if (err == 0) { - /* Assign bus interface call back */ - sdiod->bus_if->dev = sdiod->dev; - sdiod->bus_if->ops = &brcmf_sdio_bus_ops; - sdiod->bus_if->chip = bus->ci->chip; - sdiod->bus_if->chiprev = bus->ci->chiprev; - /* Allow full data communication using DPC from now on. */ brcmf_sdiod_change_state(bus->sdiodev, BRCMF_SDIOD_DATA); @@ -4254,6 +4241,12 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, sdio_release_host(sdiod->func1); + /* Assign bus interface call back */ + sdiod->bus_if->dev = sdiod->dev; + sdiod->bus_if->ops = &brcmf_sdio_bus_ops; + sdiod->bus_if->chip = bus->ci->chip; + sdiod->bus_if->chiprev = bus->ci->chiprev; + err = brcmf_alloc(sdiod->dev, sdiod->settings); if (err) { brcmf_err("brcmf_alloc failed\n"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 575ed19e91951..7cdfde9b3dea9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -430,7 +430,6 @@ brcmf_usbdev_qinit(struct list_head *q, int qsize) usb_free_urb(req->urb); list_del(q->next); } - kfree(reqs); return NULL; } From 172344a47f5f1ff0984d59c9d1d2a685c015dce1 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 20 Jun 2017 05:11:24 -0500 Subject: [PATCH 010/135] brcmfmac: set F2 blocksize and watermark for 4373 Set F2 blocksize to 256 bytes for 4373. It fixes DMA error while having UDP bi-directional traffic. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index fc12598b2dd3f..b64600273c685 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -43,6 +43,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 +#define SDIO_4373_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -903,6 +904,7 @@ static void brcmf_sdiod_host_fixup(struct mmc_host *host) static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; + unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE; sdio_claim_host(sdiodev->func1); @@ -912,11 +914,17 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) sdio_release_host(sdiodev->func1); goto out; } - ret = sdio_set_block_size(sdiodev->func2, SDIO_FUNC2_BLOCKSIZE); + + if (sdiodev->func2->device == SDIO_DEVICE_ID_CYPRESS_4373) + f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; + + ret = sdio_set_block_size(sdiodev->func2, f2_blksz); if (ret) { brcmf_err("Failed to set F2 blocksize\n"); sdio_release_host(sdiodev->func1); goto out; + } else { + brcmf_dbg(SDIO, "set F2 blocksize to %d\n", f2_blksz); } /* increase F2 timeout */ From d2f6e05b3e528f62a34d346efafca014648e6207 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 15 Aug 2017 00:25:14 -0500 Subject: [PATCH 011/135] non-upstream: add sg parameters dts parsing broken_sg_support, sd_head_align, and sd_sgentry_align are used in brcmfmac code but not configurable in dts file. Add the parsing logic. Now they can be configured like below in dts: brcm,broken_sg_support; brcm,sd_head_align = /bits/ 16 <4>; brcm,sd_sgentry_align = /bits/ 16 <4>; Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/of.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index b886b56a5e5af..e4f3493a5d6ee 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -20,7 +20,8 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, struct property *prop; int irq; u32 irqf; - u32 val; + u32 val32; + u16 val16; /* Set board-type to the first string of the machine compatible prop */ root = of_find_node_by_path("/"); @@ -34,8 +35,15 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, !of_device_is_compatible(np, "brcm,bcm4329-fmac")) return; - if (of_property_read_u32(np, "brcm,drive-strength", &val) == 0) - sdio->drive_strength = val; + if (of_property_read_u32(np, "brcm,drive-strength", &val32) == 0) + sdio->drive_strength = val32; + + sdio->broken_sg_support = of_property_read_bool(np, + "brcm,broken_sg_support"); + if (of_property_read_u16(np, "brcm,sd_head_align", &val16) == 0) + sdio->sd_head_align = val16; + if (of_property_read_u16(np, "brcm,sd_sgentry_align", &val16) == 0) + sdio->sd_sgentry_align = val16; /* make sure there are interrupts defined in the node */ if (!of_find_property(np, "interrupts", NULL)) From b91e27a48546888e1503cfb455c17160eaaa3c16 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 11 Sep 2017 02:48:57 -0500 Subject: [PATCH 012/135] brcmfmac: set apsta to 0 when AP starts on primary interface APSTA can work on two band concurrently with using VSDB(Virtual Simultaneous Dual-Band) or RSDB(Real Simultaneous Dual-Band) features. In this case, we have to keep apsta is 1 in firmware side. If we start wpa_supplicant on wlan0 and then start hostapd on wlan1, the apsta will be set to 0, and data will be stall on wlan0(station). Because that, we only set apsta to 0 when AP start on primary interface. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index e3ebb7abbdaed..7d2855af473a7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4584,9 +4584,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, } } - if ((dev_role == NL80211_IFTYPE_AP) && - ((ifp->ifidx == 0) || - !brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + if (dev_role == NL80211_IFTYPE_AP && ifp->ifidx == 0) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); if (err < 0) { bphy_err(drvr, "BRCMF_C_DOWN error %d\n", From d353c8593d4140cfe6ebeaf63c55c334f80668ef Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 31 Jan 2018 04:07:12 -0600 Subject: [PATCH 013/135] brcmfmac: support AP isolation Hostap daemon has a parameter "ap_isolate which is used to prevent low-level bridging of frames between associated stations in the BSS. For driver side, we add cfg80211 ops method change_bss to support setting AP isolation from user space. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 7d2855af473a7..17a3df68df51d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5233,6 +5233,26 @@ static int brcmf_cfg80211_del_pmk(struct wiphy *wiphy, struct net_device *dev, return brcmf_set_pmk(ifp, NULL, 0); } +static int +brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev, + struct bss_parameters *params) +{ + struct brcmf_if *ifp; + int ret = 0; + u32 ap_isolate; + + brcmf_dbg(TRACE, "Enter\n"); + ifp = netdev_priv(dev); + if (params->ap_isolate >= 0) { + ap_isolate = (u32)params->ap_isolate; + ret = brcmf_fil_iovar_int_set(ifp, "ap_isolate", ap_isolate); + if (ret < 0) + brcmf_err("ap_isolate iovar failed: ret=%d\n", ret); + } + + return ret; +} + static struct cfg80211_ops brcmf_cfg80211_ops = { .add_virtual_intf = brcmf_cfg80211_add_iface, .del_virtual_intf = brcmf_cfg80211_del_iface, @@ -5278,6 +5298,7 @@ static struct cfg80211_ops brcmf_cfg80211_ops = { .update_connect_params = brcmf_cfg80211_update_conn_params, .set_pmk = brcmf_cfg80211_set_pmk, .del_pmk = brcmf_cfg80211_del_pmk, + .change_bss = brcmf_cfg80211_change_bss, }; struct cfg80211_ops *brcmf_cfg80211_get_ops(struct brcmf_mp_device *settings) From b3965b34a56460d0a2ccc1b0c38bfae9da2d49cd Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 2 Mar 2018 00:45:32 -0600 Subject: [PATCH 014/135] brcmfmac: make firmware eap_restrict a module parameter When eap_restrict is enabled, firmware will toss non-802.1x frames from tx/rx data path if station not yet authorized. Internal firmware eap_restrict is disabled by default. This patch makes it possible to enable firmware eap_restrict by specifying eap_restrict=1 as module parameter. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 9 +++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 5 +++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.h | 2 ++ 3 files changed, 16 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 17a3df68df51d..9d2a382a8835e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6758,6 +6758,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) struct wireless_dev *wdev; struct brcmf_if *ifp; s32 power_mode; + s32 eap_restrict; s32 err = 0; if (cfg->dongle_up) @@ -6782,6 +6783,14 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) err = brcmf_dongle_roam(ifp); if (err) goto default_conf_out; + + eap_restrict = ifp->drvr->settings->eap_restrict; + if (eap_restrict) { + err = brcmf_fil_iovar_int_set(ifp, "eap_restrict", + eap_restrict); + if (err) + brcmf_info("eap_restrict error (%d)\n", err); + } err = brcmf_cfg80211_change_iface(wdev->wiphy, ndev, wdev->iftype, NULL); if (err) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index dec25e4156199..cda6bef2749c5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -67,6 +67,10 @@ static int brcmf_iapp_enable; module_param_named(iapp, brcmf_iapp_enable, int, 0); MODULE_PARM_DESC(iapp, "Enable partial support for the obsoleted Inter-Access Point Protocol"); +static int brcmf_eap_restrict; +module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); +MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -413,6 +417,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; settings->iapp = !!brcmf_iapp_enable; + settings->eap_restrict = !!brcmf_eap_restrict; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 144cf4570bc3e..059f09c47afe7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -37,6 +37,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @feature_disable: Feature_disable bitmask. * @fcmode: FWS flow control. * @roamoff: Firmware roaming off? + * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -47,6 +48,7 @@ struct brcmf_mp_device { int fcmode; bool roamoff; bool iapp; + bool eap_restrict; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; const char *board_type; From 7e779a9457ad93f82259791ac7216cebc65b853a Mon Sep 17 00:00:00 2001 From: Double Lo Date: Tue, 6 Feb 2018 05:07:05 -0600 Subject: [PATCH 015/135] non-upstream: support wake on ping packet Host driver needs to provide a dummy wowlan filter for kernel and provided the the well configured wowlan stack. So the system will keep driver in connected state in suspend mode and can be wake up by ping packet. Enable unicast packet filter before system suspend and disable it after resume. Signed-off-by: Double Lo Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 31 ++- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 7 + .../broadcom/brcm80211/brcmfmac/common.c | 18 +- .../broadcom/brcm80211/brcmfmac/core.c | 176 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/core.h | 7 +- .../broadcom/brcm80211/brcmfmac/feature.c | 1 - .../broadcom/brcm80211/brcmfmac/fwil.h | 2 + .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ++ 8 files changed, 249 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 9d2a382a8835e..3815f14f23574 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3662,6 +3662,10 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) brcmf_notify_sched_scan_results); cfg->wowl.nd_enabled = false; } + + /* disable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, false); + } return 0; } @@ -3720,6 +3724,9 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg, brcmf_fil_iovar_int_set(ifp, "wowl_activate", 1); brcmf_bus_wowl_config(cfg->pub->bus_if, true); cfg->wowl.active = true; + + /* enable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, true); } static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, @@ -3766,7 +3773,7 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, brcmf_set_mpc(ifp, 1); } else { - /* Configure WOWL paramaters */ + /* Configure WOWL parameters */ brcmf_configure_wowl(cfg, ifp, wowl); } @@ -6598,6 +6605,7 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_pub *drvr = cfg->pub; struct wiphy_wowlan_support *wowl; + struct cfg80211_wowlan *brcmf_wowlan_config = NULL; wowl = kmemdup(&brcmf_wowlan_support, sizeof(brcmf_wowlan_support), GFP_KERNEL); @@ -6620,6 +6628,27 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) } wiphy->wowlan = wowl; + + /* wowlan_config structure report for kernels */ + brcmf_wowlan_config = kzalloc(sizeof(*brcmf_wowlan_config), + GFP_KERNEL); + if (brcmf_wowlan_config) { + brcmf_wowlan_config->any = false; + brcmf_wowlan_config->disconnect = true; + brcmf_wowlan_config->eap_identity_req = true; + brcmf_wowlan_config->four_way_handshake = true; + brcmf_wowlan_config->rfkill_release = false; + brcmf_wowlan_config->patterns = NULL; + brcmf_wowlan_config->n_patterns = 0; + brcmf_wowlan_config->tcp = NULL; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK)) + brcmf_wowlan_config->gtk_rekey_failure = true; + else + brcmf_wowlan_config->gtk_rekey_failure = false; + } else { + brcmf_err("Can not allocate memory for brcm_wowlan_config\n"); + } + wiphy->wowlan_config = brcmf_wowlan_config; #endif } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 14d5bbad1db13..2e95ad40465be 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -75,6 +75,13 @@ #define BRCMF_VIF_EVENT_TIMEOUT msecs_to_jiffies(1500) +/* cfg80211 wowlan definitions */ +#define WL_WOWLAN_MAX_PATTERNS 8 +#define WL_WOWLAN_MIN_PATTERN_LEN 1 +#define WL_WOWLAN_MAX_PATTERN_LEN 255 +#define WL_WOWLAN_PKT_FILTER_ID_FIRST 201 +#define WL_WOWLAN_PKT_FILTER_ID_LAST (WL_WOWLAN_PKT_FILTER_ID_FIRST + \ + WL_WOWLAN_MAX_PATTERNS - 1) /** * enum brcmf_scan_status - scan engine status * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index cda6bef2749c5..3926d1ef64f98 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -206,7 +206,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) char *ptr; s32 err; - /* retreive mac address */ + /* retrieve mac addresses */ err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr, sizeof(ifp->mac_addr)); if (err < 0) { @@ -340,6 +340,18 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) /* Enable tx beamforming, errors can be ignored (not supported) */ (void)brcmf_fil_iovar_int_set(ifp, "txbf", 1); + + /* add unicast packet filter */ + err = brcmf_pktfilter_add_remove(ifp->ndev, + BRCMF_UNICAST_FILTER_NUM, true); + if (err == -BRCMF_FW_UNSUPPORTED) { + /* FW not support can be ignored */ + err = 0; + goto done; + } else if (err) { + bphy_err(drvr, "Add unicast filter error (%d)\n", err); + } + done: return err; } @@ -411,7 +423,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, if (!settings) return NULL; - /* start by using the module paramaters */ + /* start by using the module parameters */ settings->p2p_enable = !!brcmf_p2p_enable; settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; @@ -497,7 +509,7 @@ static int __init brcmfmac_module_init(void) if (err == -ENODEV) brcmf_dbg(INFO, "No platform data available.\n"); - /* Initialize global module paramaters */ + /* Initialize global module parameters */ brcmf_mp_attach(); /* Continue the initialization by registering the different busses */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 85cf96461ddeb..1b473f02c84ea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1475,3 +1475,179 @@ void __exit brcmf_core_exit(void) #endif } +int +brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, bool add) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pkt_filter_le *pkt_filter; + int filter_fixed_len = offsetof(struct brcmf_pkt_filter_le, u); + int pattern_fixed_len = offsetof(struct brcmf_pkt_filter_pattern_le, + mask_and_pattern); + u16 mask_and_pattern[MAX_PKTFILTER_PATTERN_SIZE]; + int buflen = 0; + int ret = 0; + + brcmf_dbg(INFO, "%s packet filter number %d\n", + (add ? "add" : "remove"), filter_num); + + pkt_filter = kzalloc(sizeof(*pkt_filter) + + (MAX_PKTFILTER_PATTERN_SIZE * 2), GFP_ATOMIC); + if (!pkt_filter) + return -ENOMEM; + + switch (filter_num) { + case BRCMF_UNICAST_FILTER_NUM: + pkt_filter->id = 100; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 1; + mask_and_pattern[0] = 0x0001; + break; + case BRCMF_BROADCAST_FILTER_NUM: + //filter_pattern = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF"; + pkt_filter->id = 101; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0xFFFF; + mask_and_pattern[4] = 0xFFFF; + mask_and_pattern[5] = 0xFFFF; + break; + case BRCMF_MULTICAST4_FILTER_NUM: + //filter_pattern = "102 0 0 0 0xFFFFFF 0x01005E"; + pkt_filter->id = 102; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 3; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x01FF; + mask_and_pattern[2] = 0x5E00; + break; + case BRCMF_MULTICAST6_FILTER_NUM: + //filter_pattern = "103 0 0 0 0xFFFF 0x3333"; + pkt_filter->id = 103; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x3333; + break; + case BRCMF_MDNS_FILTER_NUM: + //filter_pattern = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB"; + pkt_filter->id = 104; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0001; + mask_and_pattern[4] = 0x005E; + mask_and_pattern[5] = 0xFB00; + break; + case BRCMF_ARP_FILTER_NUM: + //filter_pattern = "105 0 0 12 0xFFFF 0x0806"; + pkt_filter->id = 105; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 12; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x0608; + break; + case BRCMF_BROADCAST_ARP_FILTER_NUM: + //filter_pattern = "106 0 0 0 + //0xFFFFFFFFFFFF0000000000000806 + //0xFFFFFFFFFFFF0000000000000806"; + pkt_filter->id = 106; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 14; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0000; + mask_and_pattern[4] = 0x0000; + mask_and_pattern[5] = 0x0000; + mask_and_pattern[6] = 0x0608; + mask_and_pattern[7] = 0xFFFF; + mask_and_pattern[8] = 0xFFFF; + mask_and_pattern[9] = 0xFFFF; + mask_and_pattern[10] = 0x0000; + mask_and_pattern[11] = 0x0000; + mask_and_pattern[12] = 0x0000; + mask_and_pattern[13] = 0x0608; + break; + default: + ret = -EINVAL; + goto failed; + } + memcpy(pkt_filter->u.pattern.mask_and_pattern, mask_and_pattern, + pkt_filter->u.pattern.size_bytes * 2); + buflen = filter_fixed_len + pattern_fixed_len + + pkt_filter->u.pattern.size_bytes * 2; + + if (add) { + /* Add filter */ + ifp->fwil_fwerr = true; + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_add", + pkt_filter, buflen); + ifp->fwil_fwerr = false; + if (ret) + goto failed; + drvr->pkt_filter[filter_num].id = pkt_filter->id; + drvr->pkt_filter[filter_num].enable = 0; + + } else { + /* Delete filter */ + ret = brcmf_fil_iovar_int_set(ifp, "pkt_filter_delete", + pkt_filter->id); + if (ret == -ENOENT) + ret = 0; + if (ret) + goto failed; + + drvr->pkt_filter[filter_num].id = 0; + drvr->pkt_filter[filter_num].enable = 0; + } +failed: + if (ret) + brcmf_err("%s packet filter failed, ret=%d\n", + (add ? "add" : "remove"), ret); + + kfree(pkt_filter); + return ret; +} + +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + int ret = 0; + int idx = 0; + + for (idx = 0; idx < MAX_PKT_FILTER_COUNT; ++idx) { + if (drvr->pkt_filter[idx].id != 0) { + drvr->pkt_filter[idx].enable = enable; + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_enable", + &drvr->pkt_filter[idx], + sizeof(struct brcmf_pkt_filter_enable_le)); + if (ret) { + brcmf_err("%s packet filter id(%d) failed, ret=%d\n", + (enable ? "enable" : "disable"), + drvr->pkt_filter[idx].id, ret); + } + } + } + return ret; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 6699637d3bf8c..6def862f60084 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -12,6 +12,7 @@ #include #include "fweh.h" +#include "fwil_types.h" #define TOE_TX_CSUM_OL 0x00000001 #define TOE_RX_CSUM_OL 0x00000002 @@ -136,6 +137,8 @@ struct brcmf_pub { struct work_struct bus_reset; u8 clmver[BRCMF_DCMD_SMLEN]; + struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; + }; /* forward declarations */ @@ -213,5 +216,7 @@ void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_setcarrier(struct brcmf_if *ifp, bool on); int __init brcmf_core_init(void); void __exit brcmf_core_exit(void); - +int brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, + bool add); +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable); #endif /* BRCMFMAC_CORE_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 2c3526aeca6fc..565db6c11bcd0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -16,7 +16,6 @@ #include "feature.h" #include "common.h" -#define BRCMF_FW_UNSUPPORTED 23 /* * expand feature list to array of feature strings. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h index 0ff6f5212a94d..f2e00a3c19e02 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h @@ -77,6 +77,8 @@ #define BRCMF_C_SET_VAR 263 #define BRCMF_C_SET_WSEC_PMK 268 +#define BRCMF_FW_UNSUPPORTED 23 + s32 brcmf_fil_cmd_data_set(struct brcmf_if *ifp, u32 cmd, void *data, u32 len); s32 brcmf_fil_cmd_data_get(struct brcmf_if *ifp, u32 cmd, void *data, u32 len); s32 brcmf_fil_cmd_int_set(struct brcmf_if *ifp, u32 cmd, u32 data); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 37c512036e0e3..f3f7b1514167a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -136,6 +136,19 @@ #define BRCMF_WOWL_MAXPATTERNS 8 #define BRCMF_WOWL_MAXPATTERNSIZE 128 +enum { + BRCMF_UNICAST_FILTER_NUM = 0, + BRCMF_BROADCAST_FILTER_NUM, + BRCMF_MULTICAST4_FILTER_NUM, + BRCMF_MULTICAST6_FILTER_NUM, + BRCMF_MDNS_FILTER_NUM, + BRCMF_ARP_FILTER_NUM, + BRCMF_BROADCAST_ARP_FILTER_NUM, + MAX_PKT_FILTER_COUNT +}; + +#define MAX_PKTFILTER_PATTERN_SIZE 16 + #define BRCMF_COUNTRY_BUF_SZ 4 #define BRCMF_ANT_MAX 4 From 8e2d8da2a34572a0f73ebfc3fc41412bcff18d5b Mon Sep 17 00:00:00 2001 From: Double Lo Date: Tue, 13 Mar 2018 03:22:20 -0500 Subject: [PATCH 016/135] non-upstream: remove WOWL configuration in disconnect state Set wowl configuration in disconnect state is redundant. Remove it to fix no scan result issue after resume. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 3815f14f23574..d34448de78082 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3753,7 +3753,8 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status)) brcmf_abort_scanning(cfg); - if (wowl == NULL) { + if (!wowl || !test_bit(BRCMF_VIF_STATUS_CONNECTED, + &ifp->vif->sme_state)) { brcmf_bus_wowl_config(cfg->pub->bus_if, false); list_for_each_entry(vif, &cfg->vif_list, list) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) From 76398f3c1d384b2bbb86cee98d98e4f0e54a8bf9 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Sun, 25 Mar 2018 21:53:42 -0500 Subject: [PATCH 017/135] brcmfmac: make setting SDIO workqueue WQ_HIGHPRI a module parameter With setting sdio_wq_highpri=1 in module parameters, tasks submitted to SDIO workqueue will put at the head of the queue and run immediately. This parameter is for getting higher TX/RX throughput with SDIO bus. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/common.c | 5 +++++ .../broadcom/brcm80211/brcmfmac/common.h | 2 ++ .../broadcom/brcm80211/brcmfmac/sdio.c | 22 ++++++++++++------- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 3926d1ef64f98..41169bf171586 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -71,6 +71,10 @@ static int brcmf_eap_restrict; module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); +static int brcmf_sdio_wq_highpri; +module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0); +MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -430,6 +434,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->roamoff = !!brcmf_roamoff; settings->iapp = !!brcmf_iapp_enable; settings->eap_restrict = !!brcmf_eap_restrict; + settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 059f09c47afe7..0cb39b62947c7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -38,6 +38,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @fcmode: FWS flow control. * @roamoff: Firmware roaming off? * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds + * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -49,6 +50,7 @@ struct brcmf_mp_device { bool roamoff; bool iapp; bool eap_restrict; + bool sdio_wq_highpri; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; const char *board_type; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 264ad63232f87..fbababf188a24 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4322,9 +4322,21 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) bus->txminmax = BRCMF_TXMINMAX; bus->tx_seq = SDPCM_SEQ_WRAP - 1; + /* attempt to attach to the dongle */ + if (!(brcmf_sdio_probe_attach(bus))) { + brcmf_err("brcmf_sdio_probe_attach failed\n"); + goto fail; + } + /* single-threaded workqueue */ - wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, - dev_name(&sdiodev->func1->dev)); + if (sdiodev->settings->sdio_wq_highpri) { + wq = alloc_workqueue("brcmf_wq/%s", + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, + 1, dev_name(&sdiodev->func1->dev)); + } else { + wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, + dev_name(&sdiodev->func1->dev)); + } if (!wq) { brcmf_err("insufficient memory to create txworkqueue\n"); goto fail; @@ -4333,12 +4345,6 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) INIT_WORK(&bus->datawork, brcmf_sdio_dataworker); bus->brcmf_wq = wq; - /* attempt to attach to the dongle */ - if (!(brcmf_sdio_probe_attach(bus))) { - brcmf_err("brcmf_sdio_probe_attach failed\n"); - goto fail; - } - spin_lock_init(&bus->rxctl_lock); spin_lock_init(&bus->txq_lock); init_waitqueue_head(&bus->ctrl_wait); From 5acdf64f6dd6851a83e8a48767d671d83878da7c Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 25 May 2018 02:02:04 -0500 Subject: [PATCH 018/135] brcmfmac: remove "arp_hostip_clear" from "brcmf_netdev_stop" The firmware does not respond ARP request and causes ping failed with following steps. 1. Bring up interface ifconfig wlan0 up or start wpa_supplicant 2. Set the IP address ifconfig wlan0 192.168.100.10 3. Bring down interface or ifconfig wlan0 down or kill wpa_supplicant 4. Bring up interface again and set the same IP address 5. Connect to AP(192.168.100.1) and ping to AP will be failed. FMAC clears arp_hostip when bringing down the interface, but not set it back if setting the same IP address. We are able to see the IP address in interface info(inconfig wlan0) but the ping still cannot work because the firmware ARP offload does not respond the ARP request. Because of that, we remove "arp_hostip_clear" from function "brcmf_netdev_stop" Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 1b473f02c84ea..c4857a0f03b0b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -579,9 +579,6 @@ static int brcmf_netdev_stop(struct net_device *ndev) brcmf_cfg80211_down(ndev); - if (ifp->drvr->bus_if->state == BRCMF_BUS_UP) - brcmf_fil_iovar_data_set(ifp, "arp_hostip_clear", NULL, 0); - brcmf_net_setcarrier(ifp, false); return 0; From c5fae3caa33956e37e38b03b294331d4a4819ee2 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 23 May 2018 17:22:44 +0530 Subject: [PATCH 019/135] brcmfmac: P2P CERT 6.1.9-Support GOUT handling P2P Presence Request Send P2P Presence Response from the p2p interface address instead of the p2p device address. Signed-off-by: Madhan Mohan R Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 7ba9f6a686459..7233b37cd1168 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1491,6 +1491,7 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, { struct brcmf_pub *drvr = p2p->cfg->pub; struct brcmf_cfg80211_vif *vif; + struct brcmf_p2p_action_frame *p2p_act_frame; s32 err = 0; s32 timeout = 0; @@ -1500,7 +1501,13 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, clear_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); clear_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); - vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + /* check if it is a p2p_presence response */ + p2p_act_frame = (struct brcmf_p2p_action_frame *)af_params->action_frame.data; + if (p2p_act_frame->subtype == P2P_AF_PRESENCE_RSP) + vif = p2p->bss_idx[P2PAPI_BSSCFG_CONNECTION].vif; + else + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe", af_params, sizeof(*af_params)); if (err) { From 80206b8d7448ec0cd696d7f777b029d93a39a5a2 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Fri, 1 Jun 2018 05:22:05 -0500 Subject: [PATCH 020/135] brcmfmac: only generate random p2p address when needed P2p spec mentioned that the p2p device address should be the globally administered address with locally administered bit set. Therefore, follow this guideline by default. When the primary interface is set to a locally administered address, the locally administered bit cannot be set again. Generate a random locally administered address for this case. Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/p2p.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 7233b37cd1168..4f15f4a449d69 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -457,10 +457,21 @@ static int brcmf_p2p_set_firmware(struct brcmf_if *ifp, u8 *p2p_mac) */ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) { + struct brcmf_if *pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; bool random_addr = false; + bool local_admin = false; - if (!dev_addr || is_zero_ether_addr(dev_addr)) - random_addr = true; + if (!dev_addr || is_zero_ether_addr(dev_addr)) { + /* If the primary interface address is already locally + * administered, create a new random address. + */ + if (pri_ifp->mac_addr[0] & 0x02) { + random_addr = true; + } else { + dev_addr = pri_ifp->mac_addr; + local_admin = true; + } + } /* Generate the P2P Device Address obtaining a random ethernet * address with the locally administered bit set. @@ -470,6 +481,9 @@ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) else memcpy(p2p->dev_addr, dev_addr, ETH_ALEN); + if (local_admin) + p2p->dev_addr[0] |= 0x02; + /* Generate the P2P Interface Address. If the discovery and connection * BSSCFGs need to simultaneously co-exist, then this address must be * different from the P2P Device Address, but also locally administered. From 13d4bc38de9d9891de7d424a34b067458b60895d Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Thu, 24 May 2018 14:19:00 +0530 Subject: [PATCH 021/135] brcmfmac: increase max hanger slots from 1K to 3K in fws layer commit 2bc50d8828ad7cc52ecd99f50074aa25f957af4 upstream. Will enable FMAC to push more packets to bus tx queue and help improve throughput when fws queuing is enabled. This change is required to tune the throughput for passing WMM CERT tests. Signed-off-by: Madhan Mohan R Signed-off-by: Chi-hsien Lin Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1585124429-97371-5-git-send-email-chi-hsien.lin@cypress.com --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 2bd892df83cc5..9f7a72ff8683d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -404,7 +404,7 @@ struct brcmf_fws_mac_descriptor { u8 traffic_lastreported_bmp; }; -#define BRCMF_FWS_HANGER_MAXITEMS 1024 +#define BRCMF_FWS_HANGER_MAXITEMS 3072 /** * enum brcmf_fws_hanger_item_state - state of hanger item. From c5a8b8085645d0001679d37cfffabb969d9944a0 Mon Sep 17 00:00:00 2001 From: Saravanan Shanmugham Date: Wed, 13 Sep 2017 14:04:14 +0900 Subject: [PATCH 022/135] brcmfmac: map 802.1d priority to precedence level based on AP WMM params In WLAN, priority among various access categories of traffic is always set by the AP using WMM parameters and this may not always follow the standard 802.1d priority. In this change, priority is adjusted based on the AP WMM params received as part of the Assoc Response and the same is later used to map the priority of all incoming traffic. This change should fix the following 802.11n certification tests: * 5.2.31 ACM Bit Conformance test * 5.2.32 AC Parameter Modification test Signed-off-by: Saravanan Shanmugham Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 145 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 24 +++ .../broadcom/brcm80211/brcmfmac/common.h | 4 + .../broadcom/brcm80211/brcmfmac/fwsignal.c | 4 +- .../broadcom/brcm80211/brcmfmac/sdio.c | 17 +- 5 files changed, 183 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d34448de78082..0cd0ea3907348 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5454,12 +5454,126 @@ static void brcmf_clear_assoc_ies(struct brcmf_cfg80211_info *cfg) conn_info->resp_ie_len = 0; } +u8 brcmf_map_prio_to_prec(void *config, u8 prio) +{ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (!cfg) + return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? + (prio ^ 2) : prio; + + /* For those AC(s) with ACM flag set to 1, convert its 4-level priority + * to an 8-level precedence which is the same as BE's + */ + if (prio > PRIO_8021D_EE && + cfg->ac_priority[prio] == cfg->ac_priority[PRIO_8021D_BE]) + return cfg->ac_priority[prio] * 2; + + /* Conversion of 4-level priority to 8-level precedence */ + if (prio == PRIO_8021D_BE || prio == PRIO_8021D_BK || + prio == PRIO_8021D_CL || prio == PRIO_8021D_VO) + return cfg->ac_priority[prio] * 2; + else + return cfg->ac_priority[prio] * 2 + 1; +} + +u8 brcmf_map_prio_to_aci(void *config, u8 prio) +{ + /* Prio here refers to the 802.1d priority in range of 0 to 7. + * ACI here refers to the WLAN AC Index in range of 0 to 3. + * This function will return ACI corresponding to input prio. + */ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (cfg) + return cfg->ac_priority[prio]; + + return prio; +} + +static void brcmf_wifi_prioritize_acparams(const + struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority) +{ + u8 aci; + u8 aifsn; + u8 ecwmin; + u8 ecwmax; + u8 acm; + u8 ranking_basis[EDCF_AC_COUNT]; + u8 aci_prio[EDCF_AC_COUNT]; /* AC_BE, AC_BK, AC_VI, AC_VO */ + u8 index; + + for (aci = 0; aci < EDCF_AC_COUNT; aci++, acp++) { + aifsn = acp->ACI & EDCF_AIFSN_MASK; + acm = (acp->ACI & EDCF_ACM_MASK) ? 1 : 0; + ecwmin = acp->ECW & EDCF_ECWMIN_MASK; + ecwmax = (acp->ECW & EDCF_ECWMAX_MASK) >> EDCF_ECWMAX_SHIFT; + brcmf_dbg(CONN, "ACI %d aifsn %d acm %d ecwmin %d ecwmax %d\n", + aci, aifsn, acm, ecwmin, ecwmax); + /* Default AC_VO will be the lowest ranking value */ + ranking_basis[aci] = aifsn + ecwmin + ecwmax; + /* Initialise priority starting at 0 (AC_BE) */ + aci_prio[aci] = 0; + + /* If ACM is set, STA can't use this AC as per 802.11. + * Change the ranking to BE + */ + if (aci != AC_BE && aci != AC_BK && acm == 1) + ranking_basis[aci] = ranking_basis[AC_BE]; + } + + /* Ranking method which works for AC priority + * swapping when values for cwmin, cwmax and aifsn are varied + * Compare each aci_prio against each other aci_prio + */ + for (aci = 0; aci < EDCF_AC_COUNT; aci++) { + for (index = 0; index < EDCF_AC_COUNT; index++) { + if (index != aci) { + /* Smaller ranking value has higher priority, + * so increment priority for each ACI which has + * a higher ranking value + */ + if (ranking_basis[aci] < ranking_basis[index]) + aci_prio[aci]++; + } + } + } + + /* By now, aci_prio[] will be in range of 0 to 3. + * Use ACI prio to get the new priority value for + * each 802.1d traffic type, in this range. + */ + + /* 802.1d 0,3 maps to BE */ + priority[0] = aci_prio[AC_BE]; + priority[3] = aci_prio[AC_BE]; + + /* 802.1d 1,2 maps to BK */ + priority[1] = aci_prio[AC_BK]; + priority[2] = aci_prio[AC_BK]; + + /* 802.1d 4,5 maps to VO */ + priority[4] = aci_prio[AC_VI]; + priority[5] = aci_prio[AC_VI]; + + /* 802.1d 6,7 maps to VO */ + priority[6] = aci_prio[AC_VO]; + priority[7] = aci_prio[AC_VO]; + + brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n", + priority[0], priority[1], priority[2], priority[3]); + + brcmf_dbg(CONN, "Adj prio VI 4->%d, VI 5->%d, VO 6->%d, VO 7->%d\n", + priority[4], priority[5], priority[6], priority[7]); +} + static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) { struct brcmf_pub *drvr = cfg->pub; struct brcmf_cfg80211_assoc_ielen_le *assoc_info; struct brcmf_cfg80211_connect_info *conn_info = cfg_to_conn(cfg); + struct brcmf_cfg80211_edcf_acparam edcf_acparam_info[EDCF_AC_COUNT]; u32 req_len; u32 resp_len; s32 err = 0; @@ -5508,6 +5622,17 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, GFP_KERNEL); if (!conn_info->resp_ie) conn_info->resp_ie_len = 0; + + err = brcmf_fil_iovar_data_get(ifp, "wme_ac_sta", + edcf_acparam_info, + sizeof(edcf_acparam_info)); + if (err) { + brcmf_err("could not get wme_ac_sta (%d)\n", err); + return err; + } + + brcmf_wifi_prioritize_acparams(edcf_acparam_info, + cfg->ac_priority); } else { conn_info->resp_ie_len = 0; conn_info->resp_ie = NULL; @@ -5825,6 +5950,25 @@ static void brcmf_init_conf(struct brcmf_cfg80211_conf *conf) conf->retry_long = (u32)-1; } +static void brcmf_init_wmm_prio(u8 *priority) +{ + /* Initialize AC priority array to default + * 802.1d priority as per following table: + * 802.1d prio 0,3 maps to BE + * 802.1d prio 1,2 maps to BK + * 802.1d prio 4,5 maps to VI + * 802.1d prio 6,7 maps to VO + */ + priority[0] = AC_BE; + priority[3] = AC_BE; + priority[1] = AC_BK; + priority[2] = AC_BK; + priority[4] = AC_VI; + priority[5] = AC_VI; + priority[6] = AC_VO; + priority[7] = AC_VO; +} + static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) { brcmf_fweh_register(cfg->pub, BRCMF_E_LINK, @@ -5919,6 +6063,7 @@ static s32 wl_init_priv(struct brcmf_cfg80211_info *cfg) mutex_init(&cfg->usr_sync); brcmf_init_escan(cfg); brcmf_init_conf(cfg->conf); + brcmf_init_wmm_prio(cfg->ac_priority); init_completion(&cfg->vif_disabled); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 2e95ad40465be..e229df705b579 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -23,6 +23,23 @@ #define WL_ROAM_TRIGGER_LEVEL -75 #define WL_ROAM_DELTA 20 +/* WME Access Category Indices (ACIs) */ +#define AC_BE 0 /* Best Effort */ +#define AC_BK 1 /* Background */ +#define AC_VI 2 /* Video */ +#define AC_VO 3 /* Voice */ +#define EDCF_AC_COUNT 4 +#define MAX_8021D_PRIO 8 + +#define EDCF_ACI_MASK 0x60 +#define EDCF_ACI_SHIFT 5 +#define EDCF_ACM_MASK 0x10 +#define EDCF_ECWMIN_MASK 0x0f +#define EDCF_ECWMAX_SHIFT 4 +#define EDCF_AIFSN_MASK 0x0f +#define EDCF_AIFSN_MAX 15 +#define EDCF_ECWMAX_MASK 0xf0 + /* Keep BRCMF_ESCAN_BUF_SIZE below 64K (65536). Allocing over 64K can be * problematic on some systems and should be avoided. */ @@ -210,6 +227,12 @@ struct brcmf_cfg80211_assoc_ielen_le { __le32 resp_len; }; +struct brcmf_cfg80211_edcf_acparam { + u8 ACI; + u8 ECW; + u16 TXOP; /* stored in network order (ls octet first) */ +}; + /* dongle escan state */ enum wl_escan_state { WL_ESCAN_STATE_IDLE, @@ -328,6 +351,7 @@ struct brcmf_cfg80211_info { struct brcmf_assoclist_le assoclist; struct brcmf_cfg80211_wowl wowl; struct brcmf_pno_info *pno; + u8 ac_priority[MAX_8021D_PRIO]; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 0cb39b62947c7..34991a853c06a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -76,4 +76,8 @@ static inline void brcmf_dmi_probe(struct brcmf_mp_device *settings, u32 chip, u32 chiprev) {} #endif +u8 brcmf_map_prio_to_prec(void *cfg, u8 prio); + +u8 brcmf_map_prio_to_aci(void *cfg, u8 prio); + #endif /* BRCMFMAC_COMMON_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 9f7a72ff8683d..a3fe8a9648136 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2130,8 +2130,10 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) skcb->if_flags = 0; skcb->state = BRCMF_FWS_SKBSTATE_NEW; brcmf_skb_if_flags_set_field(skb, INDEX, ifp->ifidx); + + /* mapping from 802.1d priority to firmware fifo index */ if (!multicast) - fifo = brcmf_fws_prio2fifo[skb->priority]; + fifo = brcmf_map_prio_to_aci(drvr->config, skb->priority); brcmf_fws_lock(fws); if (fifo != BRCMF_FWS_FIFO_AC_BE && fifo < BRCMF_FWS_FIFO_BCMC) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index fbababf188a24..59dd78feee23e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -313,15 +313,6 @@ struct rte_console { #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US) #define BRCMF_SDIO_MAX_ACCESS_ERRORS 5 -/* - * Conversion of 802.1D priority to precedence level - */ -static uint prio2prec(u32 prio) -{ - return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? - (prio^2) : prio; -} - #ifdef DEBUG /* Device console log buffer state */ struct brcmf_console { @@ -2767,7 +2758,13 @@ static int brcmf_sdio_bus_txdata(struct device *dev, struct sk_buff *pkt) skb_push(pkt, bus->tx_hdrlen); /* precondition: IS_ALIGNED((unsigned long)(pkt->data), 2) */ - prec = prio2prec((pkt->priority & PRIOMASK)); + /* In WLAN, priority is always set by the AP using WMM parameters + * and this need not always follow the standard 802.1d priority. + * Based on AP WMM config, map from 802.1d priority to corresponding + * precedence level. + */ + prec = brcmf_map_prio_to_prec(bus_if->drvr->config, + (pkt->priority & PRIOMASK)); /* Check for existing queue, current flow-control, pending event, or pending clock */ From 2e8eb6757e1d00813bfb7b41b5a9e3e3be1d595c Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 24 Jul 2018 01:21:30 -0500 Subject: [PATCH 023/135] brcmfmac: set state of hanger slot to FREE when flushing PSQ When USB or SDIO device got abnormal bus disconnection, host driver tried to clean up the SKBs in PSQ and TXQ. The SKBs pointer in hanger linked to PSQ and TSQ, so we should set the state of skb hanger slot to BRCMF_FWS_HANGER_ITEM_STATE_FREE before freeing skb. In brcmf_fws_bus_txq_cleanup it already set BRCMF_FWS_HANGER_ITEM_STATE_FREE before freeing SKB, therefore we add this in brcmf_fws_psq_flush to avoid following warning message. [ 1580.012880] ------------ [ cut here ]------------ [ 1580.017550] WARNING: CPU: 3 PID: 3065 at drivers/net/wireless/broadcom/brcm80211/brcmutil/utils.c:49 brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 1580.184017] Call Trace: [ 1580.186514] brcmf_fws_cleanup+0x14e/0x190 [brcmfmac] [ 1580.191594] brcmf_fws_del_interface+0x70/0x90 [brcmfmac] [ 1580.197029] brcmf_proto_bcdc_del_if+0xe/0x10 [brcmfmac] [ 1580.202418] brcmf_remove_interface+0x69/0x190 [brcmfmac] [ 1580.207888] brcmf_detach+0x90/0xe0 [brcmfmac] [ 1580.212385] brcmf_usb_disconnect+0x76/0xb0 [brcmfmac] [ 1580.217557] usb_unbind_interface+0x72/0x260 [ 1580.221857] device_release_driver_internal+0x141/0x200 [ 1580.227152] device_release_driver+0x12/0x20 [ 1580.231460] bus_remove_device+0xfd/0x170 [ 1580.235504] device_del+0x1d9/0x300 [ 1580.239041] usb_disable_device+0x9e/0x270 [ 1580.243160] usb_disconnect+0x94/0x270 [ 1580.246980] hub_event+0x76d/0x13b0 [ 1580.250499] process_one_work+0x144/0x360 [ 1580.254564] worker_thread+0x4d/0x3c0 [ 1580.258247] kthread+0x109/0x140 [ 1580.261515] ? rescuer_thread+0x340/0x340 [ 1580.265543] ? kthread_park+0x60/0x60 [ 1580.269237] ? SyS_exit_group+0x14/0x20 [ 1580.273118] ret_from_fork+0x25/0x30 [ 1580.300446] ------------ [ cut here ]------------ Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index a3fe8a9648136..477db9ba8310e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -643,6 +643,7 @@ static inline int brcmf_fws_hanger_poppkt(struct brcmf_fws_hanger *h, static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, int ifidx) { + struct brcmf_fws_hanger_item *hi; bool (*matchfn)(struct sk_buff *, void *) = NULL; struct sk_buff *skb; int prec; @@ -654,6 +655,9 @@ static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); while (skb) { hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); + hi = &fws->hanger.items[hslot]; + WARN_ON(skb != hi->pkt); + hi->state = BRCMF_FWS_HANGER_ITEM_STATE_FREE; brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, true); brcmu_pkt_buf_free_skb(skb); From 21e0ebb5db700bf227f1d0b79a0917a717184a16 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 12 Dec 2019 00:52:50 +0100 Subject: [PATCH 024/135] brcmfmac: add RSDB condition when setting interface combinations commit 837482e69a3f0d7cbc73922020012f83635f5ddb upstream. With firmware RSDB feature 1. The maximum support interface is four. 2. The maximum difference channel is two. 3. The maximum interfaces of {station/p2p client/AP} are two. 4. The maximum interface of p2p device is one. Signed-off-by: Wright Feng Signed-off-by: Soeren Moch Reviewed-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 0cd0ea3907348..35064d34b343c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6626,6 +6626,9 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #AP <= 1, channels = 1, 2 total * #AP <= 4, matching BI, channels = 1, 4 total * + * no p2p and rsdb: + * #STA <= 2, #AP <= 2, channels = 2, 4 total + * * p2p, no mchan, and mbss: * * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 1, 3 total @@ -6637,6 +6640,10 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 2, 3 total * #STA <= 1, #P2P-DEV <= 1, #AP <= 1, #P2P-CL <= 1, channels = 1, 4 total * #AP <= 4, matching BI, channels = 1, 4 total + * + * p2p, rsdb, and no mbss: + * #STA <= 2, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total */ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) { @@ -6644,13 +6651,14 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_iface_limit *c0_limits = NULL; struct ieee80211_iface_limit *p2p_limits = NULL; struct ieee80211_iface_limit *mbss_limits = NULL; - bool mbss, p2p; + bool mbss, p2p, rsdb; int i, c, n_combos; mbss = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS); p2p = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_P2P); + rsdb = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB); - n_combos = 1 + !!p2p + !!mbss; + n_combos = 1 + !!(p2p && !rsdb) + !!mbss; combo = kcalloc(n_combos, sizeof(*combo), GFP_KERNEL); if (!combo) goto err; @@ -6661,16 +6669,36 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c = 0; i = 0; - c0_limits = kcalloc(p2p ? 3 : 2, sizeof(*c0_limits), GFP_KERNEL); + if (p2p && rsdb) + c0_limits = kcalloc(4, sizeof(*c0_limits), GFP_KERNEL); + else if (p2p) + c0_limits = kcalloc(3, sizeof(*c0_limits), GFP_KERNEL); + else + c0_limits = kcalloc(2, sizeof(*c0_limits), GFP_KERNEL); if (!c0_limits) goto err; - c0_limits[i].max = 1; - c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); - if (p2p) { + if (p2p && rsdb) { + combo[c].num_different_channels = 2; + wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO) | + BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 5; + } else if (p2p) { if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN)) combo[c].num_different_channels = 2; else combo[c].num_different_channels = 1; + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); @@ -6679,16 +6707,26 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO); + combo[c].max_interfaces = i; + } else if (rsdb) { + combo[c].num_different_channels = 2; + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 3; } else { combo[c].num_different_channels = 1; c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = i; } - combo[c].max_interfaces = i; combo[c].n_limits = i; combo[c].limits = c0_limits; - if (p2p) { + if (p2p && !rsdb) { c++; i = 0; p2p_limits = kcalloc(4, sizeof(*p2p_limits), GFP_KERNEL); From 25d6a4e401ffda8acb5c5132fd6e4c983ede205c Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 12 Dec 2019 00:52:51 +0100 Subject: [PATCH 025/135] brcmfmac: not set mbss in vif if firmware does not support MBSS commit 2635853ce4ab7654a77ab7080fb56de83408606b upstream. With RSDB mode, FMAC and firmware are able to create 2 or more AP, so we should not set mbss in vif structure if firmware does not support MBSS feature. Signed-off-by: Wright Feng Signed-off-by: Soeren Moch Reviewed-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 35064d34b343c..1b7f9991aa3dc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5328,6 +5328,7 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif_walk; struct brcmf_cfg80211_vif *vif; bool mbss; + struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); brcmf_dbg(TRACE, "allocating virtual interface (size=%zu)\n", sizeof(*vif)); @@ -5340,7 +5341,8 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, brcmf_init_prof(&vif->profile); - if (type == NL80211_IFTYPE_AP) { + if (type == NL80211_IFTYPE_AP && + brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) { mbss = false; list_for_each_entry(vif_walk, &cfg->vif_list, list) { if (vif_walk->wdev.iftype == NL80211_IFTYPE_AP) { From a0191303cfce1f7e689cf79f8e92b757037506f0 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 29 Jun 2018 02:07:02 -0500 Subject: [PATCH 026/135] brcmfmac: support the second p2p connection With RSDB feature, firmware is able to support two P2P-AGO or two P2P-GC at the same time. So we add the second p2p connection type to maps to driver's second P2P connection bsscfg. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/p2p.c | 56 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/p2p.h | 9 ++- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 4f15f4a449d69..5128e35c99f96 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -17,6 +17,7 @@ #include "fwil_types.h" #include "p2p.h" #include "cfg80211.h" +#include "feature.h" /* parameters used for p2p escan */ #define P2PAPI_SCAN_NPROBES 1 @@ -488,9 +489,13 @@ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) * BSSCFGs need to simultaneously co-exist, then this address must be * different from the P2P Device Address, but also locally administered. */ - memcpy(p2p->int_addr, p2p->dev_addr, ETH_ALEN); - p2p->int_addr[0] |= 0x02; - p2p->int_addr[4] ^= 0x80; + memcpy(p2p->conn_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn_int_addr[0] |= 0x02; + p2p->conn_int_addr[4] ^= 0x80; + + memcpy(p2p->conn2_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn2_int_addr[0] |= 0x02; + p2p->conn2_int_addr[4] ^= 0x90; } /** @@ -2015,7 +2020,7 @@ int brcmf_p2p_ifchange(struct brcmf_cfg80211_info *cfg, if_request.type = cpu_to_le16((u16)if_type); if_request.chspec = cpu_to_le16(chanspec); - memcpy(if_request.addr, p2p->int_addr, sizeof(if_request.addr)); + memcpy(if_request.addr, p2p->conn_int_addr, sizeof(if_request.addr)); brcmf_cfg80211_arm_vif_event(cfg, vif); err = brcmf_fil_iovar_data_set(vif->ifp, "p2p_ifupd", &if_request, @@ -2169,6 +2174,27 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, return ERR_PTR(err); } +int brcmf_p2p_get_conn_idx(struct brcmf_cfg80211_info *cfg) +{ + int i; + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + + if (!ifp) + return -ENODEV; + + for (i = P2PAPI_BSSCFG_CONNECTION; i < P2PAPI_BSSCFG_MAX; i++) { + if (!cfg->p2p.bss_idx[i].vif) { + if (i == P2PAPI_BSSCFG_CONNECTION2 && + !(brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + brcmf_err("Multi p2p not supported"); + return -EIO; + } + return i; + } + } + return -EIO; +} + /** * brcmf_p2p_add_vif() - create a new P2P virtual interface. * @@ -2188,7 +2214,9 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, struct brcmf_pub *drvr = cfg->pub; struct brcmf_cfg80211_vif *vif; enum brcmf_fil_p2p_if_types iftype; - int err; + int err = 0; + int connidx; + u8 *p2p_intf_addr; if (brcmf_cfg80211_vif_event_armed(cfg)) return ERR_PTR(-EBUSY); @@ -2214,9 +2242,21 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, return (struct wireless_dev *)vif; brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, cfg->p2p.int_addr, - iftype); + connidx = brcmf_p2p_get_conn_idx(cfg); + + if (connidx == P2PAPI_BSSCFG_CONNECTION) + p2p_intf_addr = cfg->p2p.conn_int_addr; + else if (connidx == P2PAPI_BSSCFG_CONNECTION2) + p2p_intf_addr = cfg->p2p.conn2_int_addr; + else + err = -EINVAL; + + if (!err) + err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, + p2p_intf_addr, iftype); + if (err) { + brcmf_err("request p2p interface failed\n"); brcmf_cfg80211_arm_vif_event(cfg, NULL); goto fail; } @@ -2248,7 +2288,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, goto fail; } - cfg->p2p.bss_idx[P2PAPI_BSSCFG_CONNECTION].vif = vif; + cfg->p2p.bss_idx[connidx].vif = vif; /* Disable firmware roaming for P2P interface */ brcmf_fil_iovar_int_set(ifp, "roam_off", 1); if (iftype == BRCMF_FIL_P2P_IF_GO) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index 64ab9b6a677df..d2ecee565bf2e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -14,13 +14,15 @@ struct brcmf_cfg80211_info; * * @P2PAPI_BSSCFG_PRIMARY: maps to driver's primary bsscfg. * @P2PAPI_BSSCFG_DEVICE: maps to driver's P2P device discovery bsscfg. - * @P2PAPI_BSSCFG_CONNECTION: maps to driver's P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION: maps to driver's 1st P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION2: maps to driver's 2nd P2P connection bsscfg. * @P2PAPI_BSSCFG_MAX: used for range checking. */ enum p2p_bss_type { P2PAPI_BSSCFG_PRIMARY, /* maps to driver's primary bsscfg */ P2PAPI_BSSCFG_DEVICE, /* maps to driver's P2P device discovery bsscfg */ - P2PAPI_BSSCFG_CONNECTION, /* maps to driver's P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION, /* driver's 1st P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION2, /* driver's 2nd P2P connection bsscfg */ P2PAPI_BSSCFG_MAX }; @@ -119,7 +121,8 @@ struct brcmf_p2p_info { struct brcmf_cfg80211_info *cfg; unsigned long status; u8 dev_addr[ETH_ALEN]; - u8 int_addr[ETH_ALEN]; + u8 conn_int_addr[ETH_ALEN]; + u8 conn2_int_addr[ETH_ALEN]; struct p2p_bss bss_idx[P2PAPI_BSSCFG_MAX]; struct timer_list listen_timer; u8 listen_channel; From c2433727338ea351d09346b5538ed3c3906b93b2 Mon Sep 17 00:00:00 2001 From: Soeren Moch Date: Thu, 12 Dec 2019 00:52:49 +0100 Subject: [PATCH 027/135] brcmfmac: add support for BCM4359 SDIO chipset commit 6581154be4a2b0f77ff789b09fade7c14ec3078c upstream. BCM4359 is a 2x2 802.11 abgn+ac Dual-Band HT80 combo chip and it supports Real Simultaneous Dual Band feature. Based on a similar patch by: Wright Feng Signed-off-by: Soeren Moch Acked-by: Chi-Hsien Lin Acked-by: Ulf Hansson --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 1 + drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 2 ++ include/linux/mmc/sdio_ids.h | 2 ++ 4 files changed, 7 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index b64600273c685..3ab5ab67f26c9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -977,8 +977,10 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_89359), { /* end: all zeroes */ } }; MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index dd586a96b57a6..e2dc1cfeffa56 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1361,6 +1361,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) addr = CORE_CC_REG(base, sr_control0); reg = chip->ops->read32(chip->ctx, addr); return (reg & CC_SR_CTL0_ENABLE_MASK) != 0; + case BRCM_CC_4359_CHIP_ID: case CY_CC_43012_CHIP_ID: addr = CORE_CC_REG(pmu->base, retention_ctl); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 59dd78feee23e..b7a0fa3f26938 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -605,6 +605,7 @@ BRCMF_FW_DEF(43455, "brcmfmac43455-sdio"); BRCMF_FW_DEF(43456, "brcmfmac43456-sdio"); BRCMF_FW_DEF(4354, "brcmfmac4354-sdio"); BRCMF_FW_DEF(4356, "brcmfmac4356-sdio"); +BRCMF_FW_DEF(4359, "brcmfmac4359-sdio"); BRCMF_FW_DEF(4373, "brcmfmac4373-sdio"); BRCMF_FW_DEF(43012, "brcmfmac43012-sdio"); @@ -627,6 +628,7 @@ static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFDC0, 43455), BRCMF_FW_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), BRCMF_FW_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), + BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373), BRCMF_FW_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 08b25c02b5a11..2e9a6e4634eb3 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -41,8 +41,10 @@ #define SDIO_DEVICE_ID_BROADCOM_43455 0xa9bf #define SDIO_DEVICE_ID_BROADCOM_4354 0x4354 #define SDIO_DEVICE_ID_BROADCOM_4356 0x4356 +#define SDIO_DEVICE_ID_BROADCOM_4359 0x4359 #define SDIO_DEVICE_ID_CYPRESS_4373 0x4373 #define SDIO_DEVICE_ID_CYPRESS_43012 43012 +#define SDIO_DEVICE_ID_CYPRESS_89359 0x4355 #define SDIO_VENDOR_ID_INTEL 0x0089 #define SDIO_DEVICE_ID_INTEL_IWMC3200WIMAX 0x1402 From 0ecd78e54d344f118c678a1c0add433c8dd33d8b Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 9 May 2019 09:48:26 +0000 Subject: [PATCH 028/135] brcmfmac: send port authorized event for FT-802.1X commit be898fed355e70b9115b439b4b9587b5fea2aca1 upstream. With FT-802.1X, driver should send a port authorized event right after sending a roamed event. It is used to indicate that a new AP is already authorized so 802.1X is not required. Acked-by: Arend van Spriel Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin Signed-off-by: Kalle Valo --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 8 ++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + 2 files changed, 9 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 1b7f9991aa3dc..188426fc01919 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1647,6 +1647,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 count; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + profile->is_ft = false; if (!sme->crypto.n_akm_suites) return 0; @@ -1691,11 +1692,13 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) break; case WLAN_AKM_SUITE_FT_8021X: val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT; + profile->is_ft = true; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; + profile->is_ft = true; break; default: bphy_err(drvr, "invalid cipher group (%d)\n", @@ -5708,6 +5711,11 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { + cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); + brcmf_dbg(CONN, "Report port authorized\n"); + } + set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index e229df705b579..3df57e1a29be0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -146,6 +146,7 @@ struct brcmf_cfg80211_profile { struct brcmf_cfg80211_security sec; struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; + bool is_ft; }; /** From 54947fc9046d57b72616020a339e7e4cf9df2b6d Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Mon, 20 Aug 2018 16:27:41 +0900 Subject: [PATCH 029/135] brcmfmac: add vendor ie for association responses Miracast Certification clause 6.1.2 may fail if there is no WFD IE in P2P assoc response. This change allows WFD IE to be added to P2P assoc response. Related WFA certification. 6.1.2 P-SnUT operating as a Group Owner accepts a WFD Session with a Reference Source Signed-off-by: Ryohei Kondo Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 14 ++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 188426fc01919..0f217c70fe073 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4319,6 +4319,11 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, mgmt_ie_len = &saved_ie->assoc_req_ie_len; mgmt_ie_buf_len = sizeof(saved_ie->assoc_req_ie); break; + case BRCMF_VNDR_IE_ASSOCRSP_FLAG: + mgmt_ie_buf = saved_ie->assoc_res_ie; + mgmt_ie_len = &saved_ie->assoc_res_ie_len; + mgmt_ie_buf_len = sizeof(saved_ie->assoc_res_ie); + break; default: err = -EPERM; bphy_err(drvr, "not suitable type\n"); @@ -4465,6 +4470,15 @@ brcmf_config_ap_mgmt_ie(struct brcmf_cfg80211_vif *vif, else brcmf_dbg(TRACE, "Applied Vndr IEs for Probe Resp\n"); + /* Set Assoc Response IEs to FW */ + err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_ASSOCRSP_FLAG, + beacon->assocresp_ies, + beacon->assocresp_ies_len); + if (err) + brcmf_err("Set Assoc Resp IE Failed\n"); + else + brcmf_dbg(TRACE, "Applied Vndr IEs for Assoc Resp\n"); + return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 3df57e1a29be0..c9b3345191016 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -176,19 +176,23 @@ enum brcmf_vif_status { * @probe_req_ie: IE info for probe request. * @probe_res_ie: IE info for probe response. * @beacon_ie: IE info for beacon frame. + * @assoc_res_ie: IE info for association response frame. * @probe_req_ie_len: IE info length for probe request. * @probe_res_ie_len: IE info length for probe response. * @beacon_ie_len: IE info length for beacon frame. + * @assoc_res_ie_len: IE info length for association response frame. */ struct vif_saved_ie { u8 probe_req_ie[IE_MAX_LEN]; u8 probe_res_ie[IE_MAX_LEN]; u8 beacon_ie[IE_MAX_LEN]; u8 assoc_req_ie[IE_MAX_LEN]; + u8 assoc_res_ie[IE_MAX_LEN]; u32 probe_req_ie_len; u32 probe_res_ie_len; u32 beacon_ie_len; u32 assoc_req_ie_len; + u32 assoc_res_ie_len; }; /** From c26ea4ed74efb51d163b615bbf2034e1d49e1916 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Wed, 5 Sep 2018 01:18:25 -0500 Subject: [PATCH 030/135] brcmfmac: fix 4339 CRC error under SDIO 3.0 SDR104 mode This patch fixes 4339 CRC error while running Tput test with suspend/resume test script. The continuous failure messages before system crash: brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 25600 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 24576 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index b7a0fa3f26938..fb1071f879555 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -42,6 +42,10 @@ #define DEFAULT_F2_WATERMARK 0x8 #define CY_4373_F2_WATERMARK 0x40 #define CY_43012_F2_WATERMARK 0x60 +#define CY_4339_F2_WATERMARK 48 +#define CY_4339_MES_WATERMARK 80 +#define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG @@ -4204,6 +4208,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, &err); break; + case SDIO_DEVICE_ID_BROADCOM_4339: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 4339\n", + CY_4339_F2_WATERMARK); + brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, + CY_4339_F2_WATERMARK, &err); + devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, + CY_4339_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From 3cd37da2c6cc9cecfc70fadf645851aa03cd01ef Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 4 Sep 2018 00:10:49 -0500 Subject: [PATCH 031/135] brcmfmac: fix the incorrect return value in brcmf_inform_single_bss(). commit bd9944918ceb28ede97f715d209e220db5e92c09 upstream. The function brcmf_inform_single_bss returns the value as success, even when the length exceeds the maximum value. The fix is to send appropriate code on this error. This issue is observed when SVT reported random fmac crashes when running their tests and the path was identified from the crash logs. With this fix the random failure issue in SVT was resolved. Reviewed-by: Arend van Spriel Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1585124429-97371-4-git-send-email-chi-hsien.lin@cypress.com --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 0f217c70fe073..9e42b50d59cbd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2815,7 +2815,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) { bphy_err(drvr, "Bss info is larger than buffer. Discarding\n"); - return 0; + return -EINVAL; } if (!bi->ctl_ch) { From 97d79ab814149672c319b52efc113dbfd5799dd1 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Fri, 7 Sep 2018 13:49:55 -0500 Subject: [PATCH 032/135] brcmfmac: Fix double freeing in the fmac usb data path commit78179869dc3f5c0059bbf5d931a2717f1ad97ecd upstream. When the brcmf_fws_process_skb() fails to get hanger slot for queuing the skb, it tries to free the skb. But the caller brcmf_netdev_start_xmit() of that funciton frees the packet on error return value. This causes the double freeing and which caused the kernel crash. Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1585124429-97371-3-git-send-email-chi-hsien.lin@cypress.com --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 477db9ba8310e..a5db264ca7218 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2151,8 +2151,7 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) brcmf_fws_enq(fws, BRCMF_FWS_SKBSTATE_DELAYED, fifo, skb); brcmf_fws_schedule_deq(fws); } else { - bphy_err(drvr, "drop skb: no hanger slot\n"); - brcmf_txfinalize(ifp, skb, false); + bphy_err(drvr, "no hanger slot available\n"); rc = -ENOMEM; } brcmf_fws_unlock(fws); From 86e6ee5a71aac1680060acbf6c7edb9db6f905c5 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Fri, 7 Sep 2018 14:07:07 -0500 Subject: [PATCH 033/135] brcmfmac: Fix driver crash on USB control transfer timeout commit 93a5bfbc7cad8bf3dea81c9bc07761c1226a0860 upstream. When the control transfer gets timed out, the error status was returned without killing that urb, this leads to using the same urb. This issue causes the kernel crash as the same urb is sumbitted multiple times. The fix is to kill the urb for timeout transfer before returning error Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1585124429-97371-2-git-send-email-chi-hsien.lin@cypress.com --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 7cdfde9b3dea9..6e0b636fa0e96 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -328,11 +328,12 @@ static int brcmf_usb_tx_ctlpkt(struct device *dev, u8 *buf, u32 len) return err; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("Txctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); err = -EIO; } + clear_bit(0, &devinfo->ctl_op); return err; } @@ -358,11 +359,12 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) } timeout = brcmf_usb_ioctl_resp_wait(devinfo); err = devinfo->ctl_urb_status; - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("rxctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); err = -EIO; } + clear_bit(0, &devinfo->ctl_op); if (!err) return devinfo->ctl_urb_actual_length; else From ce6d0140abf6751a2c66f8a9e38e42c1860e4693 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 13 Sep 2018 03:07:29 -0500 Subject: [PATCH 034/135] brcmfmac: avoid network disconnection during suspend with linux-3.8 and above kernel From linux-3.8 kernel, wowlan packet filter is mandated to avoid the disconnection of connected network before suspend in commit 812569699119 ("cfg80211/mac80211: disconnect on suspend"). So as a dummy wowlan filter is configured for kernels linux-3.8 and above. Later all private command implementation of packet filters can be moved to wowlan based filters. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 9e42b50d59cbd..ae3f2d2c8c1d5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6931,8 +6931,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->vendor_commands = brcmf_vendor_cmds; wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) - brcmf_wiphy_wowl_params(wiphy, ifp); + brcmf_wiphy_wowl_params(wiphy, ifp); err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist, sizeof(bandlist)); if (err) { From 541b37cf16669486749cb3ab4fa9999723460cca Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 11 Sep 2018 21:28:25 -0500 Subject: [PATCH 035/135] brcmfmac: allow credit borrowing for all access categories Current credit borrowing allows only the access category BE to borrow the credits. This change is to fix the credit borrowing logic, to make borrowing available for all access categories and also to borrow only from the lower categories. Signed-off-by: Raveendran Somu Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index a5db264ca7218..9f836c07b3d72 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -405,6 +405,7 @@ struct brcmf_fws_mac_descriptor { }; #define BRCMF_FWS_HANGER_MAXITEMS 3072 +#define BRCMF_BORROW_RATIO 3 /** * enum brcmf_fws_hanger_item_state - state of hanger item. @@ -501,7 +502,8 @@ struct brcmf_fws_info { u32 fifo_enqpkt[BRCMF_FWS_FIFO_COUNT]; int fifo_credit[BRCMF_FWS_FIFO_COUNT]; int init_fifo_credit[BRCMF_FWS_FIFO_COUNT]; - int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1]; + int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1] + [BRCMF_FWS_FIFO_AC_VO + 1]; int deq_node_pos[BRCMF_FWS_FIFO_COUNT]; u32 fifo_credit_map; u32 fifo_delay_map; @@ -1211,13 +1213,11 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (!credits) return; - fws->fifo_credit_map |= 1 << fifo; - - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->credits_borrowed[0])) { + if (fifo > BRCMF_FWS_FIFO_AC_BK && + fifo <= BRCMF_FWS_FIFO_AC_VO) { for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0; lender_ac--) { - borrowed = &fws->credits_borrowed[lender_ac]; + borrowed = &fws->credits_borrowed[fifo][lender_ac]; if (*borrowed) { fws->fifo_credit_map |= (1 << lender_ac); fifo_credit = &fws->fifo_credit[lender_ac]; @@ -1234,7 +1234,11 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, } } - fws->fifo_credit[fifo] += credits; + if (credits) { + fws->fifo_credit[fifo] += credits; + fws->fifo_credit_map |= 1 << fifo; + } + if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) fws->fifo_credit[fifo] = fws->init_fifo_credit[fifo]; @@ -2031,27 +2035,31 @@ static void brcmf_fws_rollback_toq(struct brcmf_fws_info *fws, } } -static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws) +static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws, + int highest_lender_ac, int borrower_ac, + bool borrow_all) { - int lender_ac; + int lender_ac, borrow_limit = 0; - if (time_after(fws->borrow_defer_timestamp, jiffies)) { - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); - return -ENAVAIL; - } + for (lender_ac = 0; lender_ac <= highest_lender_ac; lender_ac++) { - for (lender_ac = 0; lender_ac <= BRCMF_FWS_FIFO_AC_VO; lender_ac++) { - if (fws->fifo_credit[lender_ac] > 0) { - fws->credits_borrowed[lender_ac]++; + if (!borrow_all) + borrow_limit = + fws->fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; + else + borrow_limit = 0; + + if (fws->fifo_credit[lender_ac] > borrow_limit) { + fws->credits_borrowed[borrower_ac][lender_ac]++; fws->fifo_credit[lender_ac]--; if (fws->fifo_credit[lender_ac] == 0) fws->fifo_credit_map &= ~(1 << lender_ac); - fws->fifo_credit_map |= (1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map |= (1 << borrower_ac); brcmf_dbg(DATA, "borrow credit from: %d\n", lender_ac); return 0; } } - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map &= ~(1 << borrower_ac); return -ENAVAIL; } @@ -2242,9 +2250,10 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) } continue; } - while ((fws->fifo_credit[fifo] > 0) || + + while ((fws->fifo_credit[fifo]) || ((!fws->bcmc_credit_check) && - (fifo == BRCMF_FWS_FIFO_BCMC))) { + (fifo == BRCMF_FWS_FIFO_BCMC))) { skb = brcmf_fws_deq(fws, fifo); if (!skb) break; @@ -2254,10 +2263,14 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) if (fws->bus_flow_blocked) break; } - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->fifo_credit[fifo] <= 0) && - (!fws->bus_flow_blocked)) { - while (brcmf_fws_borrow_credit(fws) == 0) { + + if (fifo >= BRCMF_FWS_FIFO_AC_BE && + fifo <= BRCMF_FWS_FIFO_AC_VO && + fws->fifo_credit[fifo] == 0 && + !fws->bus_flow_blocked) { + while (brcmf_fws_borrow_credit(fws, + fifo - 1, fifo, + true) == 0) { skb = brcmf_fws_deq(fws, fifo); if (!skb) { brcmf_fws_return_credits(fws, fifo, 1); From ff5c2b4a6b323246205a0374bfe132b1473b15cc Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 11 Sep 2018 19:06:06 -0500 Subject: [PATCH 036/135] non-upstream: Changes to improve USB Tx throughput. The inbound buffer been duplicated and returned to the upper layer to increase the througput. Below the improvement observed in different traffic UDP Rx UDP Tx TCP Rx TCP Tx 237 138 161 71 Without Tx improvement 238 155 162 137 With Tx improvement Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 6e0b636fa0e96..f834561b24f1b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -603,6 +603,7 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } + skb_orphan(skb); req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, From 235d5bdadd20629c158f36c765996a3a89c18f80 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 12 Dec 2019 00:52:45 +0100 Subject: [PATCH 037/135] brcmfmac: reset two D11 cores if chip has two D11 cores commit 1b8d2e0a9e4221b99eea375c079507ce8ef655f5 upstream. There are two D11 cores in RSDB chips like 4359. We have to reset two D11 cores simutaneously before firmware download, or the firmware may not be initialized correctly and cause "fw initialized failed" error. Signed-off-by: Wright Feng Signed-off-by: Soeren Moch Reviewed-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/chip.c | 50 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/chip.h | 1 + .../broadcom/brcm80211/brcmfmac/pcie.c | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index e2dc1cfeffa56..85cf4088acb48 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -433,11 +433,25 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; + struct brcmf_core *d11core2 = NULL; + struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; + /* special handle two D11 cores reset */ + if (core->pub.id == BCMA_CORE_80211) { + d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); + if (d11core2) { + brcmf_dbg(INFO, "found two d11 cores, reset both\n"); + d11priv2 = container_of(d11core2, + struct brcmf_core_priv, pub); + } + } + /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); + if (d11priv2) + brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -449,9 +463,30 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } + if (d11priv2) { + count = 0; + while (ci->ops->read32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL) & + BCMA_RESET_CTL_RESET) { + ci->ops->write32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL, + 0); + count++; + if (count > 50) + break; + usleep_range(40, 60); + } + } + ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); + + if (d11priv2) { + ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, + postreset | BCMA_IOCTL_CLK); + ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); + } } char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len) @@ -1113,6 +1148,21 @@ void brcmf_chip_detach(struct brcmf_chip *pub) kfree(chip); } +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit) +{ + struct brcmf_chip_priv *chip; + struct brcmf_core_priv *core; + + chip = container_of(pub, struct brcmf_chip_priv, pub); + list_for_each_entry(core, &chip->cores, list) { + if (core->pub.id == BCMA_CORE_80211) { + if (unit-- == 0) + return &core->pub; + } + } + return NULL; +} + struct brcmf_core *brcmf_chip_get_core(struct brcmf_chip *pub, u16 coreid) { struct brcmf_chip_priv *chip; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index 7b00f6a59e89e..8fa38658e727a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -74,6 +74,7 @@ struct brcmf_chip *brcmf_chip_attach(void *ctx, const struct brcmf_buscore_ops *ops); void brcmf_chip_detach(struct brcmf_chip *chip); struct brcmf_core *brcmf_chip_get_core(struct brcmf_chip *chip, u16 coreid); +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit); struct brcmf_core *brcmf_chip_get_chipcommon(struct brcmf_chip *chip); struct brcmf_core *brcmf_chip_get_pmu(struct brcmf_chip *pub); bool brcmf_chip_iscoreup(struct brcmf_core *core); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 3be60aef54650..3e32347392537 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -78,7 +78,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), }; -#define BRCMF_PCIE_FW_UP_TIMEOUT 2000 /* msec */ +#define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ #define BRCMF_PCIE_REG_MAP_SIZE (32 * 1024) From 3b1e286b442f7db9520c9a405eeb0233cc31bbb1 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Mon, 17 Sep 2018 04:51:08 -0500 Subject: [PATCH 038/135] brcmfmac: introduce module parameter to configure default PM mode Add module parameter max_pm to allow using PM_MAX as default power management mode. Default PM mode is set to PM_MAX when max_pm=1, and is set to PM_FAST when max_pm=0 or max_pm is not set. Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 ++-- drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c | 6 ++++++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ae3f2d2c8c1d5..26a88d44a1704 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2777,7 +2777,7 @@ brcmf_cfg80211_set_power_mgmt(struct wiphy *wiphy, struct net_device *ndev, goto done; } - pm = enabled ? PM_FAST : PM_OFF; + pm = enabled ? ifp->drvr->settings->default_pm : PM_OFF; /* Do not enable the power save after assoc if it is a p2p interface */ if (ifp->vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT) { brcmf_dbg(INFO, "Do not enable power save for P2P clients\n"); @@ -7009,7 +7009,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) brcmf_dongle_scantime(ifp); - power_mode = cfg->pwr_save ? PM_FAST : PM_OFF; + power_mode = cfg->pwr_save ? ifp->drvr->settings->default_pm : PM_OFF; err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PM, power_mode); if (err) goto default_conf_out; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 41169bf171586..7805bbacc80ae 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -20,6 +20,7 @@ #include "of.h" #include "firmware.h" #include "chip.h" +#include "defs.h" MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -75,6 +76,10 @@ static int brcmf_sdio_wq_highpri; module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0); MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority"); +static int brcmf_max_pm; +module_param_named(max_pm, brcmf_max_pm, int, 0); +MODULE_PARM_DESC(max_pm, "Use max power management mode by default"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -435,6 +440,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->iapp = !!brcmf_iapp_enable; settings->eap_restrict = !!brcmf_eap_restrict; settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; + settings->default_pm = !!brcmf_max_pm ? PM_MAX : PM_FAST; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 34991a853c06a..c0bf5867af5f3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -39,6 +39,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @roamoff: Firmware roaming off? * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. + * @default_pm: default power management (PM) mode. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -51,6 +52,7 @@ struct brcmf_mp_device { bool iapp; bool eap_restrict; bool sdio_wq_highpri; + int default_pm; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; const char *board_type; From ce28f6ee37f3cec3cb78e25b5691284d0583a712 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 20 Sep 2018 01:44:01 -0500 Subject: [PATCH 039/135] brcmfmac: configure wowl parameters in suspend function only if firmware support wowl This patch removes the redundant wowl configuration for none wowl FW. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 26a88d44a1704..eeb034dd6d7ad 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3777,8 +3777,9 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, brcmf_set_mpc(ifp, 1); } else { - /* Configure WOWL parameters */ - brcmf_configure_wowl(cfg, ifp, wowl); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) + /* Configure WOWL parameters */ + brcmf_configure_wowl(cfg, ifp, wowl); } exit: From 5976803a470f84b5529a6be2cfa88a951d5c4899 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 14 Nov 2018 23:13:02 -0600 Subject: [PATCH 040/135] brcmfmac: keep SDIO watchdog running when console_interval is non-zero brcmfmac host driver makes SDIO bus sleep and stops SDIO watchdog if no pending event or data. As a result, host driver does not poll firmware console buffer before buffer overflow, which leads to missing firmware logs. We should not stop SDIO watchdog if console_interval is non-zero in debug build. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index fb1071f879555..0c2db76ab3b58 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3685,7 +3685,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) if (bus->idlecount > bus->idletime) { brcmf_dbg(SDIO, "idle\n"); sdio_claim_host(bus->sdiodev->func1); - brcmf_sdio_wd_timer(bus, false); +#ifdef DEBUG + if (!BRCMF_FWCON_ON() || + bus->console_interval == 0) +#endif + brcmf_sdio_wd_timer(bus, false); bus->idlecount = 0; brcmf_sdio_bus_sleep(bus, true, false); sdio_release_host(bus->sdiodev->func1); From 17dc2186fe858d0b2a40511e14496294b47304cc Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Mon, 12 Nov 2018 20:32:11 -0600 Subject: [PATCH 041/135] brcmfmac: To fix kernel crash on out of boundary access To trunkcate the addtional bytes, if extra bytes been received. Current code only have a warning and proceed without handling it. But in one of the crash reported by DVT, these causes the crash intermittently. So the processing is limit to the skb->len. Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 9f836c07b3d72..e81204d69af91 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1873,6 +1873,9 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) WARN_ON(siglen > skb->len); + if (siglen > skb->len) + siglen = skb->len; + if (!siglen) return; /* if flow control disabled, skip to packet data and leave */ From d75f940cd1b55a441ac916e4aa12604d5a7a4c28 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 20 Nov 2018 21:20:41 -0600 Subject: [PATCH 042/135] brcmfmac: reduce maximum station interface from 2 to 1 in RSDB mode The firmware state machines are not fully suitable for concurrent station interface support, it may hit unexpected error if we have 2 different SSIDs and the roaming scenarios concurrently. To avoid the bad user-experience if this is not fully validated, we dis-allow user to create two concurrent station interfaces. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index eeb034dd6d7ad..7e51a7035baeb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6652,7 +6652,7 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #AP <= 4, matching BI, channels = 1, 4 total * * no p2p and rsdb: - * #STA <= 2, #AP <= 2, channels = 2, 4 total + * #STA <= 1, #AP <= 2, channels = 2, 3 total * * p2p, no mchan, and mbss: * @@ -6667,8 +6667,14 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #AP <= 4, matching BI, channels = 1, 4 total * * p2p, rsdb, and no mbss: - * #STA <= 2, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, * channels = 2, 4 total + * + * p2p, rsdb, mbss + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total + * #AP <= 4, matching BI, channels = 1, 4 total + * */ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) { @@ -6707,7 +6713,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); - c0_limits[i].max = 2; + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE); @@ -6716,7 +6722,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) BIT(NL80211_IFTYPE_P2P_GO); c0_limits[i].max = 2; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); - combo[c].max_interfaces = 5; + combo[c].max_interfaces = 4; } else if (p2p) { if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN)) combo[c].num_different_channels = 2; @@ -6735,7 +6741,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) combo[c].max_interfaces = i; } else if (rsdb) { combo[c].num_different_channels = 2; - c0_limits[i].max = 2; + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); c0_limits[i].max = 2; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); From 72b56f62a72c877f066fa759a104604965bac5a5 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 21 Nov 2018 03:16:55 -0600 Subject: [PATCH 043/135] brcmfmac: validate ifp pointer in brcmf_txfinalize We got ifp null pointer kernel panic in brcmf_txfinalize after removing Wi-Fi USB dongle when data was transmitting, The root cause is that interface was removed before calling brcmf_txfinalize in brcmf_fws_dequeue_worker and finally caused kernel panic. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index c4857a0f03b0b..175f5447a2646 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -536,6 +536,11 @@ void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success) struct ethhdr *eh; u16 type; + if (!ifp) { + brcmu_pkt_buf_free_skb(txp); + return; + } + eh = (struct ethhdr *)(txp->data); type = ntohs(eh->h_proto); From 9c38ff6973e6493b278f6e0a6fa7291d094c036b Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 21 Nov 2018 20:42:39 -0600 Subject: [PATCH 044/135] brcmfmac: clean up iface mac descriptor before de-initializing it We saw following warning message after removing USB Wi-Fi dongle with data connection. The root cause is that some skb were queued in iface mac descriptor psq and occupied the hanger slots, and we didn't de-queue the skb before de-initializing iface mac descriptor. So it triggered WARN_ON when cleaning up hanger slot eventually. backports/drivers/net/wireless/broadcom/brcm80211/brcmutil/utils.c:49 brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 977.523200] Modules linked in: brcmfmac(OE) brcmutil(OE) cfg80211(OE) compat(OE) sdhci_pci(OE) sdhci(OE) mmc_core(OE) rfkill ip6table_filter ip6_tables ebtable_nat ebtables dns_resolver fscache e1000e mei_me mei tpm_tis tpm_tis_core tpm iTCO_wdt ppdev iTCO_vendor_support pcspkr lpc_ich i2c_i801 mfd_core ptp pps_core parport_pc parport wmi uinput tcp_bic i915 iosf_mbi i2c_algo_bit drm_kms_helper drm i2c_core video [last unloaded: brcmfmac] [ 977.523219] CPU: 1 PID: 1306 Comm: kworker/1:1 Tainted: G OE 4.12.0 #1 [ 977.523220] Hardware name: /DH77EB, BIOS EBH7710H.86A.0100.2013.0312.1351 03/12/2013 [ 977.523223] Workqueue: usb_hub_wq hub_event [ 977.523224] task: ffff880118703600 task.stack: ffffc90000be4000 [ 977.523226] RIP: 0010:brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 977.523227] RSP: 0018:ffffc90000be7a98 EFLAGS: 00010086 [ 977.523228] RAX: 0000000000000045 RBX: ffffffffa03fa850 RCX: 0000000000000006 [ 977.523228] RDX: 0000000000000000 RSI: 0000000000000092 RDI: ffff8801106a3ce8 [ 977.523229] RBP: ffffc90000be7a98 R08: 0000000000000000 R09: 000000000000045b [ 977.523229] R10: 0000000000000001 R11: 0000000000aaaaaa R12: 0000000000000988 [ 977.523230] R13: ffff880110569938 R14: 0000000000000002 R15: ffff8801106a3ce8 [ 977.523231] FS: 0000000000000000(0000) GS:ffff88011f280000(0000) knlGS:0000000000000000 [ 977.523232] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.523232] CR2: 00007f7873490000 CR3: 0000000001c09000 CR4: 00000000001406e0 [ 977.523233] Call Trace: [ 977.523244] brcmf_fws_cleanup+0x1f2/0x230 [brcmfmac] [ 977.523251] brcmf_fws_del_interface+0x70/0x90 [brcmfmac] [ 977.523257] brcmf_proto_bcdc_del_if+0xe/0x10 [brcmfmac] [ 977.523262] brcmf_remove_interface+0x69/0x190 [brcmfmac] [ 977.523267] brcmf_detach+0x90/0xe0 [brcmfmac] [ 977.523273] brcmf_usb_disconnect+0x76/0xb0 [brcmfmac] [ 977.523275] usb_unbind_interface+0x72/0x260 [ 977.523279] device_release_driver_internal+0x141/0x200 [ 977.523280] device_release_driver+0x12/0x20 [ 977.523282] bus_remove_device+0xfd/0x170 [ 977.523283] device_del+0x1d9/0x300 [ 977.523284] usb_disable_device+0x9e/0x270 [ 977.523286] usb_disconnect+0x94/0x270 [ 977.523287] usb_disconnect+0x1f1/0x270 [ 977.523288] hub_event+0x76d/0x13b0 [ 977.523291] process_one_work+0x144/0x360 [ 977.523293] worker_thread+0x4d/0x3c0 [ 977.523294] kthread+0x112/0x150 [ 977.523295] ? rescuer_thread+0x340/0x340 [ 977.523296] ? kthread_park+0x60/0x60 [ 977.523298] ? SyS_exit_group+0x14/0x20 [ 977.523300] ret_from_fork+0x25/0x30 [ 977.523301] Code: 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 48 85 ff 74 15 48 83 3f 00 55 48 89 e5 75 0d be 01 00 00 00 e8 82 5a 3e e1 5d f3 c3 <0f> ff eb ef 66 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index e81204d69af91..03153be91ecdc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2208,6 +2208,7 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_lock(fws); ifp->fws_desc = NULL; + brcmf_fws_macdesc_cleanup(fws, entry, ifp->ifidx); brcmf_dbg(TRACE, "deleting %s\n", entry->name); brcmf_fws_macdesc_cleanup(fws, &fws->desc.iface[ifp->ifidx], ifp->ifidx); From 0f70803f07d4c948a1211059748eff745200fd7d Mon Sep 17 00:00:00 2001 From: Prasanna Kerekoppa Date: Tue, 4 Dec 2018 00:30:31 -0600 Subject: [PATCH 045/135] brcmfmac: To fix Bss Info flag definition Bug Bss info flag definition need to be fixed from 0x2 to 0x4 This flag is for rssi info received on channel. All Firmware branches defined as 0x4 and this is bug in brcmfmac. Signed-off-by: Prasanna Kerekoppa Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index f3f7b1514167a..ee728d6af2a3b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -19,7 +19,7 @@ #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 #define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ -#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0002 +#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ #define BRCMF_STA_WME 0x00000002 /* WMM association */ From 4da49b8fcb7f3083825693450c637398b735131f Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 17 Dec 2018 00:54:07 -0600 Subject: [PATCH 046/135] brcmfmac: disable command decode in sdio_aos for 4356 AOS is a part of the SDIOD core that becomes active when the rest of SDIOD is sleeping to keep SDIO bus alive responding to reduced set of commands. Transaction between AOS and SDIOD is not protected, and if cmd 52 is received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 0c2db76ab3b58..0ebbecef20e0d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3378,7 +3378,9 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus, static bool brcmf_sdio_aos_no_decode(struct brcmf_sdio *bus) { - if (bus->ci->chip == CY_CC_43012_CHIP_ID) + if (bus->ci->chip == CY_CC_43012_CHIP_ID || + bus->ci->chip == BRCM_CC_4354_CHIP_ID || + bus->ci->chip == BRCM_CC_4356_CHIP_ID) return true; else return false; From 74667048182fe10f5d5c62da08d3de02874338ec Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Thu, 8 Nov 2018 10:37:29 +0900 Subject: [PATCH 047/135] brcmfmac: increase default max WOWL patterns to 16 4373 has support of 16 WOWL patterns thus increasing the default value Signed-off-by: Ryohei Kondo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index ee728d6af2a3b..f504fd231dfd7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -133,7 +133,7 @@ /* Link Down indication in WoWL mode: */ #define BRCMF_WOWL_LINKDOWN (1 << 31) -#define BRCMF_WOWL_MAXPATTERNS 8 +#define BRCMF_WOWL_MAXPATTERNS 16 #define BRCMF_WOWL_MAXPATTERNSIZE 128 enum { From d5d4eeb002898c61f5144b33824669678013983b Mon Sep 17 00:00:00 2001 From: Robert Trask Date: Thu, 20 Dec 2018 11:55:30 -0600 Subject: [PATCH 048/135] non-upstream: Enable, Process, and forward PHY_TEMP event. New code enables PHY_TEMP event in firmware, receives it, processes into cfg80211 vendor specific event, and forwards to waiting host. -1- Enable rx PHY_TEMP event from underlying hardware -2- Process PHY_TEMP event into vendor specific event -3- Forward vendor specific event to host layer Signed-off-by: Robert Trask Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 5 ++ .../broadcom/brcm80211/brcmfmac/fweh.h | 1 + .../broadcom/brcm80211/brcmfmac/vendor.c | 57 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/vendor.h | 19 +++++++ 4 files changed, 82 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 7e51a7035baeb..2460054cb7219 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6870,6 +6870,7 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); const struct ieee80211_iface_combination *combo; struct ieee80211_supported_band *band; u16 max_interfaces = 0; @@ -6937,6 +6938,10 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) /* vendor commands/events support */ wiphy->vendor_commands = brcmf_vendor_cmds; wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; + wiphy->vendor_events = brcmf_vendor_events; + wiphy->n_vendor_events = BRCMF_VNDR_EVTS_LAST; + brcmf_fweh_register(cfg->pub, BRCMF_E_PHY_TEMP, + brcmf_wiphy_phy_temp_evt_handler); brcmf_wiphy_wowl_params(wiphy, ifp); err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index a82f51bc1e69b..8210509f03f3d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -90,6 +90,7 @@ struct brcmf_cfg80211_info; BRCMF_ENUM_DEF(FIFO_CREDIT_MAP, 74) \ BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \ BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \ + BRCMF_ENUM_DEF(PHY_TEMP, 111) \ BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) #define BRCMF_ENUM_DEF(id, val) \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d07e7c7355d9d..4ed3287522cbe 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -104,6 +104,56 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +s32 +brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) + +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct wiphy *wiphy = cfg_to_wiphy(cfg); + struct sk_buff *skb; + struct nlattr *phy_temp_data; + u32 version, temp, tempdelta; + struct brcmf_phy_temp_evt *phy_temp_evt; + + phy_temp_evt = (struct brcmf_phy_temp_evt *)data; + + version = le32_to_cpu(phy_temp_evt->version); + temp = le32_to_cpu(phy_temp_evt->temp); + tempdelta = le32_to_cpu(phy_temp_evt->tempdelta); + + skb = cfg80211_vendor_event_alloc(wiphy, NULL, + sizeof(*phy_temp_evt), + BRCMF_VNDR_EVTS_PHY_TEMP, + GFP_KERNEL); + + if (!skb) { + brcmf_dbg(EVENT, "NO MEM: can't allocate skb for vendor PHY_TEMP_EVENT\n"); + return -ENOMEM; + } + + phy_temp_data = nla_nest_start(skb, NL80211_ATTR_VENDOR_EVENTS); + if (!phy_temp_data) { + nla_nest_cancel(skb, phy_temp_data); + kfree_skb(skb); + brcmf_dbg(EVENT, "skb could not nest vendor attributes\n"); + return -EMSGSIZE; + } + + if (nla_put_u32(skb, BRCMF_NLATTR_VERS, version) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMP, temp) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMPDELTA, tempdelta)) { + kfree_skb(skb); + brcmf_dbg(EVENT, "NO ROOM in skb for vendor PHY_TEMP_EVENT\n"); + return -EMSGSIZE; + } + + nla_nest_end(skb, phy_temp_data); + + cfg80211_vendor_event(skb, GFP_KERNEL); + return 0; +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -116,3 +166,10 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, }; + +const struct nl80211_vendor_cmd_info brcmf_vendor_events[] = { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_EVTS_PHY_TEMP, + }, +}; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 418f33ea6fd3f..3bdf473697883 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -14,6 +14,11 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_LAST }; +enum brcmf_vndr_evts { + BRCMF_VNDR_EVTS_PHY_TEMP, + BRCMF_VNDR_EVTS_LAST +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * @@ -25,11 +30,21 @@ enum brcmf_nlattrs { BRCMF_NLATTR_LEN, BRCMF_NLATTR_DATA, + BRCMF_NLATTR_VERS, + BRCMF_NLATTR_PHY_TEMP, + BRCMF_NLATTR_PHY_TEMPDELTA, __BRCMF_NLATTR_AFTER_LAST, BRCMF_NLATTR_MAX = __BRCMF_NLATTR_AFTER_LAST - 1 }; +/* structure of event sent up by firmware: is this the right place for it? */ +struct brcmf_phy_temp_evt { + __le32 version; + __le32 temp; + __le32 tempdelta; +} __packed; + /** * struct brcmf_vndr_dcmd_hdr - message header for cfg80211 vendor command dcmd * support @@ -49,5 +64,9 @@ struct brcmf_vndr_dcmd_hdr { }; extern const struct wiphy_vendor_command brcmf_vendor_cmds[]; +extern const struct nl80211_vendor_cmd_info brcmf_vendor_events[]; +s32 brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data); #endif /* _vendor_h_ */ From 77bb17a7937bb2108cdba8aa52c8c28f5fdd4416 Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 23 Jan 2019 03:32:16 -0600 Subject: [PATCH 049/135] brcmfmac: Use FW priority definition to initialize WMM AC priority array It was observed that TCP Tx/Rx and UDP Tx showed very low throughput when fcmode=2. This is caused by incorrect default FIFO priority. BK FIFO queue was used for default 802.1D BE priority traffic. Hostapd relies on the correct priority setting to deliver good throughput. Signed-off-by: Justin Li Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 17 +++++++------- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 22 ------------------ .../broadcom/brcm80211/brcmfmac/fwsignal.h | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 2460054cb7219..8e17bb793a257 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -22,6 +22,7 @@ #include "p2p.h" #include "btcoex.h" #include "pno.h" +#include "fwsignal.h" #include "cfg80211.h" #include "feature.h" #include "fwil.h" @@ -5984,14 +5985,14 @@ static void brcmf_init_wmm_prio(u8 *priority) * 802.1d prio 4,5 maps to VI * 802.1d prio 6,7 maps to VO */ - priority[0] = AC_BE; - priority[3] = AC_BE; - priority[1] = AC_BK; - priority[2] = AC_BK; - priority[4] = AC_VI; - priority[5] = AC_VI; - priority[6] = AC_VO; - priority[7] = AC_VO; + priority[0] = BRCMF_FWS_FIFO_AC_BE; + priority[3] = BRCMF_FWS_FIFO_AC_BE; + priority[1] = BRCMF_FWS_FIFO_AC_BK; + priority[2] = BRCMF_FWS_FIFO_AC_BK; + priority[4] = BRCMF_FWS_FIFO_AC_VI; + priority[5] = BRCMF_FWS_FIFO_AC_VI; + priority[6] = BRCMF_FWS_FIFO_AC_VO; + priority[7] = BRCMF_FWS_FIFO_AC_VO; } static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 03153be91ecdc..86c6fc233bb47 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -311,28 +311,6 @@ struct brcmf_skbuff_cb { /* How long to defer borrowing in jiffies */ #define BRCMF_FWS_BORROW_DEFER_PERIOD (HZ / 10) -/** - * enum brcmf_fws_fifo - fifo indices used by dongle firmware. - * - * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. - * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. - * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. - * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. - * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. - * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). - * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). - * @BRCMF_FWS_FIFO_COUNT: number of fifos. - */ -enum brcmf_fws_fifo { - BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BE, - BRCMF_FWS_FIFO_AC_VI, - BRCMF_FWS_FIFO_AC_VO, - BRCMF_FWS_FIFO_BCMC, - BRCMF_FWS_FIFO_ATIM, - BRCMF_FWS_FIFO_COUNT -}; /** * enum brcmf_fws_txstatus - txstatus flag values. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index b486d578ec963..b16a9d1c0508e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -6,6 +6,29 @@ #ifndef FWSIGNAL_H_ #define FWSIGNAL_H_ +/** + * enum brcmf_fws_fifo - fifo indices used by dongle firmware. + * + * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. + * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. + * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. + * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. + * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. + * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). + * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). + * @BRCMF_FWS_FIFO_COUNT: number of fifos. + */ +enum brcmf_fws_fifo { + BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BE, + BRCMF_FWS_FIFO_AC_VI, + BRCMF_FWS_FIFO_AC_VO, + BRCMF_FWS_FIFO_BCMC, + BRCMF_FWS_FIFO_ATIM, + BRCMF_FWS_FIFO_COUNT +}; + struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr); void brcmf_fws_detach(struct brcmf_fws_info *fws); void brcmf_fws_debugfs_create(struct brcmf_pub *drvr); From 4a62d5893f408c300cfdae8d2d1d0d7100f07052 Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Wed, 30 Jan 2019 21:22:04 -0600 Subject: [PATCH 050/135] brcmfmac: Fix P2P Group Formation failure via Go-neg method P2P group formation fails since either peer is not able to send go-neg confirm or dut is not able to send go-neg response. To fix this, retry limit should be increased and dwell time check should be added. Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/p2p.c | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 5128e35c99f96..31862caa0c69d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -60,7 +60,7 @@ #define P2P_AF_MIN_DWELL_TIME 100 #define P2P_AF_MED_DWELL_TIME 400 #define P2P_AF_LONG_DWELL_TIME 1000 -#define P2P_AF_TX_MAX_RETRY 1 +#define P2P_AF_TX_MAX_RETRY 5 #define P2P_AF_MAX_WAIT_TIME msecs_to_jiffies(2000) #define P2P_INVALID_CHANNEL -1 #define P2P_CHANNEL_SYNC_RETRY 5 @@ -93,6 +93,9 @@ #define P2PSD_ACTION_ID_GAS_CRESP 0x0d /* GAS Comback Response AF */ #define BRCMF_P2P_DISABLE_TIMEOUT msecs_to_jiffies(500) + +/* Mask for retry counter of custom dwell time */ +#define CUSTOM_RETRY_MASK 0xff000000 /** * struct brcmf_p2p_disc_st_le - set discovery state in firmware. * @@ -1666,6 +1669,17 @@ static s32 brcmf_p2p_pub_af_tx(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_p2p_check_dwell_overflow(s32 requested_dwell, + unsigned long dwell_jiffies) +{ + if ((requested_dwell & CUSTOM_RETRY_MASK) && + (jiffies_to_msecs(jiffies - dwell_jiffies) > + (requested_dwell & ~CUSTOM_RETRY_MASK))) { + brcmf_err("Action frame TX retry time over dwell time!\n"); + return true; + } + return false; +} /** * brcmf_p2p_send_action_frame() - send action frame . * @@ -1690,6 +1704,10 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, s32 tx_retry; s32 extra_listen_time; uint delta_ms; + unsigned long dwell_jiffies = 0; + bool dwell_overflow = false; + + s32 requested_dwell = af_params->dwell_time; action_frame = &af_params->action_frame; action_frame_len = le16_to_cpu(action_frame->len); @@ -1801,12 +1819,18 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, /* update channel */ af_params->channel = cpu_to_le32(afx_hdl->peer_chan); } + dwell_jiffies = jiffies; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); tx_retry = 0; while (!p2p->block_gon_req_tx && - (ack == false) && (tx_retry < P2P_AF_TX_MAX_RETRY)) { + (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) && + !dwell_overflow) { ack = !brcmf_p2p_tx_action_frame(p2p, af_params); tx_retry++; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); } if (ack == false) { bphy_err(drvr, "Failed to send Action Frame(retry %d)\n", From cc34bb6a39c79d01135ffbfa7c32cd22356b33e9 Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 27 Feb 2019 02:47:49 -0600 Subject: [PATCH 051/135] brcmfmac: Add P2P Action Frame retry delay to fix GAS Comeback Response failure issue It was observed that P2P Cert. 5.1.19: DEVUT responds to Service Discovery request failed due to DUT did not send GAS Comeback Response after receiving request from test bed P2P peer. To fix this issue, we need to add P2P Action Frame retry delay to enhance P2P connection under VSDB and noisy environment, since the peer can be in other channels under VSDB. Signed-off-by: Justin Li Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 31862caa0c69d..793b5a03ea274 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -66,6 +66,7 @@ #define P2P_CHANNEL_SYNC_RETRY 5 #define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(450) #define P2P_DEFAULT_SLEEP_TIME_VSDB 200 +#define P2P_AF_RETRY_DELAY_TIME 40 /* WiFi P2P Public Action Frame OUI Subtypes */ #define P2P_PAF_GON_REQ 0 /* Group Owner Negotiation Req */ @@ -1827,6 +1828,9 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, while (!p2p->block_gon_req_tx && (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) && !dwell_overflow) { + if (af_params->channel) + msleep(P2P_AF_RETRY_DELAY_TIME); + ack = !brcmf_p2p_tx_action_frame(p2p, af_params); tx_retry++; dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, From 14640db60e45eac1ebcb5fd3419322598f0661e0 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 27 Feb 2019 03:36:42 -0600 Subject: [PATCH 052/135] brcmfmac: Use default FW priority when EDCA params same for all ACs In brcmfmac driver, all ACs were assgined a FW priority based on the EDCA parameters from AP. In a specific scenario where EDCA parameters are configured to be same for all ACs, we propose to use the default FW priority definition to avoid queuing packets of all ACs to the same priority queue. Also in case of fcmode=2, throughput of any AC would have depended on available credits of a single AC, without this fix. Fixes 11AC CERT 5.2.33 TXOP Limit test Signed-off-by: Madhan Mohan R Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 8e17bb793a257..ce2a79beb6bd0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5512,6 +5512,25 @@ u8 brcmf_map_prio_to_aci(void *config, u8 prio) return prio; } +static void brcmf_init_wmm_prio(u8 *priority) +{ + /* Initialize AC priority array to default + * 802.1d priority as per following table: + * 802.1d prio 0,3 maps to BE + * 802.1d prio 1,2 maps to BK + * 802.1d prio 4,5 maps to VI + * 802.1d prio 6,7 maps to VO + */ + priority[0] = BRCMF_FWS_FIFO_AC_BE; + priority[3] = BRCMF_FWS_FIFO_AC_BE; + priority[1] = BRCMF_FWS_FIFO_AC_BK; + priority[2] = BRCMF_FWS_FIFO_AC_BK; + priority[4] = BRCMF_FWS_FIFO_AC_VI; + priority[5] = BRCMF_FWS_FIFO_AC_VI; + priority[6] = BRCMF_FWS_FIFO_AC_VO; + priority[7] = BRCMF_FWS_FIFO_AC_VO; +} + static void brcmf_wifi_prioritize_acparams(const struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority) { @@ -5564,22 +5583,30 @@ static void brcmf_wifi_prioritize_acparams(const * Use ACI prio to get the new priority value for * each 802.1d traffic type, in this range. */ + if (!(aci_prio[AC_BE] == aci_prio[AC_BK] && + aci_prio[AC_BK] == aci_prio[AC_VI] && + aci_prio[AC_VI] == aci_prio[AC_VO])) { - /* 802.1d 0,3 maps to BE */ - priority[0] = aci_prio[AC_BE]; - priority[3] = aci_prio[AC_BE]; + /* 802.1d 0,3 maps to BE */ + priority[0] = aci_prio[AC_BE]; + priority[3] = aci_prio[AC_BE]; - /* 802.1d 1,2 maps to BK */ - priority[1] = aci_prio[AC_BK]; - priority[2] = aci_prio[AC_BK]; + /* 802.1d 1,2 maps to BK */ + priority[1] = aci_prio[AC_BK]; + priority[2] = aci_prio[AC_BK]; - /* 802.1d 4,5 maps to VO */ - priority[4] = aci_prio[AC_VI]; - priority[5] = aci_prio[AC_VI]; + /* 802.1d 4,5 maps to VO */ + priority[4] = aci_prio[AC_VI]; + priority[5] = aci_prio[AC_VI]; - /* 802.1d 6,7 maps to VO */ - priority[6] = aci_prio[AC_VO]; - priority[7] = aci_prio[AC_VO]; + /* 802.1d 6,7 maps to VO */ + priority[6] = aci_prio[AC_VO]; + priority[7] = aci_prio[AC_VO]; + + } else { + /* Initialize to default priority */ + brcmf_init_wmm_prio(priority); + } brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n", priority[0], priority[1], priority[2], priority[3]); @@ -5976,25 +6003,6 @@ static void brcmf_init_conf(struct brcmf_cfg80211_conf *conf) conf->retry_long = (u32)-1; } -static void brcmf_init_wmm_prio(u8 *priority) -{ - /* Initialize AC priority array to default - * 802.1d priority as per following table: - * 802.1d prio 0,3 maps to BE - * 802.1d prio 1,2 maps to BK - * 802.1d prio 4,5 maps to VI - * 802.1d prio 6,7 maps to VO - */ - priority[0] = BRCMF_FWS_FIFO_AC_BE; - priority[3] = BRCMF_FWS_FIFO_AC_BE; - priority[1] = BRCMF_FWS_FIFO_AC_BK; - priority[2] = BRCMF_FWS_FIFO_AC_BK; - priority[4] = BRCMF_FWS_FIFO_AC_VI; - priority[5] = BRCMF_FWS_FIFO_AC_VI; - priority[6] = BRCMF_FWS_FIFO_AC_VO; - priority[7] = BRCMF_FWS_FIFO_AC_VO; -} - static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) { brcmf_fweh_register(cfg->pub, BRCMF_E_LINK, From c7532d6baf1aa5c4514d47334aaa3db71749ed91 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 27 Feb 2019 02:06:18 -0600 Subject: [PATCH 053/135] brcmfmac: fix continuous 802.1x tx pending timeout error The race condition in brcmf_msgbuf_txflow and brcmf_msgbuf_delete_flowring makes tx_msghdr writing after brcmf_msgbuf_remove_flowring. Host driver should delete flowring after txflow complete and all txstatus back, or pend_8021x_cnt will never be zero and cause every connection 950 milliseconds(MAX_WAIT_FOR_8021X_TX) delay. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/core.c | 4 +++- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 175f5447a2646..5bded9bb4a7dc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1407,8 +1407,10 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp) !brcmf_get_pend_8021x_cnt(ifp), MAX_WAIT_FOR_8021X_TX); - if (!err) + if (!err) { bphy_err(drvr, "Timed out waiting for no pending 802.1x packets\n"); + atomic_set(&ifp->pend_8021x_cnt, 0); + } return !err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index e3dd8623be4ec..1af9552d29dfc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -70,6 +70,7 @@ #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS 32 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS 48 +#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES 10 struct msgbuf_common_hdr { u8 msgtype; @@ -805,8 +806,12 @@ static int brcmf_msgbuf_tx_queue_data(struct brcmf_pub *drvr, int ifidx, flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx); if (flowid == BRCMF_FLOWRING_INVALID_ID) { flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb); - if (flowid == BRCMF_FLOWRING_INVALID_ID) + if (flowid == BRCMF_FLOWRING_INVALID_ID) { return -ENOMEM; + } else { + brcmf_flowring_enqueue(flow, flowid, skb); + return 0; + } } queue_count = brcmf_flowring_enqueue(flow, flowid, skb); force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0); @@ -1394,9 +1399,25 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; + struct brcmf_commonring *commonring_del; + void *ret_ptr; u8 ifidx; int err; + int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; + + /* wait for commonring txflow finished */ + commonring_del = msgbuf->flowrings[flowid]; + brcmf_commonring_lock(commonring_del); + while (retry && atomic_read(&commonring_del->outstanding_tx)) { + usleep_range(5000, 10000); + retry--; + } + brcmf_commonring_unlock(commonring_del); + if (!retry && atomic_read(&commonring_del->outstanding_tx)) { + brcmf_err("timed out waiting for txstatus\n"); + atomic_set(&commonring_del->outstanding_tx, 0); + } /* no need to submit if firmware can not be reached */ if (drvr->bus_if->state != BRCMF_BUS_UP) { From 5222fc26340ebc5a0b3ec0274c358011f14f0dd2 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 7 Mar 2019 01:37:27 -0600 Subject: [PATCH 054/135] brcmfmac: add sleep in bus suspend and cfg80211 resume functions With asynchronous suspend/resume feature, suspend and resume callbacks to be executed in parallel with each other. It makes bus changes the state to BRCMF_BUS_DOWN before all brcmf_cfg80211_suspend IOVAR executions. The same situation also happens in resume procedure and causes PM mode keeps in PM_MAX after resume. In order to fix the race condition, We add one second sleep in bus suspend and cfg80211 resume function. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 19 ++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 25 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 10 ++++++++ .../broadcom/brcm80211/brcmfmac/pcie.c | 12 +++++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 15 +++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 3ab5ab67f26c9..a9571071624b4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -36,6 +36,7 @@ #include "sdio.h" #include "core.h" #include "common.h" +#include "cfg80211.h" #define SDIOH_API_ACCESS_RETRY_LIMIT 2 @@ -1049,6 +1050,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, dev_set_drvdata(&func->dev, bus_if); dev_set_drvdata(&sdiodev->func1->dev, bus_if); sdiodev->dev = &sdiodev->func1->dev; + dev_set_drvdata(&sdiodev->func2->dev, bus_if); brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN); @@ -1065,6 +1067,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, fail: dev_set_drvdata(&func->dev, NULL); dev_set_drvdata(&sdiodev->func1->dev, NULL); + dev_set_drvdata(&sdiodev->func2->dev, NULL); kfree(sdiodev); kfree(bus_if); return err; @@ -1119,14 +1122,26 @@ static int brcmf_ops_sdio_suspend(struct device *dev) struct brcmf_bus *bus_if; struct brcmf_sdio_dev *sdiodev; mmc_pm_flag_t sdio_flags; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; func = container_of(dev, struct sdio_func, dev); + bus_if = dev_get_drvdata(dev); + config = bus_if->drvr->config; + brcmf_dbg(SDIO, "Enter: F%d\n", func->num); + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + if (func->num != 1) return 0; - - bus_if = dev_get_drvdata(dev); sdiodev = bus_if->bus_priv.sdio; brcmf_sdiod_freezer_on(sdiodev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ce2a79beb6bd0..9779ac47b6f3c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3647,10 +3647,24 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + struct brcmf_cfg80211_info *config = drvr->config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMING; + if (cfg->wowl.active) { + /* wait for bus resumed */ + while (retry && bus_if->state != BRCMF_BUS_UP) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && bus_if->state != BRCMF_BUS_UP) + brcmf_err("timed out wait for bus resume\n"); + brcmf_report_wowl_wakeind(wiphy, ifp); brcmf_fil_iovar_int_set(ifp, "wowl_clear", 0); brcmf_config_wowl_pattern(ifp, "clr", NULL, 0, NULL, 0); @@ -3671,6 +3685,7 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) brcmf_pktfilter_enable(ifp->ndev, false); } + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; return 0; } @@ -3740,9 +3755,12 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_vif *vif; + struct brcmf_cfg80211_info *config = ifp->drvr->config; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDING; + /* if the primary net_device is not READY there is nothing * we can do but pray resume goes smoothly. */ @@ -3784,9 +3802,13 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, } exit: - brcmf_dbg(TRACE, "Exit\n"); + /* set cfg80211 pm state to cfg80211 suspended state */ + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDED; + /* clear any scanning activity */ cfg->scan_status = 0; + + brcmf_dbg(TRACE, "Exit\n"); return 0; } @@ -7328,6 +7350,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->pub = drvr; + cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index c9b3345191016..0af3c1ae4208b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -92,6 +92,8 @@ #define BRCMF_VIF_EVENT_TIMEOUT msecs_to_jiffies(1500) +#define BRCMF_PM_WAIT_MAXRETRY 100 + /* cfg80211 wowlan definitions */ #define WL_WOWLAN_MAX_PATTERNS 8 #define WL_WOWLAN_MIN_PATTERN_LEN 1 @@ -170,6 +172,13 @@ enum brcmf_vif_status { BRCMF_VIF_STATUS_ASSOC_SUCCESS, }; +enum brcmf_cfg80211_pm_state { + BRCMF_CFG80211_PM_STATE_RESUMED, + BRCMF_CFG80211_PM_STATE_RESUMING, + BRCMF_CFG80211_PM_STATE_SUSPENDED, + BRCMF_CFG80211_PM_STATE_SUSPENDING, +}; + /** * struct vif_saved_ie - holds saved IEs for a virtual interface. * @@ -357,6 +366,7 @@ struct brcmf_cfg80211_info { struct brcmf_cfg80211_wowl wowl; struct brcmf_pno_info *pno; u8 ac_priority[MAX_8021D_PRIO]; + u8 pm_state; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 3e32347392537..5118d8ee3c692 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -38,6 +38,7 @@ #include "chip.h" #include "core.h" #include "common.h" +#include "cfg80211.h" enum brcmf_pcie_state { @@ -2011,11 +2012,22 @@ static int brcmf_pcie_pm_enter_D3(struct device *dev) { struct brcmf_pciedev_info *devinfo; struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(PCIE, "Enter\n"); bus = dev_get_drvdata(dev); devinfo = bus->bus_priv.pcie->devinfo; + config = bus->drvr->config; + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err(bus, "timed out wait for cfg80211 suspended\n"); brcmf_bus_change_state(bus, BRCMF_BUS_DOWN); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index f834561b24f1b..e910cc0294e48 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -19,6 +19,7 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "cfg80211.h" #define IOCTL_RESP_TIMEOUT msecs_to_jiffies(2000) @@ -1444,8 +1445,22 @@ static int brcmf_usb_suspend(struct usb_interface *intf, pm_message_t state) { struct usb_device *usb = interface_to_usbdev(intf); struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); + struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(USB, "Enter\n"); + + bus = devinfo->bus_pub.bus; + config = bus->drvr->config; + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP; if (devinfo->wowl_enabled) { brcmf_cancel_all_urbs(devinfo); From ef3542fb54708f0828830a9967cdf24865017a9c Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 7 Mar 2019 21:13:02 -0600 Subject: [PATCH 055/135] brcmfmac: fix 43455 CRC error under SDIO 3.0 SDR104 mode This patch fixes 43455 CRC error while running throughput test with suspend/resume stress test. The continuous failure messages before system crash: brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 25600 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 24576 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 0ebbecef20e0d..b73f4ef9fb5f6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -46,6 +46,10 @@ #define CY_4339_MES_WATERMARK 80 #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | \ SBSDIO_MESBUSYCTRL_ENAB) +#define CY_43455_F2_WATERMARK 0x60 +#define CY_43455_MES_WATERMARK 0x50 +#define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG @@ -4227,6 +4231,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err); break; + case SDIO_DEVICE_ID_BROADCOM_43455: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43455\n", + CY_43455_F2_WATERMARK); + brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, + CY_43455_F2_WATERMARK, &err); + devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43455_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From 21651b80fc6f14f4b47d24bb59a9e134471f46da Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 12 Dec 2019 00:52:46 +0100 Subject: [PATCH 056/135] brcmfmac: set F2 blocksize and watermark for 4359 commit e0029595fa0f7afe284b51dcf6d43704c2f9f88b upstream. Set F2 blocksize to 256 bytes and watermark to 0x40 for 4359. Also enable and configure F1 MesBusyCtrl. It fixes DMA error while having UDP bi-directional traffic. Signed-off-by: Chung-Hsien Hsu [slightly adapted for rebase on mainline linux] Signed-off-by: Soeren Moch [use switch-case to replace if-else for F2 blksz setting] Reviewed-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 11 ++++++++++- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index a9571071624b4..29197f3a28665 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -45,6 +45,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 +#define SDIO_4359_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -916,8 +917,16 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) goto out; } - if (sdiodev->func2->device == SDIO_DEVICE_ID_CYPRESS_4373) + switch (sdiodev->func2->device) { + case SDIO_DEVICE_ID_CYPRESS_4373: f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; + break; + case SDIO_DEVICE_ID_BROADCOM_4359: + f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; + break; + default: + break; + } ret = sdio_set_block_size(sdiodev->func2, f2_blksz); if (ret) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index b73f4ef9fb5f6..86d332bb911ab 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -50,6 +50,8 @@ #define CY_43455_MES_WATERMARK 0x50 #define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ SBSDIO_MESBUSYCTRL_ENAB) +#define CY_4359_F2_WATERMARK 0x40 +#define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG @@ -4244,6 +4246,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, CY_43455_MESBUSYCTRL, &err); break; + case SDIO_DEVICE_ID_BROADCOM_4359: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4359_F2_WATERMARK); + brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, + CY_4359_F2_WATERMARK, &err); + devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, + CY_4359_F1_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From f21168aaee48375bd803ff61d8e6874704a2d238 Mon Sep 17 00:00:00 2001 From: Amar Shankar Date: Mon, 15 Oct 2018 05:32:26 -0500 Subject: [PATCH 057/135] brcmfmac: reserve 2 credits for host tx control path It is observed that sometimes when sdiod is low in tx credits in low rssi scenarios, the data path consumes all sdiod rx all credits and there is no sdiod rx credit available for control path causing host and card to go out of sync resulting in link loss between host and card. So in order to prevent it some credits are reserved for control path. Note that TXCTL_CREDITS can't be larger than the firmware default credit update threshold 2; otherwise there will be a deadlock for both side waiting for each other. Signed-off-by: Amar Shankar Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/sdio.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 86d332bb911ab..13676cc709c76 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -643,6 +643,8 @@ static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; +#define TXCTL_CREDITS 2 + static void pkt_align(struct sk_buff *p, int len, int align) { uint datalign; @@ -653,8 +655,17 @@ static void pkt_align(struct sk_buff *p, int len, int align) __skb_trim(p, len); } -/* To check if there's window offered */ +/* To check if there's window offered + * Reserve 3 credits for txctl + */ static bool data_ok(struct brcmf_sdio *bus) +{ + return (u8)(bus->tx_max - bus->tx_seq) > TXCTL_CREDITS && + ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; +} + +/* To check if there's window offered */ +static bool txctl_ok(struct brcmf_sdio *bus) { return (u8)(bus->tx_max - bus->tx_seq) != 0 && ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; @@ -2660,7 +2671,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_sdio_clrintr(bus); if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && - data_ok(bus)) { + txctl_ok(bus)) { sdio_claim_host(bus->sdiodev->func1); if (bus->ctrl_frame_stat) { err = brcmf_sdio_tx_ctrlframe(bus, bus->ctrl_frame_buf, @@ -2668,6 +2679,9 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) bus->ctrl_frame_err = err; wmb(); bus->ctrl_frame_stat = false; + if (err) + brcmf_err("sdio ctrlframe tx failed err=%d\n", + err); } sdio_release_host(bus->sdiodev->func1); brcmf_sdio_wait_event_wakeup(bus); From f0ba8298ff0618f12632199ea3b88111c7f30356 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 20 Jun 2019 01:31:16 -0500 Subject: [PATCH 058/135] brcmfmac: update tx status flags to sync with firmware There is a mismatch of tx status flag values between host and firmware. It makes the host mistake the flags and have incorrect behavior of credit returns. So update the flags to sync with the firmware ones. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 86c6fc233bb47..74a6272e58d29 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -323,6 +323,10 @@ struct brcmf_skbuff_cb { * firmware suppress the packet as device is already in PS mode. * @BRCMF_FWS_TXSTATUS_FW_TOSSED: * firmware tossed the packet. + * @BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK: + * firmware tossed the packet after retries. + * @BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED: + * firmware wrongly reported suppressed previously, now fixing to acked. * @BRCMF_FWS_TXSTATUS_HOST_TOSSED: * host tossed the packet. */ @@ -331,6 +335,8 @@ enum brcmf_fws_txstatus { BRCMF_FWS_TXSTATUS_CORE_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_TOSSED, + BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK, + BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED, BRCMF_FWS_TXSTATUS_HOST_TOSSED }; @@ -1459,6 +1465,10 @@ brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_TOSSED) fws->stats.txs_tossed += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK) + fws->stats.txs_discard += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED) + fws->stats.txs_discard += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED) fws->stats.txs_host_tossed += compcnt; else From 2afcc6b9f8cabf437d677bebb47a3ca0dd67b61c Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 20 Jun 2019 02:59:26 -0500 Subject: [PATCH 059/135] brcmfmac: fix credit reserve for each access category Commit 7f52880c5ecd ("brcmfmac: Allow credit borrowing for all access categories") added a ratio to reserve the credits for each access category. However, calculating the number of reserved credits with the runtime one let an access category lend all its credits. Fix this by using the number of initial credits to determine the reserved one. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 74a6272e58d29..030452ca5fcac 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2036,7 +2036,7 @@ static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws, if (!borrow_all) borrow_limit = - fws->fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; + fws->init_fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; else borrow_limit = 0; From 9e1778d012c0c0e78563d5e12b30eca6d471b900 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 27 Jun 2019 04:00:02 -0500 Subject: [PATCH 060/135] brcmfmac: fix throughput zero stalls on PM 1 mode due to credit map This patch move the credit map setting to right place to avoid brcmf_fws_return_credits() return without setting the credit map. It fix the thoughput zero stalls issue in softAP mode when STA using PM 1 mode. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 030452ca5fcac..bed7cac20935c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1197,6 +1197,8 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (!credits) return; + fws->fifo_credit_map |= 1 << fifo; + if (fifo > BRCMF_FWS_FIFO_AC_BK && fifo <= BRCMF_FWS_FIFO_AC_VO) { for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0; @@ -1220,7 +1222,6 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (credits) { fws->fifo_credit[fifo] += credits; - fws->fifo_credit_map |= 1 << fifo; } if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) From bed0a2ea6ea850baa9e05d71d4271e836dbf7f23 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Sun, 18 Aug 2019 22:31:19 -0500 Subject: [PATCH 061/135] brcmfmac: 43012 Update MES Watermark Set MES watermark size to 0x50 for 43012. It fixes SDIO bus hang issue when running at high throughput. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 13676cc709c76..5c9b9b84bd4b2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -42,6 +42,9 @@ #define DEFAULT_F2_WATERMARK 0x8 #define CY_4373_F2_WATERMARK 0x40 #define CY_43012_F2_WATERMARK 0x60 +#define CY_43012_MES_WATERMARK 0x50 +#define CY_43012_MESBUSYCTRL (CY_43012_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #define CY_4339_F2_WATERMARK 48 #define CY_4339_MES_WATERMARK 80 #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | \ @@ -4233,6 +4236,8 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43012_MESBUSYCTRL, &err); break; case SDIO_DEVICE_ID_BROADCOM_4339: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 4339\n", From 144f7cff21be9c645bfad41d05b6b9349c079086 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 8 Oct 2019 01:28:40 -0500 Subject: [PATCH 062/135] brcmfmac: add support for CYW89359 SDIO chipset Add support for CYW89359 SDIO chipset. CYW89359 is a 2x2 dual-band 11ac chipset with 20/40/80Mhz channel support. Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 1 + drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 29197f3a28665..896546d482ac8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -922,6 +922,7 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; break; case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; break; default: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 5c9b9b84bd4b2..31b4fb25ebbe0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4266,6 +4266,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, CY_43455_MESBUSYCTRL, &err); break; case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", CY_4359_F2_WATERMARK); brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, From 4b24b3939aed3a77bfaa4fb8c3aaccba133d6419 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 12:13:09 -0600 Subject: [PATCH 063/135] brcmfmac: add CYW43570 PCIE device CYW43570 is a 3-antenna, 2x2 MIMO,802.11a/b/g/n/ac, PCIe 3.0 for WLAN. It is BT/WIFI combo. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 1 + drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 5118d8ee3c692..02e9b01c425b9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -2113,6 +2113,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_RAW_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4358_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4359_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_DEVICE_ID), diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index d1037b6ef2d6f..8919cdcfa48d1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -68,6 +68,7 @@ #define BRCM_PCIE_4356_DEVICE_ID 0x43ec #define BRCM_PCIE_43567_DEVICE_ID 0x43d3 #define BRCM_PCIE_43570_DEVICE_ID 0x43d9 +#define BRCM_PCIE_43570_RAW_DEVICE_ID 0xaa31 #define BRCM_PCIE_4358_DEVICE_ID 0x43e9 #define BRCM_PCIE_4359_DEVICE_ID 0x43ef #define BRCM_PCIE_43602_DEVICE_ID 0x43ba From eeb0edecb4dc05687244d7ce869e3b96401d49e6 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 13:05:35 -0600 Subject: [PATCH 064/135] brcmfmac: Use seq/seq_len and set iv_initialize when plumbing of rxiv in (GTK) keys When plumbing rxiv for (GTK) keys, current code does not use seq/seq_len when present nor set iv_initialized for iovar wsec_key. This could result in missing broadcast traffic after GTK rekey. The fix is setting iv_initialized and using seq/seq_len for iovar wsec_key. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 9779ac47b6f3c..d2c929945df48 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2331,6 +2331,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, if (!ext_key) key->flags = BRCMF_PRIMARY_KEY; + if (params->seq && params->seq_len == 6) { + /* rx iv */ + u8 *ivptr; + + ivptr = (u8 *)params->seq; + key->rxiv.hi = (ivptr[5] << 24) | (ivptr[4] << 16) | + (ivptr[3] << 8) | ivptr[2]; + key->rxiv.lo = (ivptr[1] << 8) | ivptr[0]; + key->iv_initialized = true; + } + switch (params->cipher) { case WLAN_CIPHER_SUITE_WEP40: key->algo = CRYPTO_ALGO_WEP1; From 5aee53d7d06139f07d6e2c189f175cf3bcfe72f6 Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Tue, 5 Nov 2019 12:39:56 +0900 Subject: [PATCH 065/135] brcmfmac: use actframe_abort to cancel ongoing action frame The driver sends an action frame down and waits for dwell time to be completed or aborted before sending out the next action frame. Driver issues "scan abort" to cancel the current time slot, but this doesn't have any effect because, we are not using scan engine for sending action frame. Fix is to use "actframe_abort" to cancels the current action frame. Signed-off-by: Ryohei Kondo Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/p2p.c | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 793b5a03ea274..a399714ee7b94 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1267,6 +1267,30 @@ bool brcmf_p2p_scan_finding_common_channel(struct brcmf_cfg80211_info *cfg, return true; } +/** + * brcmf_p2p_abort_action_frame() - abort action frame. + * + * @cfg: common configuration struct. + * + */ +static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_p2p_info *p2p = &cfg->p2p; + struct brcmf_cfg80211_vif *vif; + s32 err; + s32 int_val = 1; + + brcmf_dbg(TRACE, "Enter\n"); + + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val, + sizeof(s32)); + if (err) + brcmf_err(" aborting action frame has failed (%d)\n", err); + + return err; +} + /** * brcmf_p2p_stop_wait_next_action_frame() - finish scan if af tx complete. * @@ -1278,6 +1302,7 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) { struct brcmf_p2p_info *p2p = &cfg->p2p; struct brcmf_if *ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; + s32 err; if (test_bit(BRCMF_P2P_STATUS_SENDING_ACT_FRAME, &p2p->status) && (test_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status) || @@ -1286,8 +1311,13 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) /* if channel is not zero, "actfame" uses off channel scan. * So abort scan for off channel completion. */ - if (p2p->af_sent_channel) - brcmf_notify_escan_complete(cfg, ifp, true, true); + if (p2p->af_sent_channel) { + /* abort actframe using actframe_abort or abort scan */ + err = brcmf_p2p_abort_action_frame(cfg); + if (err) + brcmf_notify_escan_complete(cfg, ifp, true, + true); + } } else if (test_bit(BRCMF_P2P_STATUS_WAITING_NEXT_AF_LISTEN, &p2p->status)) { brcmf_dbg(TRACE, "*** Wake UP ** abort listen for next af frame\n"); From c11140cee374d41b26f51e0a2da9d8af4db6b5d2 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 5 Nov 2019 19:54:22 -0600 Subject: [PATCH 066/135] brcmfmac: fix scheduling while atomic issue when deleting flowring We should not sleep while holding the spin lock. It makes 'scheduling while atomic' in brcmf_msgbuf_delete_flowring. And to avoid race condition between deleting flowring and txflow, we only hold spin lock when seting flowring status to RING_CLOSING. Signed-off-by: Wright Feng Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/flowring.c | 5 +---- .../wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index 8e9d067bdfed0..b1727c442377e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -419,7 +419,6 @@ void brcmf_flowring_configure_addr_mode(struct brcmf_flowring *flow, int ifidx, flowid = flow->hash[i].flowid; if (flow->rings[flowid]->status != RING_OPEN) continue; - flow->rings[flowid]->status = RING_CLOSING; brcmf_msgbuf_delete_flowring(drvr, flowid); } } @@ -458,10 +457,8 @@ void brcmf_flowring_delete_peer(struct brcmf_flowring *flow, int ifidx, if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) && (hash[i].ifidx == ifidx)) { flowid = flow->hash[i].flowid; - if (flow->rings[flowid]->status == RING_OPEN) { - flow->rings[flowid]->status = RING_CLOSING; + if (flow->rings[flowid]->status == RING_OPEN) brcmf_msgbuf_delete_flowring(drvr, flowid); - } } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 1af9552d29dfc..5cecbe1286cb4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -1399,22 +1399,24 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; - struct brcmf_commonring *commonring_del; - + struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid]; + struct brcmf_flowring *flow = msgbuf->flow; void *ret_ptr; u8 ifidx; int err; int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; - /* wait for commonring txflow finished */ - commonring_del = msgbuf->flowrings[flowid]; + /* make sure it is not in txflow */ brcmf_commonring_lock(commonring_del); + flow->rings[flowid]->status = RING_CLOSING; + brcmf_commonring_unlock(commonring_del); + + /* wait for commonring txflow finished */ while (retry && atomic_read(&commonring_del->outstanding_tx)) { usleep_range(5000, 10000); retry--; } - brcmf_commonring_unlock(commonring_del); - if (!retry && atomic_read(&commonring_del->outstanding_tx)) { + if (!retry) { brcmf_err("timed out waiting for txstatus\n"); atomic_set(&commonring_del->outstanding_tx, 0); } From 0673a0148e09454dadcc4c8ab041a1b1878c64dc Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Wed, 6 Nov 2019 16:31:12 -0600 Subject: [PATCH 067/135] brcmfmac: increase message buffer size for control packets In wifi firmware, max length of IOCTL/IOVAR buffer size is 8192. Increase the message buffer max size same as wifi firmware for control packets so return buffers can come back. Signed-off-by: Soontak Lee Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 5cecbe1286cb4..9f1e8adde8e33 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -54,6 +54,7 @@ #define BRCMF_IOCTL_REQ_PKTID 0xFFFE #define BRCMF_MSGBUF_MAX_PKT_SIZE 2048 +#define BRCMF_MSGBUF_MAX_CTL_PKT_SIZE 8192 #define BRCMF_MSGBUF_RXBUFPOST_THRESHOLD 32 #define BRCMF_MSGBUF_MAX_IOCTLRESPBUF_POST 8 #define BRCMF_MSGBUF_MAX_EVENTBUF_POST 8 @@ -1033,7 +1034,7 @@ brcmf_msgbuf_rxbuf_ctrl_post(struct brcmf_msgbuf *msgbuf, bool event_buf, rx_bufpost = (struct msgbuf_rx_ioctl_resp_or_event *)ret_ptr; memset(rx_bufpost, 0, sizeof(*rx_bufpost)); - skb = brcmu_pkt_buf_get_skb(BRCMF_MSGBUF_MAX_PKT_SIZE); + skb = brcmu_pkt_buf_get_skb(BRCMF_MSGBUF_MAX_CTL_PKT_SIZE); if (skb == NULL) { bphy_err(drvr, "Failed to alloc SKB\n"); From dfba28e3f95c9a82606fad86d2a23075f4f234d8 Mon Sep 17 00:00:00 2001 From: "alep@cypress.com" Date: Fri, 26 Jul 2019 04:03:06 -0500 Subject: [PATCH 068/135] brcmfmac: Support 89459 pcie Adds support of 89459 chip pcie device and save restore support. Signed-off-by: alep@cypress.com Signed-off-by: Joseph chuang Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 3 +++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 4 ++++ drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 85cf4088acb48..2e9b3248973cb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -724,6 +724,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) return 0x200000; case CY_CC_4373_CHIP_ID: return 0x160000; + case CY_CC_89459_CHIP_ID: + return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000); default: brcmf_err("unknown chip: %s\n", ci->pub.name); break; @@ -1407,6 +1409,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) reg = chip->ops->read32(chip->ctx, addr); return reg != 0; case CY_CC_4373_CHIP_ID: + case CY_CC_89459_CHIP_ID: /* explicitly check SR engine enable bit */ addr = CORE_CC_REG(base, sr_control0); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 02e9b01c425b9..681270417b597 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -58,6 +58,7 @@ BRCMF_FW_DEF(4365C, "brcmfmac4365c-pcie"); BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie"); BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); +BRCMF_FW_DEF(4355, "brcmfmac89459-pcie"); static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -77,6 +78,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), + BRCMF_FW_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355), }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -2128,6 +2130,8 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 8919cdcfa48d1..8ee9a04957054 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -50,6 +50,7 @@ #define BRCM_CC_4371_CHIP_ID 0x4371 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 +#define CY_CC_89459_CHIP_ID 0x4355 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e @@ -82,7 +83,8 @@ #define BRCM_PCIE_4366_2G_DEVICE_ID 0x43c4 #define BRCM_PCIE_4366_5G_DEVICE_ID 0x43c5 #define BRCM_PCIE_4371_DEVICE_ID 0x440d - +#define CY_PCIE_89459_DEVICE_ID 0x4415 +#define CY_PCIE_89459_RAW_DEVICE_ID 0x4355 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ From 49221488d880f92003fb73cf97a2079bac6ad40d Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Thu, 7 Nov 2019 12:49:40 -0600 Subject: [PATCH 069/135] brcmfmac: Fix for unable to return to visible SSID Unable to change back to visiable SSID because there is no disable hidden ssid routine. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d2c929945df48..89ed0c78ce0cd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4708,12 +4708,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, goto exit; } - if (settings->hidden_ssid) { - err = brcmf_fil_iovar_int_set(ifp, "closednet", 1); - if (err) { - bphy_err(drvr, "closednet error (%d)\n", err); - goto exit; - } + err = brcmf_fil_iovar_int_set(ifp, "closednet", + settings->hidden_ssid); + if (err) { + bphy_err(drvr, "%s closednet error (%d)\n", + settings->hidden_ssid ? + "enabled" : "disabled", + err); + goto exit; } brcmf_dbg(TRACE, "AP mode configuration complete\n"); From 8547859c2f1fe65416c4db59b46644439d5f08eb Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Wed, 6 Nov 2019 18:57:12 -0600 Subject: [PATCH 070/135] brcmfmac: Fix for wrong disconnection event source information. Add event source argument on link down handler. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 89ed0c78ce0cd..8f9d865df45a6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1283,7 +1283,8 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } -static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) +static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, + bool locally_generated) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy); struct brcmf_pub *drvr = cfg->pub; @@ -1305,7 +1306,7 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) if ((vif->wdev.iftype == NL80211_IFTYPE_STATION) || (vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT)) cfg80211_disconnected(vif->wdev.netdev, reason, NULL, 0, - true, GFP_KERNEL); + locally_generated, GFP_KERNEL); } clear_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state); clear_bit(BRCMF_SCAN_STATUS_SUPPRESS, &cfg->scan_status); @@ -1484,7 +1485,7 @@ brcmf_cfg80211_leave_ibss(struct wiphy *wiphy, struct net_device *ndev) return 0; } - brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING); + brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING, true); brcmf_net_setcarrier(ifp, false); brcmf_dbg(TRACE, "Exit\n"); @@ -3796,7 +3797,7 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, * disassociate from AP to save power while system is * in suspended state */ - brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event * generated due to DISASSOC call to the fw to keep * the state fw and WPA_Supplicant state consistent @@ -5920,7 +5921,11 @@ brcmf_notify_connect_status(struct brcmf_if *ifp, if (!brcmf_is_ibssmode(ifp->vif)) { brcmf_bss_connect_done(cfg, ndev, e, false); brcmf_link_down(ifp->vif, - brcmf_map_fw_linkdown_reason(e)); + brcmf_map_fw_linkdown_reason(e), + e->event_code & + (BRCMF_E_DEAUTH_IND | + BRCMF_E_DISASSOC_IND) + ? false : true); brcmf_init_prof(ndev_to_prof(ndev)); if (ndev != cfg_to_ndev(cfg)) complete(&cfg->vif_disabled); @@ -7119,7 +7124,7 @@ static s32 __brcmf_cfg80211_down(struct brcmf_if *ifp) * from AP to save power */ if (check_vif_up(ifp->vif)) { - brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event generated due to DISASSOC call to the fw to keep From 97c34ff959e42af4bcbead3407901743189320c9 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 2 Oct 2019 09:31:12 +0000 Subject: [PATCH 071/135] brcmfmac: add support for SAE authentication offload commit 3b1e0a7bdfeedbde49b8d424aeb88ac3c0cf8182 upstream. The firmware may have SAE authentication code built-in. This is detected by the driver and indicated in the wiphy features flags. User-space can use this flag to determine whether or not to provide the password material for SAE authentication in the nl80211 CONNECT command. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin Signed-off-by: Kalle Valo --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 73 +++++++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 3 +- .../broadcom/brcm80211/brcmfmac/feature.c | 3 +- .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ++++ .../broadcom/brcm80211/include/brcmu_wifi.h | 2 + 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 8f9d865df45a6..cf5e12efaab5a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1283,6 +1283,31 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } +static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, + u16 pwd_len) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_wsec_sae_pwd_le sae_pwd; + int err; + + if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { + bphy_err(drvr, "sae_password must be less than %d\n", + BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); + return -EINVAL; + } + + sae_pwd.key_len = cpu_to_le16(pwd_len); + memcpy(sae_pwd.key, pwd_data, pwd_len); + + err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, + sizeof(sae_pwd)); + if (err < 0) + bphy_err(drvr, "failed to set SAE password in firmware (len=%u)\n", + pwd_len); + + return err; +} + static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, bool locally_generated) { @@ -1507,6 +1532,8 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2) val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; + else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) + val = WPA3_AUTH_SAE_PSK; else val = WPA_AUTH_DISABLED; brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); @@ -1539,6 +1566,10 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, val = 1; brcmf_dbg(CONN, "shared key\n"); break; + case NL80211_AUTHTYPE_SAE: + val = 3; + brcmf_dbg(CONN, "SAE authentication\n"); + break; default: val = 2; brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type); @@ -1707,6 +1738,16 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) sme->crypto.cipher_group); return -EINVAL; } + } else if (val & WPA3_AUTH_SAE_PSK) { + switch (sme->crypto.akm_suites[0]) { + case WLAN_AKM_SUITE_SAE: + val = WPA3_AUTH_SAE_PSK; + break; + default: + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); + return -EINVAL; + } } if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) @@ -1778,7 +1819,8 @@ brcmf_set_sharedkey(struct net_device *ndev, brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n", sec->wpa_versions, sec->cipher_pairwise); - if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2)) + if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)) return 0; if (!(sec->cipher_pairwise & @@ -1985,7 +2027,13 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.psk) { + if (sme->crypto.sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + } + + if (sme->crypto.psk && + profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { err = -EINVAL; goto done; @@ -2003,12 +2051,23 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) { + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); - if (err) + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + bphy_err(drvr, "failed to clean up user-space RSNE\n"); goto done; + } + err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, + sme->crypto.sae_pwd_len); + if (!err && sme->crypto.psk) + err = brcmf_set_pmk(ifp, sme->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); } + if (err) + goto done; /* Join with specific BSSID and cached SSID * If SSID is zero join based on BSSID only @@ -5443,7 +5502,8 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); - if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK) + if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK && + vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE) return true; set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); @@ -6977,6 +7037,9 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 0af3c1ae4208b..31e3a55e4c902 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -133,7 +133,8 @@ struct brcmf_cfg80211_security { enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, - BRCMF_PROFILE_FWSUP_1X + BRCMF_PROFILE_FWSUP_1X, + BRCMF_PROFILE_FWSUP_SAE }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 565db6c11bcd0..d71273428a7b2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -38,7 +38,8 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_P2P, "p2p" }, { BRCMF_FEAT_MONITOR, "monitor" }, { BRCMF_FEAT_MONITOR_FMT_RADIOTAP, "rtap" }, - { BRCMF_FEAT_DOT11H, "802.11h" } + { BRCMF_FEAT_DOT11H, "802.11h" }, + { BRCMF_FEAT_SAE, "sae" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 736a8179f62f6..280a1f6412d49 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -26,6 +26,7 @@ * MONITOR_FMT_RADIOTAP: firmware provides monitor packets with radiotap header * MONITOR_FMT_HW_RX_HDR: firmware provides monitor packets with hw/ucode header * DOT11H: firmware supports 802.11h + * SAE: simultaneous authentication of equals */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -45,7 +46,8 @@ BRCMF_FEAT_DEF(MONITOR) \ BRCMF_FEAT_DEF(MONITOR_FMT_RADIOTAP) \ BRCMF_FEAT_DEF(MONITOR_FMT_HW_RX_HDR) \ - BRCMF_FEAT_DEF(DOT11H) + BRCMF_FEAT_DEF(DOT11H) \ + BRCMF_FEAT_DEF(SAE) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index f504fd231dfd7..b7c16dc605c49 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -61,6 +61,8 @@ #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 + /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 @@ -531,6 +533,17 @@ struct brcmf_wsec_pmk_le { u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; }; +/** + * struct brcmf_wsec_sae_pwd_le - firmware SAE password material. + * + * @key_len: number of octets in key materials. + * @key: SAE password material. + */ +struct brcmf_wsec_sae_pwd_le { + __le16 key_len; + u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; +}; + /* Used to get specific STA parameters */ struct brcmf_scb_val_le { __le32 val; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 7b31c212694da..7552bdb91991c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -231,6 +231,8 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ +#define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ + #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 From fe050f828f1c95387ae3be5ff202ffc922b1f53c Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Wed, 6 Nov 2019 17:36:25 -0600 Subject: [PATCH 072/135] brcmfmac: Support multiple AP interfaces and fix STA disconnection issue Support multiple AP interfaces for STA + AP + AP usecase. And fix STA disconnection when deactivating AP interface. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 48 +++++++++++++++---- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + .../broadcom/brcm80211/brcmfmac/common.c | 5 ++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index cf5e12efaab5a..c6258d038c0c2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4608,6 +4608,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + brcmf_dbg(TRACE, "mbss %s\n", mbss ? "enabled" : "disabled"); /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -4814,6 +4815,9 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if ((err) && (!mbss)) { brcmf_set_mpc(ifp, 1); brcmf_configure_arp_nd_offload(ifp, true); + } else { + cfg->num_softap++; + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); } return err; } @@ -4826,6 +4830,7 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) s32 err; struct brcmf_fil_bss_enable_le bss_enable; struct brcmf_join_params join_params; + s32 apsta = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -4834,6 +4839,27 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) /* first to make sure they get processed by fw. */ msleep(400); + cfg->num_softap--; + + /* Clear bss configuration and SSID */ + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(0); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) + brcmf_err("bss_enable config failed %d\n", err); + + memset(&join_params, 0, sizeof(join_params)); + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, + &join_params, sizeof(join_params)); + if (err < 0) + bphy_err(drvr, "SET SSID error (%d)\n", err); + + if (cfg->num_softap) { + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); + return 0; + } + if (ifp->vif->mbss) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); return err; @@ -4843,17 +4869,18 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) if (ifp->bsscfgidx == 0) brcmf_fil_iovar_int_set(ifp, "closednet", 0); - memset(&join_params, 0, sizeof(join_params)); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, sizeof(join_params)); - if (err < 0) - bphy_err(drvr, "SET SSID error (%d)\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); - if (err < 0) - bphy_err(drvr, "BRCMF_C_DOWN error %d\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); + err = brcmf_fil_iovar_int_get(ifp, "apsta", &apsta); if (err < 0) - bphy_err(drvr, "setting AP mode failed %d\n", err); + brcmf_err("wl apsta failed (%d)\n", err); + + if (!apsta) { + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); + if (err < 0) + bphy_err(drvr, "BRCMF_C_DOWN error %d\n", err); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); + if (err < 0) + bphy_err(drvr, "Set AP mode error %d\n", err); + } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) brcmf_fil_iovar_int_set(ifp, "mbss", 0); brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY, @@ -7432,6 +7459,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->pub = drvr; cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; + cfg->num_softap = 0; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 31e3a55e4c902..74287a3487ca2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -368,6 +368,7 @@ struct brcmf_cfg80211_info { struct brcmf_pno_info *pno; u8 ac_priority[MAX_8021D_PRIO]; u8 pm_state; + u8 num_softap; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 7805bbacc80ae..75beeb4c95485 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -305,6 +305,11 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_dbg(INFO, "CLM version = %s\n", clmver); } + /* set apsta */ + err = brcmf_fil_iovar_int_set(ifp, "apsta", 1); + if (err) + brcmf_info("failed setting apsta, %d\n", err); + /* set mpc */ err = brcmf_fil_iovar_int_set(ifp, "mpc", 1); if (err) { From 7ab662579f391d8bb048f03e7a045ca2a3b030c7 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 18:33:29 -0600 Subject: [PATCH 073/135] brcmfmac: Support custom PCIE BAR window size. Certain host processors cannot support 4MB PCIE BAR window size. For example, AMLogic A113D can support 2MB size only. This patch is for host processor which support lower than 4MB PCIE BAR window size. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../net/wireless/broadcom/brcm80211/Kconfig | 11 + .../broadcom/brcm80211/brcmfmac/pcie.c | 242 +++++++++++++++++- 2 files changed, 247 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig index a5bf16c4f4951..b239e067bba86 100644 --- a/drivers/net/wireless/broadcom/brcm80211/Kconfig +++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig @@ -37,3 +37,14 @@ config BRCMDBG select WANT_DEV_COREDUMP if BRCMFMAC ---help--- Selecting this enables additional code for debug purposes. + +config BRCMFMAC_PCIE_BARWIN_SZ + bool "Custom PCIE BAR window size support for FullMAC driver" + depends on BRCMFMAC + depends on PCI + default n + ---help--- + If you say Y here, the FMAC driver will use custom PCIE BAR + window size. Say Y to allow developers to use custom PCIE + BAR window size when HOST PCIE IP can support less then 4MB + BAR window. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 681270417b597..e3f1cd536f1ed 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -272,6 +272,9 @@ struct brcmf_pciedev_info { void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value); struct brcmf_mp_device *settings; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + ulong bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ }; struct brcmf_pcie_ringbuf { @@ -343,6 +346,10 @@ static void brcmf_pcie_setup(struct device *dev, int ret, static struct brcmf_fw_request * brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ +DEFINE_RAW_SPINLOCK(pcie_lock); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ + static u32 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) { @@ -366,8 +373,24 @@ static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + u8 value; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread8(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread8(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -375,8 +398,24 @@ static u16 brcmf_pcie_read_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u16 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread16(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread16(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -385,8 +424,22 @@ brcmf_pcie_write_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } iowrite16(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite16(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -413,8 +466,24 @@ static u32 brcmf_pcie_read_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -423,17 +492,47 @@ brcmf_pcie_write_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { void __iomem *address = devinfo->tcm + mem_offset; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } static u32 brcmf_pcie_read_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); - return (ioread32(addr)); + return value; +#else + return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -441,9 +540,23 @@ static void brcmf_pcie_write_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; - iowrite32(value, addr); + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -455,12 +568,30 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *src32; __le16 *src16; u8 *src8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)srcaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)srcaddr & 2) || (len & 2)) { src8 = (u8 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite8(*src8, address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; src8++; len--; @@ -469,7 +600,22 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; src16 = (__le16 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite16(le16_to_cpu(*src16), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; src16++; len--; @@ -479,12 +625,29 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; src32 = (__le32 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite32(le32_to_cpu(*src32), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; src32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -496,12 +659,30 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *dst32; __le16 *dst16; u8 *dst8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)dstaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)dstaddr & 2) || (len & 2)) { dst8 = (u8 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst8 = ioread8(address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; dst8++; len--; @@ -510,7 +691,22 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; dst16 = (__le16 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst16 = cpu_to_le16(ioread16(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; dst16++; len--; @@ -520,12 +716,29 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; dst32 = (__le32 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst32 = cpu_to_le32(ioread32(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; dst32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -569,6 +782,9 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) BRCMF_PCIE_CFGREG_MSI_ADDR_L, BRCMF_PCIE_CFGREG_MSI_ADDR_H, BRCMF_PCIE_CFGREG_MSI_DATA, +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + BCMA_PCI_BAR1_WIN, +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ BRCMF_PCIE_CFGREG_LINK_STATUS_CTRL2, BRCMF_PCIE_CFGREG_RBAR_CTRL, BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1, @@ -1139,9 +1355,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) u16 max_flowrings; u16 max_submissionrings; u16 max_completionrings; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_dev_tomem(devinfo, devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_fromio(&ringinfo, devinfo->tcm + devinfo->shared.ring_info_addr, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ + if (devinfo->shared.version >= 6) { max_submissionrings = le16_to_cpu(ringinfo.max_submissionrings); max_flowrings = le16_to_cpu(ringinfo.max_flowrings); @@ -1210,8 +1431,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) ringinfo.d2h_r_idx_hostaddr.high_addr = cpu_to_le32(address >> 32); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_mem_todev(devinfo, + devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_toio(devinfo->tcm + devinfo->shared.ring_info_addr, &ringinfo, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ brcmf_dbg(PCIE, "Using host memory indices\n"); } @@ -1650,6 +1877,9 @@ static int brcmf_pcie_get_resource(struct brcmf_pciedev_info *devinfo) devinfo->regs = ioremap_nocache(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE); devinfo->tcm = ioremap_nocache(bar1_addr, bar1_size); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + devinfo->bar1_size = bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (!devinfo->regs || !devinfo->tcm) { brcmf_err(bus, "ioremap() failed (%p,%p)\n", devinfo->regs, From b7fc80713448317b67e220339a6bb9a247fdb1a9 Mon Sep 17 00:00:00 2001 From: Frank Kao Date: Mon, 25 Nov 2019 15:41:13 +0800 Subject: [PATCH 074/135] brcmfmac: set F2 blocksize and watermark for 4354 Set F2 blocksize to 256 bytes and watermark to 0x40 for 4354. Also enable and configure F1 MesBusyCtrl. It would resolve random driver crash issue. Signed-off-by: Frank Kao Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 4 ++++ .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 896546d482ac8..570210b915663 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -46,6 +46,7 @@ #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 #define SDIO_4359_FUNC2_BLOCKSIZE 256 +#define SDIO_4354_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -925,6 +926,9 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) case SDIO_DEVICE_ID_CYPRESS_89359: f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; break; + case SDIO_DEVICE_ID_BROADCOM_4354: + f2_blksz = SDIO_4354_FUNC2_BLOCKSIZE; + break; default: break; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 31b4fb25ebbe0..c68b5a491d18e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -55,6 +55,8 @@ SBSDIO_MESBUSYCTRL_ENAB) #define CY_4359_F2_WATERMARK 0x40 #define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_4354_F2_WATERMARK 0x40 +#define CY_4354_F1_MESBUSYCTRL (CY_4354_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG @@ -4279,6 +4281,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, CY_4359_F1_MESBUSYCTRL, &err); break; + case SDIO_DEVICE_ID_BROADCOM_4354: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4354_F2_WATERMARK); + brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, + CY_4354_F2_WATERMARK, &err); + devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, + CY_4354_F1_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From b62068580ba167ef44d1c26eb768719df290e03d Mon Sep 17 00:00:00 2001 From: "Lo(Double)Hsiang Lo" Date: Mon, 9 Dec 2019 20:49:17 -0600 Subject: [PATCH 075/135] brcmfmac: support for virtual interface creation from firmware Allow interface creation via IF_ADD event from firmware. Signed-off-by: Lo(Double)Hsiang Lo Signed-off-by: Chi-Hsien Lin SWLINUX-1291 --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index c6258d038c0c2..edf7d52bccff9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -229,6 +229,9 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; +#define WLC_E_IF_ROLE_STA 0 /* Infra STA */ +#define WLC_E_IF_ROLE_AP 1 /* Access Point */ + static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -5512,8 +5515,10 @@ void brcmf_cfg80211_free_netdev(struct net_device *ndev) ifp = netdev_priv(ndev); vif = ifp->vif; - if (vif) + if (vif) { brcmf_free_vif(vif); + ifp->vif = NULL; + } } static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, @@ -6075,6 +6080,9 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, struct brcmf_if_event *ifevent = (struct brcmf_if_event *)data; struct brcmf_cfg80211_vif_event *event = &cfg->vif_event; struct brcmf_cfg80211_vif *vif; + enum nl80211_iftype iftype = NL80211_IFTYPE_UNSPECIFIED; + bool vif_pend = false; + int err; brcmf_dbg(TRACE, "Enter: action %u flags %u ifidx %u bsscfgidx %u\n", ifevent->action, ifevent->flags, ifevent->ifidx, @@ -6087,9 +6095,28 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, switch (ifevent->action) { case BRCMF_E_IF_ADD: /* waiting process may have timed out */ - if (!cfg->vif_event.vif) { + if (!vif) { + /* handle IF_ADD event from firmware */ spin_unlock(&event->vif_event_lock); - return -EBADF; + vif_pend = true; + if (ifevent->role == WLC_E_IF_ROLE_STA) + iftype = NL80211_IFTYPE_STATION; + else if (ifevent->role == WLC_E_IF_ROLE_AP) + iftype = NL80211_IFTYPE_AP; + else + vif_pend = false; + + if (vif_pend) { + vif = brcmf_alloc_vif(cfg, iftype); + if (IS_ERR(vif)) { + brcmf_err("Role:%d failed to alloc vif\n", + ifevent->role); + return PTR_ERR(vif); + } + } else { + brcmf_err("Invalid Role:%d\n", ifevent->role); + return -EBADF; + } } ifp->vif = vif; @@ -6099,6 +6126,18 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, ifp->ndev->ieee80211_ptr = &vif->wdev; SET_NETDEV_DEV(ifp->ndev, wiphy_dev(cfg->wiphy)); } + + if (vif_pend) { + err = brcmf_net_attach(ifp, false); + if (err) { + brcmf_err("netdevice register failed with err:%d\n", + err); + brcmf_free_vif(vif); + free_netdev(ifp->ndev); + } + return err; + } + spin_unlock(&event->vif_event_lock); wake_up(&event->vif_wq); return 0; From 8fa27412b73eefba3d63a7fc8da62680782f43aa Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Thu, 12 Dec 2019 00:02:45 -0600 Subject: [PATCH 076/135] brcmfmac: set security after reiniting interface Host driver parses and sets security params into FW passed by supplicant. This has to be done after reiniting interface in the firmware. Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 90 ++++++++++++------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index edf7d52bccff9..90ee39671239a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4580,6 +4580,48 @@ brcmf_config_ap_mgmt_ie(struct brcmf_cfg80211_vif *vif, return err; } +static s32 +brcmf_parse_configure_security(struct brcmf_if *ifp, + struct cfg80211_ap_settings *settings, + enum nl80211_iftype dev_role) +{ + const struct brcmf_tlv *rsn_ie; + const struct brcmf_vs_tlv *wpa_ie; + s32 err = 0; + + /* find the RSN_IE */ + rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, + settings->beacon.tail_len, WLAN_EID_RSN); + + /* find the WPA_IE */ + wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, + settings->beacon.tail_len); + + if (wpa_ie || rsn_ie) { + brcmf_dbg(TRACE, "WPA(2) IE is found\n"); + if (wpa_ie) { + /* WPA IE */ + err = brcmf_configure_wpaie(ifp, wpa_ie, false); + if (err < 0) + return err; + } else { + struct brcmf_vs_tlv *tmp_ie; + + tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; + + /* RSN IE */ + err = brcmf_configure_wpaie(ifp, tmp_ie, true); + if (err < 0) + return err; + } + } else { + brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); + brcmf_configure_opensecurity(ifp); + } + + return err; +} + static s32 brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, struct cfg80211_ap_settings *settings) @@ -4592,8 +4634,6 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, const struct brcmf_tlv *country_ie; struct brcmf_ssid_le ssid_le; s32 err = -EPERM; - const struct brcmf_tlv *rsn_ie; - const struct brcmf_vs_tlv *wpa_ie; struct brcmf_join_params join_params; enum nl80211_iftype dev_role; struct brcmf_fil_bss_enable_le bss_enable; @@ -4648,36 +4688,6 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, brcmf_configure_arp_nd_offload(ifp, false); } - /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, - settings->beacon.tail_len, WLAN_EID_RSN); - - /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, - settings->beacon.tail_len); - - if ((wpa_ie != NULL || rsn_ie != NULL)) { - brcmf_dbg(TRACE, "WPA(2) IE is found\n"); - if (wpa_ie != NULL) { - /* WPA IE */ - err = brcmf_configure_wpaie(ifp, wpa_ie, false); - if (err < 0) - goto exit; - } else { - struct brcmf_vs_tlv *tmp_ie; - - tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; - - /* RSN IE */ - err = brcmf_configure_wpaie(ifp, tmp_ie, true); - if (err < 0) - goto exit; - } - } else { - brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); - brcmf_configure_opensecurity(ifp); - } - /* Parameters shared by all radio interfaces */ if (!mbss) { if ((supports_11d) && (is_11d != ifp->vif->is_11d)) { @@ -4756,6 +4766,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, bphy_err(drvr, "BRCMF_C_UP error (%d)\n", err); goto exit; } + + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_AP); + if (err < 0) { + bphy_err(drvr, "brcmf_parse_configure_security error\n"); + goto exit; + } + /* On DOWN the firmware removes the WEP keys, reconfigure * them if they were set. */ @@ -4790,6 +4808,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, chanspec, err); goto exit; } + + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_P2P_GO); + if (err < 0) { + brcmf_err("brcmf_parse_configure_security error\n"); + goto exit; + } + err = brcmf_fil_bsscfg_data_set(ifp, "ssid", &ssid_le, sizeof(ssid_le)); if (err < 0) { From 9ee88be655fae76e4b7248d6753cd6c918204cc1 Mon Sep 17 00:00:00 2001 From: "Lo(Double)Hsiang Lo" Date: Mon, 16 Dec 2019 03:41:09 -0600 Subject: [PATCH 077/135] brcmfmac: increase dcmd maximum buffer size Increase dcmd maximum buffer size to match firmware configuration for new chips. Signed-off-by: Lo(Double)Hsiang Lo Signed-off-by: Chi-Hsien Lin SWLINUX-1273 --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 2c95a08a58711..705130cf27d59 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -87,6 +87,8 @@ struct brcmf_proto_bcdc_header { * plus any space that might be needed * for bus alignment padding. */ +#define ROUND_UP_MARGIN 2048 + struct brcmf_bcdc { u16 reqid; u8 bus_header[BUS_HEADER_LEN]; @@ -471,7 +473,7 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES; drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN + - sizeof(struct brcmf_proto_bcdc_dcmd); + sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN; return 0; fail: From 0e8bb4017edb11cb41a0523c60192213e5d99e49 Mon Sep 17 00:00:00 2001 From: Frank Kao Date: Wed, 18 Dec 2019 16:11:14 +0800 Subject: [PATCH 078/135] brcmfmac: set F2 blocksize and watermark for 4356 SDIO Set F2 blocksize to 256 bytes and watermark to 0x40 for 4356 SDIO. Also enable and configure F1 MesBusyCtrl. It would resolve random driver crash issue. Signed-off-by: Frank Kao Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 8 +++--- .../broadcom/brcm80211/brcmfmac/sdio.c | 27 +++++-------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 570210b915663..3ad922494a791 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -45,8 +45,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 -#define SDIO_4359_FUNC2_BLOCKSIZE 256 -#define SDIO_4354_FUNC2_BLOCKSIZE 256 +#define SDIO_435X_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -924,10 +923,9 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) break; case SDIO_DEVICE_ID_BROADCOM_4359: case SDIO_DEVICE_ID_CYPRESS_89359: - f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; - break; case SDIO_DEVICE_ID_BROADCOM_4354: - f2_blksz = SDIO_4354_FUNC2_BLOCKSIZE; + case SDIO_DEVICE_ID_BROADCOM_4356: + f2_blksz = SDIO_435X_FUNC2_BLOCKSIZE; break; default: break; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index c68b5a491d18e..de090ee148288 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -53,11 +53,9 @@ #define CY_43455_MES_WATERMARK 0x50 #define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ SBSDIO_MESBUSYCTRL_ENAB) -#define CY_4359_F2_WATERMARK 0x40 -#define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) -#define CY_4354_F2_WATERMARK 0x40 -#define CY_4354_F1_MESBUSYCTRL (CY_4354_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) - +#define CY_435X_F2_WATERMARK 0x40 +#define CY_435X_F1_MESBUSYCTRL (CY_435X_F2_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -4269,30 +4267,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, break; case SDIO_DEVICE_ID_BROADCOM_4359: case SDIO_DEVICE_ID_CYPRESS_89359: - brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", - CY_4359_F2_WATERMARK); - brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, - CY_4359_F2_WATERMARK, &err); - devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, - &err); - devctl |= SBSDIO_DEVCTL_F2WM_ENAB; - brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, - &err); - brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, - CY_4359_F1_MESBUSYCTRL, &err); - break; case SDIO_DEVICE_ID_BROADCOM_4354: + case SDIO_DEVICE_ID_BROADCOM_4356: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", - CY_4354_F2_WATERMARK); + CY_435X_F2_WATERMARK); brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, - CY_4354_F2_WATERMARK, &err); + CY_435X_F2_WATERMARK, &err); devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, &err); devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, &err); brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, - CY_4354_F1_MESBUSYCTRL, &err); + CY_435X_F1_MESBUSYCTRL, &err); break; default: brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, From 53c5ff0d31befec249b00cfbd0d642531bcf11b6 Mon Sep 17 00:00:00 2001 From: Kurt Lee/TAIPEI Date: Tue, 31 Dec 2019 03:47:08 -0600 Subject: [PATCH 079/135] brcmfmac: set net carrier on via test tool for AP mode Host parses ioctl cmd via test tool, then set itself iftype to ap mode and report netif_carrier_on to upper layer Signed-off-by: Kurt Lee Signed-off-by: Chi-Hsien Lin SWLINUX-1322 --- .../net/wireless/broadcom/brcm80211/brcmfmac/vendor.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 4ed3287522cbe..0bad78d5f5c98 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,6 +64,15 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + if (cmdhdr->cmd == BRCMF_C_SET_AP) { + if (*(int *)(dcmd_buf) == 1) { + ifp->vif->wdev.iftype = NL80211_IFTYPE_AP; + brcmf_net_setcarrier(ifp, true); + } else { + ifp->vif->wdev.iftype = NL80211_IFTYPE_STATION; + } + } + if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); From dbfdba9b253fef23cb8a3390641a2c4aedde57a6 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 14 Feb 2019 16:33:53 +0800 Subject: [PATCH 080/135] nl80211: add authorized flag back to ROAM event Commit 503c1fb98ba3 ("cfg80211/nl80211: add a port authorized event") added the NL80211_CMD_PORT_AUTHORIZED event to indicate that a connection is authorized. It replaced the PORT_AUTHORIZED attribute and the authorized flag added in commit f45cbe6e691f ("nl80211: add authorized flag to ROAM event"). However, for offload FT, using PORT_AUTHORIZED event mechanism induces wpa_supplicant to start a full EAP exchange after a successful roaming. This patch adds the flag mechanism back to the ROAM event for drivers to fix the offload FT roaming issue. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- include/net/cfg80211.h | 4 ++++ include/uapi/linux/nl80211.h | 5 ++++- net/wireless/nl80211.c | 4 +++- net/wireless/sme.c | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 68782ba8b6e8d..598cf54e29890 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -6570,6 +6570,9 @@ cfg80211_connect_timeout(struct net_device *dev, const u8 *bssid, * @resp_ie: association response IEs (may be %NULL) * @resp_ie_len: assoc response IEs length * @fils: FILS related roaming information. + * @authorized: true if the 802.1X authentication was done by the driver or is + * not needed (e.g., when Fast Transition protocol was used), false + * otherwise. Ignored for networks that don't use 802.1X authentication. */ struct cfg80211_roam_info { struct ieee80211_channel *channel; @@ -6580,6 +6583,7 @@ struct cfg80211_roam_info { const u8 *resp_ie; size_t resp_ie_len; struct cfg80211_fils_resp_params fils; + bool authorized; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index beee59c831a73..4e6178fc06f97 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -2299,7 +2299,10 @@ enum nl80211_commands { * in %NL80211_CMD_CONNECT to indicate that for 802.1X authentication it * wants to use the supported offload of the 4-way handshake. * @NL80211_ATTR_PMKR0_NAME: PMK-R0 Name for offloaded FT. - * @NL80211_ATTR_PORT_AUTHORIZED: (reserved) + * @NL80211_ATTR_PORT_AUTHORIZED: flag attribute used in %NL80211_CMD_ROAMED + * notification indicating that that 802.1X authentication was done by + * the driver or is not needed (because roaming used the Fast Transition + * protocol). * * @NL80211_ATTR_EXTERNAL_AUTH_ACTION: Identify the requested external * authentication operation (u32 attribute with an diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index ec559dbad56ea..57a2a512a3054 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -15317,7 +15317,9 @@ void nl80211_send_roamed(struct cfg80211_registered_device *rdev, (info->fils.pmk && nla_put(msg, NL80211_ATTR_PMK, info->fils.pmk_len, info->fils.pmk)) || (info->fils.pmkid && - nla_put(msg, NL80211_ATTR_PMKID, WLAN_PMKID_LEN, info->fils.pmkid))) + nla_put(msg, NL80211_ATTR_PMKID, WLAN_PMKID_LEN, info->fils.pmkid)) || + (info->authorized && + nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED))) goto nla_put_failure; genlmsg_end(msg, hdr); diff --git a/net/wireless/sme.c b/net/wireless/sme.c index d32a2ec4d96ac..d341763f4a7bd 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -1020,6 +1020,7 @@ void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info, if (info->fils.update_erp_next_seq_num) ev->rm.fils.erp_next_seq_num = info->fils.erp_next_seq_num; ev->rm.bss = info->bss; + ev->rm.authorized = info->authorized; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); From ef387eb9ddf3b3d868da5e8db5e4cac43cf71d1d Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 14 Feb 2019 17:28:22 +0800 Subject: [PATCH 081/135] brcmfmac: set authorized flag in ROAM event for offload FT roaming When using PORT_AUTHORIZED event mechanism for offload FT, wpa_supplicant started a full EAP exchange after a successful roaming. It was caused by setting portEnabled to FALSE to get EAP state machine out of the SUCCESS state and eapSuccess cleared when handling ROAM event in wap_supplicant. With this patch, the authorized flag in the ROAM event is used to indicate the connection is authorized. Wpa_supplicant sets portEnabled according to the flag and no full EAP exchange is performed after the roaming. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 90ee39671239a..a71f002dceb66 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5905,14 +5905,12 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, roam_info.resp_ie = conn_info->resp_ie; roam_info.resp_ie_len = conn_info->resp_ie_len; + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) + roam_info.authorized = true; + cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { - cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); - brcmf_dbg(CONN, "Report port authorized\n"); - } - set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; From c72a8289fd81135fc5439d89c3a6d23b3ed3365a Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Sun, 3 Mar 2019 20:30:32 -0600 Subject: [PATCH 082/135] brcmfmac: set authorized flag in ROAM event for PMK caching With 4-way handshake offload for 802.1X authentication, the authorized flag in ROAM event should be set for a successful roaming with PMK caching. The roaming is identified by checking the existence of PMKID within the (Re)Association Request frame with this patch. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index a71f002dceb66..2c0245c7e5882 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5845,6 +5845,47 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool +brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -5905,7 +5946,9 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, roam_info.resp_ie = conn_info->resp_ie; roam_info.resp_ie_len = conn_info->resp_ie_len; - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft)) roam_info.authorized = true; cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); From 9ed16b3f1a5a0a17214ec2c6df1dfd70d2e8c3be Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 30 Apr 2019 17:52:41 -0500 Subject: [PATCH 083/135] nl80211: add authorized flag to CONNECT event Add authorized flag to CONNECT event. It is used for 802.1X 4-way handshake offload with PMK caching. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- include/net/cfg80211.h | 3 +++ net/wireless/nl80211.c | 2 ++ net/wireless/sme.c | 1 + 3 files changed, 6 insertions(+) diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 598cf54e29890..91939f39c8898 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -6410,6 +6410,8 @@ struct cfg80211_fils_resp_params { * not known. This value is used only if @status < 0 to indicate that the * failure is due to a timeout and not due to explicit rejection by the AP. * This value is ignored in other cases (@status >= 0). + * @authorized: Indicates whether the connection is ready to transport + * data packets. */ struct cfg80211_connect_resp_params { int status; @@ -6421,6 +6423,7 @@ struct cfg80211_connect_resp_params { size_t resp_ie_len; struct cfg80211_fils_resp_params fils; enum nl80211_timeout_reason timeout_reason; + bool authorized; }; /** diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 57a2a512a3054..8f465e368e0a0 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -15251,6 +15251,8 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev, (nla_put_flag(msg, NL80211_ATTR_TIMED_OUT) || nla_put_u32(msg, NL80211_ATTR_TIMEOUT_REASON, cr->timeout_reason))) || + (cr->authorized && + nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) || (cr->req_ie && nla_put(msg, NL80211_ATTR_REQ_IE, cr->req_ie_len, cr->req_ie)) || (cr->resp_ie && diff --git a/net/wireless/sme.c b/net/wireless/sme.c index d341763f4a7bd..407ce1d33a302 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -887,6 +887,7 @@ void cfg80211_connect_done(struct net_device *dev, ev->cr.bss = params->bss; ev->cr.status = params->status; ev->cr.timeout_reason = params->timeout_reason; + ev->cr.authorized = params->authorized; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); From 3e4727bba06efe8dde5a28d6b9c7341c4b40b54a Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 30 Apr 2019 17:55:02 -0500 Subject: [PATCH 084/135] brcmfmac: set authorized flag in CONNECT event for PMK caching With 4-way handshake offload for 802.1X authentication, the authorized flag in CONNECT event should be set for a successful connection with PMK caching. The connection is identified by checking the existence of PMKID within the Association Request frame with this patch. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 2c0245c7e5882..77f7ef49d0fe5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5988,6 +5988,11 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, conn_params.req_ie_len = conn_info->req_ie_len; conn_params.resp_ie = conn_info->resp_ie; conn_params.resp_ie_len = conn_info->resp_ie_len; + + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + brcmf_has_pmkid(conn_params.req_ie, conn_params.req_ie_len)) + conn_params.authorized = true; + cfg80211_connect_done(ndev, &conn_params, GFP_KERNEL); brcmf_dbg(CONN, "Report connect result - connection %s\n", completed ? "succeeded" : "failed"); From 53b79000c26cd8f98840c01fdd3b52f23126b748 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 28 May 2019 04:20:20 -0500 Subject: [PATCH 085/135] brcmfmac: add support for Opportunistic Key Caching The firmware may have OKC management. This is detected by the driver and supported via providing the PMK. The authorized flag in ROAM event should be set for a successful roaming with OKC. Signed-off-by: Darren Li Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 25 +++++++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 77f7ef49d0fe5..dce792eefcd5a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1674,6 +1674,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) struct brcmf_pub *drvr = ifp->drvr; s32 val; s32 err; + s32 okc_enable; const struct brcmf_tlv *rsn_ie; const u8 *ie; u32 ie_len; @@ -1684,6 +1685,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -1753,8 +1755,17 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { brcmf_dbg(INFO, "using 1X offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP)) goto skip_mfp_config; @@ -5384,17 +5395,27 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; + struct brcmf_pub *drvr; + int ret; brcmf_dbg(TRACE, "enter\n"); /* expect using firmware supplicant for 1X */ ifp = netdev_priv(dev); + drvr = ifp->drvr; if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) return -EINVAL; if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN) return -ERANGE; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + bphy_err(drvr, "okc_info_pmk iovar failed: ret=%d\n", ret); + } + return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -5948,7 +5969,7 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || - profile->is_ft)) + profile->is_ft || profile->is_okc)) roam_info.authorized = true; cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 74287a3487ca2..fcffe0c561fa2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -150,6 +150,7 @@ struct brcmf_cfg80211_profile { struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; bool is_ft; + bool is_okc; }; /** From 96fa9cf95c682ea071e52282c3f0b0f567b88d01 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:21:26 -0600 Subject: [PATCH 086/135] nl80211: support 4-way handshake offloading for WPA/WPA2-PSK in AP mode Let drivers advertise support for AP-mode WPA/WPA2-PSK 4-way handshake offloading with a new NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag. Extend use of NL80211_ATTR_PMK attribute indicating it might be passed as part of NL80211_CMD_START_AP command, and contain the PSK (which is the PMK, hence the name). The driver is assumed to handle the 4-way handshake by itself in this case, instead of relying on user space. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/uapi/linux/nl80211.h | 41 ++++++++++++++++++++++++------------ net/wireless/nl80211.c | 4 +++- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 4e6178fc06f97..cfba3709be880 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -183,18 +183,27 @@ * * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK flag drivers * can indicate they support offloading EAPOL handshakes for WPA/WPA2 - * preshared key authentication. In %NL80211_CMD_CONNECT the preshared - * key should be specified using %NL80211_ATTR_PMK. Drivers supporting - * this offload may reject the %NL80211_CMD_CONNECT when no preshared - * key material is provided, for example when that driver does not - * support setting the temporal keys through %CMD_NEW_KEY. + * preshared key authentication in station mode. In %NL80211_CMD_CONNECT + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_CONNECT when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * Similarly @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X flag can be * set by drivers indicating offload support of the PTK/GTK EAPOL - * handshakes during 802.1X authentication. In order to use the offload - * the %NL80211_CMD_CONNECT should have %NL80211_ATTR_WANT_1X_4WAY_HS - * attribute flag. Drivers supporting this offload may reject the - * %NL80211_CMD_CONNECT when the attribute flag is not present. + * handshakes during 802.1X authentication in station mode. In order to + * use the offload the %NL80211_CMD_CONNECT should have + * %NL80211_ATTR_WANT_1X_4WAY_HS attribute flag. Drivers supporting this + * offload may reject the %NL80211_CMD_CONNECT when the attribute flag is + * not present. + * + * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag drivers + * can indicate they support offloading EAPOL handshakes for WPA/WPA2 + * preshared key authentication in AP mode. In %NL80211_CMD_START_AP + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_START_AP when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * For 802.1X the PMK or PMK-R0 are set by providing %NL80211_ATTR_PMK * using %NL80211_CMD_SET_PMK. For offloaded FT support also @@ -2285,10 +2294,11 @@ enum nl80211_commands { * * @NL80211_ATTR_PMK: attribute for passing PMK key material. Used with * %NL80211_CMD_SET_PMKSA for the PMKSA identified by %NL80211_ATTR_PMKID. - * For %NL80211_CMD_CONNECT it is used to provide PSK for offloading 4-way - * handshake for WPA/WPA2-PSK networks. For 802.1X authentication it is - * used with %NL80211_CMD_SET_PMK. For offloaded FT support this attribute - * specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME is included as well. + * For %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP it is used to provide + * PSK for offloading 4-way handshake for WPA/WPA2-PSK networks. For 802.1X + * authentication it is used with %NL80211_CMD_SET_PMK. For offloaded FT + * support this attribute specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME + * is included as well. * * @NL80211_ATTR_SCHED_SCAN_MULTI: flag attribute which user-space shall use to * indicate that it supports multiple active scheduled scan requests. @@ -5487,6 +5497,10 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in * station mode (SAE password is passed as part of the connect command). * + * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way + * handshake with PSK in AP mode (PSK is passed as part of the start AP + * command). + * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. */ @@ -5532,6 +5546,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_EXT_KEY_ID, NL80211_EXT_FEATURE_STA_TX_PWR, NL80211_EXT_FEATURE_SAE_OFFLOAD, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 8f465e368e0a0..aa07036a6fe15 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -9162,7 +9162,9 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, if (nla_len(info->attrs[NL80211_ATTR_PMK]) != WLAN_PMK_LEN) return -EINVAL; if (!wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK)) + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK)) return -EINVAL; settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } From f097adf13efbd7965591a10b906743779c55cfbf Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:22:56 -0600 Subject: [PATCH 087/135] brcmfmac: support 4-way handshake offloading for WPA/WPA2-PSK in AP mode Firmware may have authenticator code built-in. This is detected by the driver and indicated in the wiphy features flags. User space can use this flag to determine whether or not to provide the pre-shared key material in the nl80211 start AP command to offload the 4-way handshake in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 23 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 12 ++++++++++ .../broadcom/brcm80211/brcmfmac/feature.c | 1 + .../broadcom/brcm80211/brcmfmac/feature.h | 4 +++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index dce792eefcd5a..ea21b6922ed03 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4641,6 +4641,8 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_pub *drvr = cfg->pub; + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; + struct cfg80211_crypto_settings *crypto = &settings->crypto; const struct brcmf_tlv *ssid_ie; const struct brcmf_tlv *country_ie; struct brcmf_ssid_le ssid_le; @@ -4778,6 +4780,17 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, goto exit; } + if (crypto->psk) { + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_PSK); + err = brcmf_set_pmk(ifp, crypto->psk, + BRCMF_WSEC_MAX_PSK_LEN); + if (err < 0) + goto exit; + } + if (profile->use_fwauth == 0) + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + err = brcmf_parse_configure_security(ifp, settings, NL80211_IFTYPE_AP); if (err < 0) { @@ -4867,6 +4880,7 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_pub *drvr = cfg->pub; + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; s32 err; struct brcmf_fil_bss_enable_le bss_enable; struct brcmf_join_params join_params; @@ -4900,6 +4914,12 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) return 0; } + if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) { + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK)) + brcmf_set_pmk(ifp, NULL, 0); + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + } + if (ifp->vif->mbss) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); return err; @@ -7200,6 +7220,9 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK); wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index fcffe0c561fa2..d4e32a8993649 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -137,6 +137,17 @@ enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_SAE }; +/** + * enum brcmf_profile_fwauth - firmware authenticator profile + * + * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK + */ +enum brcmf_profile_fwauth { + BRCMF_PROFILE_FWAUTH_NONE, + BRCMF_PROFILE_FWAUTH_PSK +}; + /** * struct brcmf_cfg80211_profile - profile information. * @@ -149,6 +160,7 @@ struct brcmf_cfg80211_profile { struct brcmf_cfg80211_security sec; struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; + u16 use_fwauth; bool is_ft; bool is_okc; }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index d71273428a7b2..b9ae7785b43ea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -40,6 +40,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MONITOR_FMT_RADIOTAP, "rtap" }, { BRCMF_FEAT_DOT11H, "802.11h" }, { BRCMF_FEAT_SAE, "sae" }, + { BRCMF_FEAT_FWAUTH, "idauth" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 280a1f6412d49..56d991cae1d0d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -27,6 +27,7 @@ * MONITOR_FMT_HW_RX_HDR: firmware provides monitor packets with hw/ucode header * DOT11H: firmware supports 802.11h * SAE: simultaneous authentication of equals + * FWAUTH: Firmware authenticator */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -47,7 +48,8 @@ BRCMF_FEAT_DEF(MONITOR_FMT_RADIOTAP) \ BRCMF_FEAT_DEF(MONITOR_FMT_HW_RX_HDR) \ BRCMF_FEAT_DEF(DOT11H) \ - BRCMF_FEAT_DEF(SAE) + BRCMF_FEAT_DEF(SAE) \ + BRCMF_FEAT_DEF(FWAUTH) /* * Quirks: From d1ef61372e57f317a1b3f4b2e01b57a83dd772a2 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:24:10 -0600 Subject: [PATCH 088/135] nl80211: support SAE authentication offload in AP mode Add support for SAE authentication offload in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/uapi/linux/nl80211.h | 10 ++++++---- net/wireless/nl80211.c | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index cfba3709be880..b660fb0e251bf 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -252,9 +252,10 @@ * DOC: SAE authentication offload * * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they - * support offloading SAE authentication for WPA3-Personal networks. In - * %NL80211_CMD_CONNECT the password for SAE should be specified using - * %NL80211_ATTR_SAE_PASSWORD. + * support offloading SAE authentication for WPA3-Personal networks. The + * password for SAE should be specified using %NL80211_ATTR_SAE_PASSWORD in + * %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP for station and AP mode + * respectively. */ /** @@ -5495,7 +5496,8 @@ enum nl80211_feature_flags { * to a station. * * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in - * station mode (SAE password is passed as part of the connect command). + * station mode (SAE password is passed as part of the connect command) + * or AP mode (SAE password is passed as part of the start AP command). * * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way * handshake with PSK in AP mode (PSK is passed as part of the start AP diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index aa07036a6fe15..1d69116b8b79d 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -4615,8 +4615,9 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return false; return true; case NL80211_CMD_START_AP: - /* SAE not supported yet */ - if (auth_type == NL80211_AUTHTYPE_SAE) + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && + auth_type == NL80211_AUTHTYPE_SAE) return false; /* FILS not supported yet */ if (auth_type == NL80211_AUTHTYPE_FILS_SK || From d81977ffbf84543e55d95a1ed90f7760afbaf83d Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:25:14 -0600 Subject: [PATCH 089/135] brcmfmac: support SAE authentication offload in AP mode Firmware may have SAE authenticator code built-in. This is detected by the driver and indicated in the wiphy features flags. User space can use this flag to determine whether or not to provide the password material in the nl80211 start AP command to offload the SAE authentication in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 28 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 4 ++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ea21b6922ed03..4b007ea09711a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -55,6 +55,7 @@ #define RSN_AKM_PSK 2 /* Pre-shared Key */ #define RSN_AKM_SHA256_1X 5 /* SHA256, 802.1X */ #define RSN_AKM_SHA256_PSK 6 /* SHA256, Pre-shared Key */ +#define RSN_AKM_SAE 8 /* SAE */ #define RSN_CAP_LEN 2 /* Length of RSN capabilities */ #define RSN_CAP_PTK_REPLAY_CNTR_MASK (BIT(2) | BIT(3)) #define RSN_CAP_MFPR_MASK BIT(6) @@ -4204,6 +4205,10 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "RSN_AKM_MFP_1X\n"); wpa_auth |= WPA2_AUTH_1X_SHA256; break; + case RSN_AKM_SAE: + brcmf_dbg(TRACE, "RSN_AKM_SAE\n"); + wpa_auth |= WPA3_AUTH_SAE_PSK; + break; default: bphy_err(drvr, "Invalid key mgmt info\n"); } @@ -4221,11 +4226,12 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "MFP Required\n"); mfp = BRCMF_MFP_REQUIRED; /* Firmware only supports mfp required in - * combination with WPA2_AUTH_PSK_SHA256 or - * WPA2_AUTH_1X_SHA256. + * combination with WPA2_AUTH_PSK_SHA256, + * WPA2_AUTH_1X_SHA256, or WPA3_AUTH_SAE_PSK. */ if (!(wpa_auth & (WPA2_AUTH_PSK_SHA256 | - WPA2_AUTH_1X_SHA256))) { + WPA2_AUTH_1X_SHA256 | + WPA3_AUTH_SAE_PSK))) { err = -EINVAL; goto exit; } @@ -4788,6 +4794,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if (err < 0) goto exit; } + if (crypto->sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_SAE); + err = brcmf_set_sae_password(ifp, crypto->sae_pwd, + crypto->sae_pwd_len); + if (err < 0) + goto exit; + } if (profile->use_fwauth == 0) profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); @@ -4917,6 +4931,8 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) { if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK)) brcmf_set_pmk(ifp, NULL, 0); + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_SAE)) + brcmf_set_sae_password(ifp, NULL, 0); profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); } @@ -7220,9 +7236,13 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); } - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) { wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); + } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index d4e32a8993649..bd4ea5e38dd05 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -142,10 +142,12 @@ enum brcmf_profile_fwsup { * * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for SAE */ enum brcmf_profile_fwauth { BRCMF_PROFILE_FWAUTH_NONE, - BRCMF_PROFILE_FWAUTH_PSK + BRCMF_PROFILE_FWAUTH_PSK, + BRCMF_PROFILE_FWAUTH_SAE }; /** From 52482864e548f45282e6e5a920197b52ef2dd8cd Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 7 Feb 2020 01:55:59 -0600 Subject: [PATCH 090/135] brcmfmac: add USB autosuspend feature support commit 7f1d42304d933c310592c91df70b661965a36121 upstream. We add enable dynamic suspend (autosuspend) support in host driver, and it can let platform cut down idle power consumption. To support autosuspend feature in host driver, kernel need to be built with CONFIG_USB_SUSPEND and autosuspend need to be turn on. And we also replace wowl feature with adding "needs_remote_wakeup", so that host still can be waken by wireless device. Signed-off-by: Wright Feng Signed-off-by: Chi-Hsien Lin Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1585124429-97371-6-git-send-email-chi-hsien.lin@cypress.com --- .../broadcom/brcm80211/brcmfmac/usb.c | 121 ++++++++++-------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index e910cc0294e48..be04fcbbba44b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -165,7 +165,6 @@ struct brcmf_usbdev_info { struct urb *bulk_urb; /* used for FW download */ - bool wowl_enabled; struct brcmf_mp_device *settings; }; @@ -313,28 +312,43 @@ static int brcmf_usb_tx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_send_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); if (!timeout) { brcmf_err("Txctl wait timed out\n"); usb_kill_urb(devinfo->ctl_urb); err = -EIO; + goto fail; } clear_bit(0, &devinfo->ctl_op); + +fail: + usb_autopm_put_interface(intf); +out: return err; } @@ -343,20 +357,30 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_recv_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); err = devinfo->ctl_urb_status; @@ -364,12 +388,15 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) brcmf_err("rxctl wait timed out\n"); usb_kill_urb(devinfo->ctl_urb); err = -EIO; + goto fail; } clear_bit(0, &devinfo->ctl_op); +fail: + usb_autopm_put_interface(intf); if (!err) return devinfo->ctl_urb_actual_length; - else - return err; +out: + return err; } static struct brcmf_usbreq *brcmf_usb_deq(struct brcmf_usbdev_info *devinfo, @@ -502,10 +529,12 @@ static void brcmf_usb_rx_complete(struct urb *urb) return; } - if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP) { + if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP || + devinfo->bus_pub.state == BRCMFMAC_USB_STATE_SLEEP) { skb_put(skb, urb->actual_length); brcmf_rx_frame(devinfo->dev, skb, true); brcmf_usb_rx_refill(devinfo, req); + usb_mark_last_busy(urb->dev); } else { brcmu_pkt_buf_free_skb(skb); brcmf_usb_enq(devinfo, &devinfo->rx_freeq, req, NULL); @@ -589,6 +618,11 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) struct brcmf_usbreq *req; int ret; unsigned long flags; + struct usb_interface *intf = to_usb_interface(dev); + + ret = usb_autopm_get_interface(intf); + if (ret) + goto out; brcmf_dbg(USB, "Enter, skb=%p\n", skb); if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { @@ -628,9 +662,10 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) devinfo->tx_flowblock = true; } spin_unlock_irqrestore(&devinfo->tx_flowblock_lock, flags); - return 0; fail: + usb_autopm_put_interface(intf); +out: return ret; } @@ -994,20 +1029,32 @@ static int brcmf_usb_fw_download(struct brcmf_usbdev_info *devinfo) { int err; + struct usb_interface *intf; brcmf_dbg(USB, "Enter\n"); - if (devinfo == NULL) - return -ENODEV; + if (!devinfo) { + err = -ENODEV; + goto out; + } if (!devinfo->image) { brcmf_err("No firmware!\n"); - return -ENOENT; + err = -ENOENT; + goto out; } + intf = to_usb_interface(devinfo->dev); + err = usb_autopm_get_interface(intf); + if (err) + goto out; + err = brcmf_usb_dlstart(devinfo, (u8 *)devinfo->image, devinfo->image_len); if (err == 0) err = brcmf_usb_dlrun(devinfo); + + usb_autopm_put_interface(intf); +out: return err; } @@ -1108,18 +1155,6 @@ struct brcmf_usbdev *brcmf_usb_attach(struct brcmf_usbdev_info *devinfo, return NULL; } -static void brcmf_usb_wowl_config(struct device *dev, bool enabled) -{ - struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); - - brcmf_dbg(USB, "Configuring WOWL, enabled=%d\n", enabled); - devinfo->wowl_enabled = enabled; - if (enabled) - device_set_wakeup_enable(devinfo->dev, true); - else - device_set_wakeup_enable(devinfo->dev, false); -} - static int brcmf_usb_get_fwname(struct device *dev, const char *ext, u8 *fw_name) { @@ -1146,7 +1181,6 @@ static const struct brcmf_bus_ops brcmf_usb_bus_ops = { .txdata = brcmf_usb_tx, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, - .wowl_config = brcmf_usb_wowl_config, .get_fwname = brcmf_usb_get_fwname, }; @@ -1335,6 +1369,8 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) usb_set_intfdata(intf, devinfo); + intf->needs_remote_wakeup = 1; + /* Check that the device supports only one configuration */ if (usb->descriptor.bNumConfigurations != 1) { brcmf_err("Number of configurations: %d not supported\n", @@ -1462,12 +1498,8 @@ static int brcmf_usb_suspend(struct usb_interface *intf, pm_message_t state) brcmf_err("timed out wait for cfg80211 suspended\n"); devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP; - if (devinfo->wowl_enabled) { - brcmf_cancel_all_urbs(devinfo); - } else { - brcmf_detach(&usb->dev); - brcmf_free(&usb->dev); - } + brcmf_cancel_all_urbs(devinfo); + device_set_wakeup_enable(devinfo->dev, true); return 0; } @@ -1480,22 +1512,10 @@ static int brcmf_usb_resume(struct usb_interface *intf) struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); brcmf_dbg(USB, "Enter\n"); - if (!devinfo->wowl_enabled) { - int err; - - err = brcmf_alloc(&usb->dev, devinfo->settings); - if (err) - return err; - - err = brcmf_attach(devinfo->dev); - if (err) { - brcmf_free(devinfo->dev); - return err; - } - } devinfo->bus_pub.state = BRCMFMAC_USB_STATE_UP; brcmf_usb_rx_fill_all(devinfo); + device_set_wakeup_enable(devinfo->dev, false); return 0; } @@ -1552,6 +1572,7 @@ static struct usb_driver brcmf_usbdrvr = { .suspend = brcmf_usb_suspend, .resume = brcmf_usb_resume, .reset_resume = brcmf_usb_reset_resume, + .supports_autosuspend = true, .disable_hub_initiated_lpm = 1, }; From adfef23115eb690466b4cc13f0461e1d081de906 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Wed, 7 Nov 2018 16:59:03 -0600 Subject: [PATCH 091/135] brcmfmac: To support printing USB console messages This change is to add support for printing the firmware console messges of a USB interface chip to the host. To enable this feature, build option '-msgtrace' should be enabled in the firmware. And in the host, debug=0x100000 should be provided as a module parameter. Signed-off-by: Raveendran Somu Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/bus.h | 1 + .../broadcom/brcm80211/brcmfmac/core.c | 9 ++ .../broadcom/brcm80211/brcmfmac/debug.c | 83 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/debug.h | 24 ++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 14 ++++ 5 files changed, 131 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 623c0168da79c..c8063ca506114 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -271,6 +271,7 @@ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len); void brcmf_bus_add_txhdrlen(struct device *dev, uint len); +int brcmf_fwlog_attach(struct device *dev); #ifdef CONFIG_BRCMFMAC_SDIO void brcmf_sdio_exit(void); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 5bded9bb4a7dc..ec7c5a9017946 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1051,6 +1051,15 @@ static int brcmf_inet6addr_changed(struct notifier_block *nb, } #endif + +int brcmf_fwlog_attach(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; + + return brcmf_debug_fwlog_init(drvr); +} + static int brcmf_revinfo_read(struct seq_file *s, void *data) { struct brcmf_bus *bus_if = dev_get_drvdata(s->private); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 120515fe8250f..26efe50160c24 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -14,6 +14,82 @@ #include "fweh.h" #include "debug.h" +static int +brcmf_debug_msgtrace_seqchk(u32 *prev, u32 cur) +{ + if ((cur == 0 && *prev == 0xFFFFFFFF) || ((cur - *prev) == 1)) { + goto done; + } else if (cur == *prev) { + brcmf_dbg(FWCON, "duplicate trace\n"); + return -1; + } else if (cur > *prev) { + brcmf_dbg(FWCON, "lost %d packets\n", cur - *prev); + } else { + brcmf_dbg(FWCON, "seq out of order, host %d, dongle %d\n", + *prev, cur); + } +done: + *prev = cur; + return 0; +} + +static int +brcmf_debug_msg_parser(void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + char *data, *s; + static u32 seqnum_prev; + + hdr = (struct msgtrace_hdr *)event_data; + data = (char *)event_data + MSGTRACE_HDRLEN; + + /* There are 2 bytes available at the end of data */ + data[ntohs(hdr->len)] = '\0'; + + if (ntohl(hdr->discarded_bytes) || ntohl(hdr->discarded_printf)) { + brcmf_dbg(FWCON, "Discarded_bytes %d discarded_printf %d\n", + ntohl(hdr->discarded_bytes), + ntohl(hdr->discarded_printf)); + } + + err = brcmf_debug_msgtrace_seqchk(&seqnum_prev, ntohl(hdr->seqnum)); + if (err) + return err; + + while (*data != '\0' && (s = strstr(data, "\n")) != NULL) { + *s = '\0'; + brcmf_dbg(FWCON, "CONSOLE: %s\n", data); + data = s + 1; + } + if (*data) + brcmf_dbg(FWCON, "CONSOLE: %s", data); + + return err; +} + +static int +brcmf_debug_trace_parser(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + + hdr = (struct msgtrace_hdr *)event_data; + if (hdr->version != MSGTRACE_VERSION) { + brcmf_dbg(FWCON, "trace version mismatch host %d dngl %d\n", + MSGTRACE_VERSION, hdr->version); + err = -EPROTO; + return err; + } + + if (hdr->trace_type == MSGTRACE_HDR_TYPE_MSG) + err = brcmf_debug_msg_parser(event_data); + + return err; +} + int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len) { @@ -42,6 +118,13 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, return 0; } + +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return brcmf_fweh_register(drvr, BRCMF_E_TRACE, + brcmf_debug_trace_parser); +} + struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr) { return drvr->wiphy->debugfsdir; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 9b221b509ade5..d61508b62d2c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -103,6 +103,10 @@ do { \ #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ +#define MSGTRACE_VERSION 1 +#define MSGTRACE_HDR_TYPE_MSG 0 +#define MSGTRACE_HDR_TYPE_LOG 1 + #define brcmf_dbg_hex_dump(test, data, len, fmt, ...) \ do { \ trace_brcmf_hexdump((void *)data, len); \ @@ -120,6 +124,7 @@ int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)); int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len); +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr); #else static inline struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr) { @@ -137,6 +142,25 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, { return 0; } + +static inline +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return 0; +} #endif +/* Message trace header */ +struct msgtrace_hdr { + u8 version; + u8 trace_type; + u16 len; /* Len of the trace */ + u32 seqnum; /* Sequence number of message */ + /* Number of discarded bytes because of trace overflow */ + u32 discarded_bytes; + /* Number of discarded printf because of trace overflow */ + u32 discarded_printf; +}; + +#define MSGTRACE_HDRLEN sizeof(struct msgtrace_hdr) #endif /* BRCMFMAC_DEBUG_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index be04fcbbba44b..ba7d659b0b79a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1220,6 +1220,12 @@ static void brcmf_usb_probe_phase2(struct device *dev, int ret, if (ret) goto error; + if (BRCMF_FWCON_ON()) { + ret = brcmf_fwlog_attach(devinfo->dev); + if (ret) + goto error; + } + /* Attach to the common driver interface */ ret = brcmf_attach(devinfo->dev); if (ret) @@ -1296,9 +1302,17 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) ret = brcmf_alloc(devinfo->dev, devinfo->settings); if (ret) goto fail; + + if (BRCMF_FWCON_ON()) { + ret = brcmf_fwlog_attach(devinfo->dev); + if (ret) + goto fail; + } + ret = brcmf_attach(devinfo->dev); if (ret) goto fail; + /* we are done */ complete(&devinfo->dev_init_done); return 0; From 00992406f580c23f8f17f473e293fc5762cf93e4 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Mon, 4 Nov 2019 02:03:38 -0600 Subject: [PATCH 092/135] brcmfmac: reset SDIO bus on a firmware crash commit 4684997d9eea ("brcmfmac: reset PCIe bus on a firmware crash") adds a reset function to recover firmware trap for PCIe bus. This commit adds an implementation for SDIO bus. Upon SDIO firmware trap, do below: - Remove the device - Reset hardware - Probe the device again Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 4 +-- .../broadcom/brcm80211/brcmfmac/sdio.c | 34 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/sdio.h | 3 ++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 3ad922494a791..87f115547cbd5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -868,7 +868,7 @@ static void brcmf_sdiod_freezer_detach(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ -static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) { sdiodev->state = BRCMF_SDIOD_DOWN; if (sdiodev->bus) { @@ -903,7 +903,7 @@ static void brcmf_sdiod_host_fixup(struct mmc_host *host) host->caps |= MMC_CAP_NONREMOVABLE; } -static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index de090ee148288..ec19258b20563 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4124,6 +4125,36 @@ int brcmf_sdio_get_fwname(struct device *dev, const char *ext, u8 *fw_name) return 0; } +static int brcmf_sdio_bus_reset(struct device *dev) +{ + int ret = 0; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + + brcmf_dbg(SDIO, "Enter\n"); + + /* start by unregistering irqs */ + brcmf_sdiod_intr_unregister(sdiodev); + + brcmf_sdiod_remove(sdiodev); + + /* reset the adapter */ + sdio_claim_host(sdiodev->func1); + mmc_hw_reset(sdiodev->func1->card->host); + sdio_release_host(sdiodev->func1); + + brcmf_bus_change_state(sdiodev->bus_if, BRCMF_BUS_DOWN); + + ret = brcmf_sdiod_probe(sdiodev); + if (ret) { + brcmf_err("Failed to probe after sdio device reset: ret %d\n", + ret); + brcmf_sdiod_remove(sdiodev); + } + + return ret; +} + static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .stop = brcmf_sdio_bus_stop, .preinit = brcmf_sdio_bus_preinit, @@ -4135,7 +4166,8 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, .get_fwname = brcmf_sdio_get_fwname, - .debugfs_create = brcmf_sdio_debugfs_create + .debugfs_create = brcmf_sdio_debugfs_create, + .reset = brcmf_sdio_bus_reset }; #define BRCMF_SDIO_FW_CODE 0 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 0bd47c119dae0..cb0d71d024eec 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -368,6 +368,9 @@ static inline void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); + struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus); From b3d51c76164c2cffe4aa16dc4eaeb7ff1294909d Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Mon, 17 Feb 2020 04:18:15 -0600 Subject: [PATCH 093/135] brcmfmac: fix for WPA/WPA2-PSK 4-way handshake and SAE offload failures An incorrect value of use_fwsup is set for 4-way handshake offload for WPA//WPA2-PSK, caused by commit 3b1e0a7bdfee ("brcmfmac: add support for SAE authentication offload"). It results in missing bit BRCMF_VIF_STATUS_EAP_SUCCESS set in brcmf_is_linkup() and causes the failure. This patch correct the value for the case. Also setting bit BRCMF_VIF_STATUS_EAP_SUCCESS for SAE offload case in brcmf_is_linkup() to fix SAE offload failure. Fixes: 3b1e0a7bdfee ("brcmfmac: add support for SAE authentication offload") Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 4b007ea09711a..557a3e69071ba 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1748,6 +1748,10 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) switch (sme->crypto.akm_suites[0]) { case WLAN_AKM_SUITE_SAE: val = WPA3_AUTH_SAE_PSK; + if (sme->crypto.sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + } break; default: bphy_err(drvr, "invalid cipher group (%d)\n", @@ -2042,11 +2046,6 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.sae_pwd) { - brcmf_dbg(INFO, "using SAE offload\n"); - profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; - } - if (sme->crypto.psk && profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { @@ -5610,7 +5609,8 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, u32 event = e->event_code; u32 status = e->status; - if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_PSK && + if ((vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_PSK || + vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_SAE) && event == BRCMF_E_PSK_SUP && status == BRCMF_E_STATUS_FWSUP_COMPLETED) set_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state); From 6a2246e8aeae19b6e01ecc6275caf675c522fec7 Mon Sep 17 00:00:00 2001 From: Ting-Ying Li Date: Fri, 21 Feb 2020 04:02:25 -0600 Subject: [PATCH 094/135] non-upstream: Fix no P2P IE in probe requests issue This reverts commit bd99a3013bdc00f8fc7534c657b39616792b4467. The original parameter "vif" is updated in brcmf_p2p_scan_prep() but the new code does not get the same vif. Fixes: bd99a30 ("brcmfmac: move configuration of probe request IEs") Signed-off-by: Ting-Ying Li Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 ----- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 557a3e69071ba..d5a21b6c100c9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1133,11 +1133,6 @@ brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (err) goto scan_out; - err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG, - request->ie, request->ie_len); - if (err) - goto scan_out; - err = brcmf_do_escan(vif->ifp, request); if (err) goto scan_out; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index a399714ee7b94..8eb6d4f74b288 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -895,7 +895,7 @@ int brcmf_p2p_scan_prep(struct wiphy *wiphy, { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_p2p_info *p2p = &cfg->p2p; - int err; + int err = 0; if (brcmf_p2p_scan_is_p2p_request(request)) { /* find my listen channel */ @@ -918,7 +918,9 @@ int brcmf_p2p_scan_prep(struct wiphy *wiphy, /* override .run_escan() callback. */ cfg->escan_info.run = brcmf_p2p_run_escan; } - return 0; + err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG, + request->ie, request->ie_len); + return err; } From 1a88919a35ae2c99aeec5d7ff96f53b627421e34 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Wed, 26 Feb 2020 04:21:22 -0600 Subject: [PATCH 095/135] brcmfmac: add 54591 PCIE device Support 54591 PCIE device. Signed-off-by: Double Lo Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 22 +++++++++++++++++-- .../broadcom/brcm80211/include/brcm_hw_ids.h | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index e3f1cd536f1ed..fbe4392ba362f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -59,6 +59,7 @@ BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie"); BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); BRCMF_FW_DEF(4355, "brcmfmac89459-pcie"); +BRCMF_FW_DEF(54591, "brcmfmac54591-pcie"); static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -79,6 +80,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), BRCMF_FW_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355), + BRCMF_FW_ENTRY(CY_CC_54591_CHIP_ID, 0xFFFFFFFF, 54591), }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -1633,12 +1635,21 @@ static int brcmf_pcie_get_fwname(struct device *dev, const char *ext, u8 *fw_name) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; struct brcmf_fw_request *fwreq; struct brcmf_fw_name fwnames[] = { { ext, fw_name }, }; + u32 chip; - fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev, + if (devinfo->ci->chip == CY_CC_89459_CHIP_ID && + devinfo->pdev->device == CY_PCIE_54591_DEVICE_ID) + chip = CY_CC_54591_CHIP_ID; + else + chip = bus_if->chip; + + fwreq = brcmf_fw_alloc_request(chip, bus_if->chiprev, brcmf_pcie_fwnames, ARRAY_SIZE(brcmf_pcie_fwnames), fwnames, ARRAY_SIZE(fwnames)); @@ -2080,8 +2091,14 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) { ".bin", devinfo->fw_name }, { ".txt", devinfo->nvram_name }, }; + u32 chip; - fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, + if (devinfo->ci->chip == CY_CC_89459_CHIP_ID && + devinfo->pdev->device == CY_PCIE_54591_DEVICE_ID) + chip = CY_CC_54591_CHIP_ID; + else + chip = devinfo->ci->chip; + fwreq = brcmf_fw_alloc_request(chip, devinfo->ci->chiprev, brcmf_pcie_fwnames, ARRAY_SIZE(brcmf_pcie_fwnames), fwnames, ARRAY_SIZE(fwnames)); @@ -2362,6 +2379,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID), BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID), BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_54591_DEVICE_ID), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 8ee9a04957054..1a041577dc8ec 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -51,6 +51,7 @@ #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 #define CY_CC_89459_CHIP_ID 0x4355 +#define CY_CC_54591_CHIP_ID 0x54591 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e @@ -85,6 +86,7 @@ #define BRCM_PCIE_4371_DEVICE_ID 0x440d #define CY_PCIE_89459_DEVICE_ID 0x4415 #define CY_PCIE_89459_RAW_DEVICE_ID 0x4355 +#define CY_PCIE_54591_DEVICE_ID 0x4417 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ From c9ed6dd66ae5827500b55fdd802d433ce192dd1d Mon Sep 17 00:00:00 2001 From: Praveen Babu C Date: Tue, 9 Jan 2018 11:33:10 +0530 Subject: [PATCH 096/135] brcmfmac: support DS1 exit firmware re-download In deep sleep mode (DS1) ARM is off and once exit trigger comes than mailbox Interrupt comes to host and whole reinitiation should be done in the ARM to start TX/RX. Also fix below issus for DS1 exit: 1. Sent Tx Control frame only after firmware redownload complete (check F2 Ready before sending Tx Control frame to Firmware) 2. intermittent High DS1 TX Exit latency time (almost 3sec) ==> This is fixed by skipping host Mailbox interrupt Multiple times (ulp state mechanism) 3. RX GlOM save/restore in Firmware 4. Add ULP event enable & event_msgs_ext iovar configuration in FMAC 5. Add ULP_EVENT_RECV state machine for sbwad support 6. Support 2 Byte Shared memory read for DS1 Exit HUDI implementation Signed-off-by: Praveen Babu C Signed-off-by: Naveen Gupta [Merge from 4.14.77 to 5.4.18; set BRCMF_SDIO_MAX_ACCESS_ERRORS to 20] Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/bus.h | 2 +- .../broadcom/brcm80211/brcmfmac/common.c | 39 +++ .../broadcom/brcm80211/brcmfmac/core.c | 13 +- .../broadcom/brcm80211/brcmfmac/debug.h | 1 + .../broadcom/brcm80211/brcmfmac/fweh.h | 27 +- .../broadcom/brcm80211/brcmfmac/pcie.c | 2 +- .../broadcom/brcm80211/brcmfmac/sdio.c | 258 +++++++++++++++++- .../broadcom/brcm80211/brcmfmac/sdio.h | 110 ++++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 4 +- .../broadcom/brcm80211/include/chipcommon.h | 2 + 10 files changed, 441 insertions(+), 17 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index c8063ca506114..e62c34849b4ca 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -255,7 +255,7 @@ void brcmf_rx_event(struct device *dev, struct sk_buff *rxp); int brcmf_alloc(struct device *dev, struct brcmf_mp_device *settings); /* Indication from bus module regarding presence/insertion of dongle. */ -int brcmf_attach(struct device *dev); +int brcmf_attach(struct device *dev, bool start_bus); /* Indication from bus module regarding removal/absence of dongle */ void brcmf_detach(struct device *dev); void brcmf_free(struct device *dev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 75beeb4c95485..e9bf5f95a5bb2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -21,6 +21,8 @@ #include "firmware.h" #include "chip.h" #include "defs.h" +#include "fweh.h" +#include MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -214,6 +216,8 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) char *clmver; char *ptr; s32 err; + struct eventmsgs_ext *eventmask_msg = NULL; + u8 msglen; /* retrieve mac addresses */ err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr, @@ -334,6 +338,41 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) goto done; } + /* Enable event_msg_ext specific to 43012 chip */ + if (bus->chip == CY_CC_43012_CHIP_ID) { + /* Program event_msg_ext to support event larger than 128 */ + msglen = (roundup(BRCMF_E_LAST, NBBY) / NBBY) + + EVENTMSGS_EXT_STRUCT_SIZE; + /* Allocate buffer for eventmask_msg */ + eventmask_msg = kzalloc(msglen, GFP_KERNEL); + if (!eventmask_msg) { + err = -ENOMEM; + goto done; + } + + /* Read the current programmed event_msgs_ext */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->len = roundup(BRCMF_E_LAST, NBBY) / NBBY; + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmask_msg, + msglen); + + /* Enable ULP event */ + brcmf_dbg(EVENT, "enable event ULP\n"); + setbit(eventmask_msg->mask, BRCMF_E_ULP); + + /* Write updated Event mask */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->command = EVENTMSGS_SET_MASK; + eventmask_msg->len = (roundup(BRCMF_E_LAST, NBBY) / NBBY); + + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmask_msg, msglen); + if (err) { + brcmf_err("Set event_msgs_ext error (%d)\n", err); + goto done; + } + } /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index ec7c5a9017946..f290a5f3cdc22 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1249,7 +1249,7 @@ int brcmf_alloc(struct device *dev, struct brcmf_mp_device *settings) return 0; } -int brcmf_attach(struct device *dev) +int brcmf_attach(struct device *dev, bool start_bus) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); struct brcmf_pub *drvr = bus_if->drvr; @@ -1280,10 +1280,13 @@ int brcmf_attach(struct device *dev) /* attach firmware event handler */ brcmf_fweh_attach(drvr); - ret = brcmf_bus_started(drvr, drvr->ops); - if (ret != 0) { - bphy_err(drvr, "dongle is not responding: err=%d\n", ret); - goto fail; + if (start_bus) { + ret = brcmf_bus_started(drvr, drvr->ops); + if (ret != 0) { + bphy_err(drvr, "dongle is not responding: err=%d\n", + ret); + goto fail; + } } return 0; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index d61508b62d2c0..f67f54c30d9e9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -29,6 +29,7 @@ #define BRCMF_MSGBUF_VAL 0x00040000 #define BRCMF_PCIE_VAL 0x00080000 #define BRCMF_FWCON_VAL 0x00100000 +#define BRCMF_ULP_VAL 0x00200000 /* set default print format */ #undef pr_fmt diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index 8210509f03f3d..e740c2f00d3b0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -91,7 +91,8 @@ struct brcmf_cfg80211_info; BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \ BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \ BRCMF_ENUM_DEF(PHY_TEMP, 111) \ - BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) + BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) \ + BRCMF_ENUM_DEF(ULP, 146) #define BRCMF_ENUM_DEF(id, val) \ BRCMF_E_##id = (val), @@ -103,7 +104,7 @@ enum brcmf_fweh_event_code { * minimum length check in device firmware so it is * hard-coded here. */ - BRCMF_E_LAST = 139 + BRCMF_E_LAST = 147 }; #undef BRCMF_ENUM_DEF @@ -284,6 +285,28 @@ struct brcmf_if_event { u8 role; }; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 +#define EVENTMSGS_EXT_STRUCT_SIZE offsetof(struct eventmsgs_ext, mask[0]) + +/* len- for SET it would be mask size from the application to the firmware */ +/* for GET it would be actual firmware mask size */ +/* maxgetsize - is only used for GET. indicate max mask size that the */ +/* application can read from the firmware */ +struct eventmsgs_ext { + u8 ver; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[1]; +}; + typedef int (*brcmf_fweh_handler_t)(struct brcmf_if *ifp, const struct brcmf_event_msg *evtmsg, void *data); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index fbe4392ba362f..d8ab8fb04d166 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -2071,7 +2071,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, brcmf_pcie_intr_enable(devinfo); brcmf_pcie_hostready(devinfo); - ret = brcmf_attach(&devinfo->pdev->dev); + ret = brcmf_attach(&devinfo->pdev->dev, true); if (ret) goto fail; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index ec19258b20563..b9e21d3df32a8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -35,9 +35,11 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "fwil.h" #define DCMD_RESP_TIMEOUT msecs_to_jiffies(2500) #define CTL_DONE_TIMEOUT msecs_to_jiffies(2500) +#define ULP_HUDI_PROC_DONE_TIME msecs_to_jiffies(2500) /* watermark expressed in number of words */ #define DEFAULT_F2_WATERMARK 0x8 @@ -325,7 +327,16 @@ struct rte_console { #define KSO_WAIT_US 50 #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US) -#define BRCMF_SDIO_MAX_ACCESS_ERRORS 5 +#define BRCMF_SDIO_MAX_ACCESS_ERRORS 20 + +static void brcmf_sdio_firmware_callback(struct device *dev, int err, + struct brcmf_fw_request *fwreq); +static struct brcmf_fw_request * + brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus); +static int brcmf_sdio_f2_ready(struct brcmf_sdio *bus); +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data); #ifdef DEBUG /* Device console log buffer state */ @@ -1086,7 +1097,7 @@ static void brcmf_sdio_get_console_addr(struct brcmf_sdio *bus) } #endif /* DEBUG */ -static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) +static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) { struct brcmf_sdio_dev *sdiod = bus->sdiodev; struct brcmf_core *core = bus->sdio_core; @@ -1175,6 +1186,9 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK)) brcmf_err("Unknown mailbox data content: 0x%02x\n", hmb_data); + /* Populate hmb_data if argument is passed for DS1 check later */ + if (hmbd) + *hmbd = hmb_data; return intstatus; } @@ -2558,6 +2572,182 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) return ret; } +/* This Function is used to retrieve important + * details from dongle related to ULP mode Mostly + * values/SHM details that will be vary depending + * on the firmware branches + */ +static void +brcmf_sdio_ulp_preinit(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + struct brcmf_if *ifp = bus_if->drvr->iflist[0]; + + brcmf_dbg(ULP, "Enter\n"); + + /* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */ + brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", + &sdiodev->fmac_ulp.ulp_shm_offset, + sizeof(sdiodev->fmac_ulp.ulp_shm_offset)); + + sdiodev->ulp = false; + + brcmf_dbg(ULP, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", + M_DS1_CTRL_SDIO(sdiodev->fmac_ulp), + M_WAKEEVENT_IND(sdiodev->fmac_ulp)); + brcmf_dbg(ULP, "m_ulp_wakeind [%x]\n", + M_ULP_WAKE_IND(sdiodev->fmac_ulp)); +} + +/* Reinitialize ARM because In DS1 mode ARM got off */ +static int +brcmf_sdio_ulp_reinit_fw(struct brcmf_sdio *bus) +{ + struct brcmf_sdio_dev *sdiodev = bus->sdiodev; + struct brcmf_fw_request *fwreq; + int err = 0; + + /* After firmware redownload tx/rx seq are reset accordingly + * these values are reset on FMAC side tx_max is initially set to 4, + * which later is updated by FW. + */ + bus->tx_seq = 0; + bus->rx_seq = 0; + bus->tx_max = 4; + + fwreq = brcmf_sdio_prepare_fw_request(bus); + if (!fwreq) + return -ENOMEM; + + err = brcmf_fw_get_firmwares(sdiodev->dev, fwreq, + brcmf_sdio_firmware_callback); + if (err != 0) { + brcmf_err("async firmware request failed: %d\n", err); + kfree(fwreq); + } + + return err; +} + +/* Check if device is in DS1 mode and handshake with ULP UCODE */ +static bool +brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus, u32 hmb_data) +{ + struct brcmf_sdio_dev *sdiod = bus->sdiodev; + int err = 0; + u32 value = 0; + u32 val32, ulp_wake_ind, wowl_wake_ind; + int reg_addr; + unsigned long timeout; + struct brcmf_ulp *fmac_ulp = &bus->sdiodev->fmac_ulp; + int i = 0; + + /* If any host mail box data is present, ignore DS1 exit sequence */ + if (hmb_data) + return false; + /* Skip if DS1 Exit is already in progress + * This can happen if firmware download is taking more time + */ + if (fmac_ulp->ulp_state == FMAC_ULP_TRIGGERED) + return false; + + value = brcmf_sdiod_func0_rb(sdiod, SDIO_CCCR_IOEx, &err); + + if (value == SDIO_FUNC_ENABLE_1) { + brcmf_dbg(ULP, "GOT THE INTERRUPT FROM UCODE\n"); + sdiod->ulp = true; + fmac_ulp->ulp_state = FMAC_ULP_TRIGGERED; + ulp_wake_ind = D11SHM_RDW(sdiod, + M_ULP_WAKE_IND(sdiod->fmac_ulp), + &err); + wowl_wake_ind = D11SHM_RDW(sdiod, + M_WAKEEVENT_IND(sdiod->fmac_ulp), + &err); + + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x state %s\n", + wowl_wake_ind, ulp_wake_ind, (fmac_ulp->ulp_state) ? + "DS1 Exit Triggered" : "IDLE State"); + + if (wowl_wake_ind || ulp_wake_ind) { + /* RX wake Don't do anything. + * Just bail out and re-download firmware. + */ + /* Print out PHY TX error block when bit 9 set */ + if ((ulp_wake_ind & C_DS1_PHY_TXERR) && + M_DS1_PHYTX_ERR_BLK(sdiod->fmac_ulp)) { + brcmf_err("Dump PHY TX Error SHM Locations\n"); + for (i = 0; i < PHYTX_ERR_BLK_SIZE; i++) { + pr_err("0x%x", + D11SHM_RDW(sdiod, + (M_DS1_PHYTX_ERR_BLK(sdiod->fmac_ulp) + + (i * 2)), &err)); + } + brcmf_err("\n"); + } + } else { + /* TX wake negotiate with MAC */ + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + &err)); + val32 = D11SHM_RD(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + &err); + D11SHM_WR(sdiod, M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + val32, (C_DS1_CTRL_SDIO_DS1_EXIT | + C_DS1_CTRL_REQ_VALID), &err); + val32 = D11REG_RD(sdiod, D11_MACCONTROL_REG, &err); + val32 = val32 | D11_MACCONTROL_REG_WAKE; + D11REG_WR(sdiod, D11_MACCONTROL_REG, val32, &err); + + /* Poll for PROC_DONE to be set by ucode */ + value = D11SHM_RDW(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + &err); + /* Wait here (polling) for C_DS1_CTRL_PROC_DONE */ + timeout = jiffies + ULP_HUDI_PROC_DONE_TIME; + while (!(value & C_DS1_CTRL_PROC_DONE)) { + value = D11SHM_RDW(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + &err); + if (time_after(jiffies, timeout)) + break; + usleep_range(1000, 2000); + } + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), &err)); + value = D11SHM_RDW(sdiod, + M_DS1_CTRL_SDIO(sdiod->fmac_ulp), + &err); + if (!(value & C_DS1_CTRL_PROC_DONE)) { + brcmf_err("Timeout Failed to enter DS1 Exit state!\n"); + return false; + } + } + + ulp_wake_ind = D11SHM_RDW(sdiod, + M_ULP_WAKE_IND(sdiod->fmac_ulp), + &err); + wowl_wake_ind = D11SHM_RDW(sdiod, + M_WAKEEVENT_IND(sdiod->fmac_ulp), + &err); + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", + wowl_wake_ind, ulp_wake_ind); + reg_addr = CORE_CC_REG( + brcmf_chip_get_pmu(bus->ci)->base, min_res_mask); + brcmf_sdiod_writel(sdiod, reg_addr, + DEFAULT_43012_MIN_RES_MASK, &err); + if (err) + brcmf_err("min_res_mask failed\n"); + + return true; + } + + return false; +} + static void brcmf_sdio_dpc(struct brcmf_sdio *bus) { struct brcmf_sdio_dev *sdiod = bus->sdiodev; @@ -2629,8 +2819,11 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) /* Handle host mailbox indication */ if (intstatus & I_HMB_HOST_INT) { + u32 hmb_data = 0; intstatus &= ~I_HMB_HOST_INT; - intstatus |= brcmf_sdio_hostmail(bus); + intstatus |= brcmf_sdio_hostmail(bus, &hmb_data); + if (brcmf_sdio_ulp_pre_redownload_check(bus, hmb_data)) + brcmf_sdio_ulp_reinit_fw(bus); } sdio_release_host(bus->sdiodev->func1); @@ -2675,7 +2868,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_sdio_clrintr(bus); if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && - txctl_ok(bus)) { + txctl_ok(bus) && brcmf_sdio_f2_ready(bus)) { sdio_claim_host(bus->sdiodev->func1); if (bus->ctrl_frame_stat) { err = brcmf_sdio_tx_ctrlframe(bus, bus->ctrl_frame_buf, @@ -3544,6 +3737,10 @@ static int brcmf_sdio_bus_preinit(struct device *dev) if (err < 0) goto done; + /* initialize SHM address from firmware for DS1 */ + if (!bus->sdiodev->ulp) + brcmf_sdio_ulp_preinit(dev); + bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN; if (sdiodev->sg_support) { bus->txglom = false; @@ -4186,7 +4383,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, u8 saveclk, bpreq; u8 devctl; - brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err); + brcmf_dbg(ULP, "Enter: dev=%s, err=%d\n", dev_name(dev), err); if (err) goto fail; @@ -4362,12 +4559,25 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, } /* Attach to the common layer, reserve hdr space */ - err = brcmf_attach(sdiod->dev); + err = brcmf_attach(sdiod->dev, !bus->sdiodev->ulp); if (err != 0) { brcmf_err("brcmf_attach failed\n"); goto free; } + /* Register for ULP events */ + if (sdiod->func1->device == SDIO_DEVICE_ID_CYPRESS_43012) + brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, + brcmf_ulp_event_notify); + + if (bus->sdiodev->ulp) { + /* For ULP, after firmware redownload complete + * set ULP state to IDLE + */ + if (bus->sdiodev->fmac_ulp.ulp_state == FMAC_ULP_TRIGGERED) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_IDLE; + } + /* ready */ return; @@ -4611,3 +4821,39 @@ int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep) return ret; } +/* Check F2 Ready bit before sending data to Firmware */ +static int +brcmf_sdio_f2_ready(struct brcmf_sdio *bus) +{ + int ret = -1; + int iordy_status = 0; + + sdio_claim_host(bus->sdiodev->func1); + /* Read the status of IOR2 */ + iordy_status = brcmf_sdiod_func0_rb(bus->sdiodev, SDIO_CCCR_IORx, NULL); + + sdio_release_host(bus->sdiodev->func1); + ret = iordy_status & SDIO_FUNC_ENABLE_2; + return ret; +} + +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data) +{ + int err = 0; + struct brcmf_bus *bus_if = ifp->drvr->bus_if; + struct brcmf_sdio_dev *sdiodev; + struct brcmf_sdio *bus; + struct brcmf_ulp_event *ulp_event = (struct brcmf_ulp_event *)data; + + sdiodev = bus_if->bus_priv.sdio; + bus = sdiodev->bus; + + brcmf_dbg(ULP, "Chip went to DS1 state : action %d\n", + ulp_event->ulp_dongle_action); + if (ulp_event->ulp_dongle_action == FMAC_ULP_ENTRY) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_ENTRY_RECV; + + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index cb0d71d024eec..d008689d0d24e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -165,6 +165,35 @@ struct brcmf_sdreg { struct brcmf_sdio; struct brcmf_sdiod_freezer; +/* ULP SHM Offsets info */ +struct ulp_shm_info { + u32 m_ulp_ctrl_sdio; + u32 m_ulp_wakeevt_ind; + u32 m_ulp_wakeind; + u32 m_ulp_phytxblk; +}; + +/* FMAC ULP state machine */ +#define FMAC_ULP_IDLE (0) +#define FMAC_ULP_ENTRY_RECV (1) +#define FMAC_ULP_TRIGGERED (2) + +/* BRCMF_E_ULP event data */ +#define FMAC_ULP_EVENT_VERSION 1 +#define FMAC_ULP_DISABLE_CONSOLE 1 /* Disable console */ +#define FMAC_ULP_UCODE_DOWNLOAD 2 /* Download ULP ucode file */ +#define FMAC_ULP_ENTRY 3 /* Inform ulp entry to Host */ + +struct brcmf_ulp { + uint ulp_state; + struct ulp_shm_info ulp_shm_offset; +}; + +struct brcmf_ulp_event { + u16 version; + u16 ulp_dongle_action; +}; + struct brcmf_sdio_dev { struct sdio_func *func1; struct sdio_func *func2; @@ -190,6 +219,8 @@ struct brcmf_sdio_dev { bool wowl_enabled; enum brcmf_sdiod_state state; struct brcmf_sdiod_freezer *freezer; + struct brcmf_ulp fmac_ulp; + bool ulp; }; /* sdio core registers */ @@ -380,4 +411,83 @@ void brcmf_sdio_wowl_config(struct device *dev, bool enabled); int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep); void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); +/* SHM offsets */ +#define M_DS1_CTRL_SDIO(ptr) ((ptr).ulp_shm_offset.m_ulp_ctrl_sdio) +#define M_WAKEEVENT_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeevt_ind) +#define M_ULP_WAKE_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeind) +#define M_DS1_PHYTX_ERR_BLK(ptr) ((ptr).ulp_shm_offset.m_ulp_phytxblk) + +#define D11_BASE_ADDR 0x18001000 +#define D11_AXI_BASE_ADDR 0xE8000000 +#define D11_SHM_BASE_ADDR (D11_AXI_BASE_ADDR + 0x4000) + +#define D11REG_ADDR(offset) (D11_BASE_ADDR + (offset)) +#define D11IHR_ADDR(offset) (D11_AXI_BASE_ADDR + 0x400 + (2 * (offset))) +#define D11SHM_ADDR(offset) (D11_SHM_BASE_ADDR + (offset)) + +/* MacControl register */ +#define D11_MACCONTROL_REG D11REG_ADDR(0x120) +#define D11_MACCONTROL_REG_WAKE 0x4000000 + +/* HUDI Sequence SHM bits */ +#define C_DS1_CTRL_SDIO_DS1_SLEEP 0x1 +#define C_DS1_CTRL_SDIO_MAC_ON 0x2 +#define C_DS1_CTRL_SDIO_RADIO_PHY_ON 0x4 +#define C_DS1_CTRL_SDIO_DS1_EXIT 0x8 +#define C_DS1_CTRL_PROC_DONE 0x100 +#define C_DS1_CTRL_REQ_VALID 0x200 + +/* M_ULP_WAKEIND bits */ +#define C_WATCHDOG_EXPIRY BIT(0) +#define C_FCBS_ERROR BIT(1) +#define C_RETX_FAILURE BIT(2) +#define C_HOST_WAKEUP BIT(3) +#define C_INVALID_FCBS_BLOCK BIT(4) +#define C_HUDI_DS1_EXIT BIT(5) +#define C_LOB_SLEEP BIT(6) +#define C_DS1_PHY_TXERR BIT(9) +#define C_DS1_WAKE_TIMER BIT(10) + +#define PHYTX_ERR_BLK_SIZE 18 +#define D11SHM_FIRST2BYTE_MASK 0xFFFF0000 +#define D11SHM_SECOND2BYTE_MASK 0x0000FFFF +#define D11SHM_2BYTE_SHIFT 16 + +#define D11SHM_RD(sdh, offset, ret) \ + brcmf_sdiod_readl(sdh, D11SHM_ADDR(offset), ret) + +/* SHM Read is motified based on SHM 4 byte alignment as SHM size is 2 bytes and + * 2 byte is currently not working on FMAC + * If SHM address is not 4 byte aligned, then right shift by 16 + * otherwise, mask the first two MSB bytes + * Suppose data in address 7260 is 0x440002 and it is 4 byte aligned + * Correct SHM value is 0x2 for this SHM offset and next SHM value is 0x44 + */ +#define D11SHM_RDW(sdh, offset, ret) \ + ((offset % 4) ? \ + (brcmf_sdiod_readl(sdh, D11SHM_ADDR(offset), ret) \ + >> D11SHM_2BYTE_SHIFT) : \ + (brcmf_sdiod_readl(sdh, D11SHM_ADDR(offset), ret) \ + & D11SHM_SECOND2BYTE_MASK)) + +/* SHM is of size 2 bytes, 4 bytes write will overwrite other SHM's + * First read 4 bytes and then clear the required two bytes based on + * 4 byte alignment, then update the required value and write the + * 4 byte value now + */ +#define D11SHM_WR(sdh, offset, val, mask, ret) \ + do { \ + if ((offset) % 4) \ + val = (val & D11SHM_SECOND2BYTE_MASK) | \ + ((mask) << D11SHM_2BYTE_SHIFT); \ + else \ + val = (mask) | (val & D11SHM_FIRST2BYTE_MASK); \ + brcmf_sdiod_writel(sdh, D11SHM_ADDR(offset), val, ret); \ + } while (0) +#define D11REG_WR(sdh, addr, val, ret) \ + brcmf_sdiod_writel(sdh, addr, val, ret) + +#define D11REG_RD(sdh, addr, ret) \ + brcmf_sdiod_readl(sdh, addr, ret) + #endif /* BRCMFMAC_SDIO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index ba7d659b0b79a..95dfcee6444b5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1227,7 +1227,7 @@ static void brcmf_usb_probe_phase2(struct device *dev, int ret, } /* Attach to the common driver interface */ - ret = brcmf_attach(devinfo->dev); + ret = brcmf_attach(devinfo->dev, true); if (ret) goto error; @@ -1309,7 +1309,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) goto fail; } - ret = brcmf_attach(devinfo->dev); + ret = brcmf_attach(devinfo->dev, true); if (ret) goto fail; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 0340bba968688..090a75bcd728a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -308,4 +308,6 @@ struct chipcregs { */ #define PMU_MAX_TRANSITION_DLY 15000 +#define DEFAULT_43012_MIN_RES_MASK 0x0f8bfe77 + #endif /* _SBCHIPC_H */ From 659d520668c7d7d4795da65cab3c5f3b603752bf Mon Sep 17 00:00:00 2001 From: Nitin Bhaskar Date: Tue, 21 Aug 2018 16:03:10 +0530 Subject: [PATCH 097/135] brcmfmac: fix 43012 insmod-after-rmmod in DS1 failure. After entering ULP, issuing rmmod does not put chip in sane state because of which next insmod fails. Fix includes writing into few LHL, PMU registers. Signed-off-by: Nitin Bhaskar [Merge from 4.14.77 to 5.4.18] Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/chip.c | 155 ++++++++++++++ .../broadcom/brcm80211/brcmfmac/chip.h | 9 +- .../broadcom/brcm80211/brcmfmac/sdio.c | 14 +- .../broadcom/brcm80211/include/chipcommon.h | 193 +++++++++++++++++- 4 files changed, 367 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 2e9b3248973cb..f40970dc3955b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -213,6 +213,18 @@ struct sbsocramregs { #define ARMCR4_BSZ_MASK 0x3f #define ARMCR4_BSZ_MULT 8192 +/* Minimum PMU resource mask for 43012C0 */ +#define CY_43012_PMU_MIN_RES_MASK 0xF8BFE77 + +/* PMU STATUS mask for 43012C0 */ +#define CY_43012_PMU_STATUS_MASK 0x1AC + +/* PMU CONTROL EXT mask for 43012C0 */ +#define CY_43012_PMU_CONTROL_EXT_MASK 0x11 + +/* PMU CONTROL EXT mask for 43012C0 */ +#define CY_43012_PMU_WATCHDOG_TICK_VAL 0x04 + struct brcmf_core_priv { struct brcmf_core pub; u32 wrapbase; @@ -1207,6 +1219,14 @@ struct brcmf_core *brcmf_chip_get_pmu(struct brcmf_chip *pub) return cc; } +struct brcmf_core *brcmf_chip_get_gci(struct brcmf_chip *pub) +{ + struct brcmf_core *gci; + + gci = brcmf_chip_get_core(pub, BCMA_CORE_GCI); + return gci; +} + bool brcmf_chip_iscoreup(struct brcmf_core *pub) { struct brcmf_core_priv *core; @@ -1432,3 +1452,138 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; } } + +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + /* SW scratch */ + addr = CORE_CC_REG(base, swscratch); + chip->ops->write32(chip->ctx, addr, 0); + + /* PMU status */ + addr = CORE_CC_REG(base, pmustatus); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_STATUS_MASK); + + /* PMU control ext */ + addr = CORE_CC_REG(base, pmucontrol_ext); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_CONTROL_EXT_MASK); + + /* PMU watchdog */ + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_WATCHDOG_TICK_VAL); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_gci(pub)->base; + + /* LHL Top Level Power Sequence Control */ + addr = CORE_GCI_REG(base, lhl_top_pwrseq_ctl_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Enable0 */ + addr = CORE_GCI_REG(base, gpio_int_en_port_adr[0]); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Status0 */ + addr = CORE_GCI_REG(base, gpio_int_st_port_adr[0]); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL ARM Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL MAC TimerInt0 */ + addr = CORE_GCI_REG(base, lhl_wl_mactim_int0_adr); + chip->ops->write32(chip->ctx, addr, 0x0); +} + +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + /* Watchdog res mask */ + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + break; + + default: + break; + } +} + diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index 8fa38658e727a..216e5540a13cd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -8,7 +8,10 @@ #include #define CORE_CC_REG(base, field) \ - (base + offsetof(struct chipcregs, field)) + ((base) + offsetof(struct chipcregs, field)) + +#define CORE_GCI_REG(base, field) \ + ((base) + offsetof(struct chipgciregs, field)) /** * struct brcmf_chip - chip level information. @@ -85,5 +88,9 @@ void brcmf_chip_set_passive(struct brcmf_chip *ci); bool brcmf_chip_set_active(struct brcmf_chip *ci, u32 rstvec); bool brcmf_chip_sr_capable(struct brcmf_chip *pub); char *brcmf_chip_name(u32 chipid, u32 chiprev, char *buf, uint len); +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub); +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub); +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub); +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub); #endif /* BRCMF_AXIDMP_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index b9e21d3df32a8..cd4f5c81e82fb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4765,7 +4765,19 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) * necessary cores. */ msleep(20); - brcmf_chip_set_passive(bus->ci); + if (bus->sdiodev->fmac_ulp.ulp_state == + FMAC_ULP_ENTRY_RECV) { + brcmf_chip_ulp_reset_lhl_regs(bus->ci); + brcmf_chip_reset_pmu_regs(bus->ci); + brcmf_chip_set_default_min_res_mask( + bus->ci); + } else { + brcmf_chip_set_passive(bus->ci); + } + /* Reset the PMU, backplane and all the + * cores by using the PMUWatchdogCounter. + */ + brcmf_chip_reset_watchdog(bus->ci); brcmf_sdio_clkctl(bus, CLK_NONE, false); sdio_release_host(bus->sdiodev->func1); } diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 090a75bcd728a..39cd34c226281 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -214,8 +214,197 @@ struct chipcregs { u32 PAD[3]; u32 retention_grpidx; /* 0x680 */ u32 retention_grpctl; /* 0x684 */ - u32 PAD[94]; - u16 sromotp[768]; + u32 mac_res_req_timer; /* 0x688 */ + u32 mac_res_req_mask; /* 0x68c */ + u32 PAD[18]; + u32 pmucontrol_ext; /* 0x6d8 */ + u32 slowclkperiod; /* 0x6dc */ + u32 PAD[8]; + u32 pmuintmask0; /* 0x700 */ + u32 pmuintmask1; /* 0x704 */ + u32 PAD[14]; + u32 pmuintstatus; /* 0x740 */ + u32 extwakeupstatus; /* 0x744 */ + u32 watchdog_res_mask; /* 0x748 */ + u32 swscratch; /* 0x750 */ + u32 PAD[3]; + u32 extwakemask[2]; /* 0x760-0x764 */ + u32 PAD[2]; + u32 extwakereqmask[2]; /* 0x770-0x774 */ + u32 PAD[2]; + u32 pmuintctrl0; /* 0x780 */ + u32 pmuintctrl1; /* 0x784 */ + u32 PAD[2]; + u32 extwakectrl[2]; /* 0x790 */ +}; + +#define CHIPGCIREGOFFS(field) offsetof(struct chipgciregs, field) + +struct chipgciregs { + u32 gci_corecaps0; /* 0x000 */ + u32 gci_corecaps1; /* 0x004 */ + u32 gci_corecaps2; /* 0x008 */ + u32 gci_corectrl; /* 0x00c */ + u32 gci_corestat; /* 0x010 */ + u32 gci_intstat; /* 0x014 */ + u32 gci_intmask; /* 0x018 */ + u32 gci_wakemask; /* 0x01c */ + u32 gci_levelintstat; /* 0x020 */ + u32 gci_eventintstat; /* 0x024 */ + u32 gci_wakelevelintstat; /* 0x028 */ + u32 gci_wakeeventintstat; /* 0x02c */ + u32 semaphoreintstatus; /* 0x030 */ + u32 semaphoreintmask; /* 0x034 */ + u32 semaphorerequest; /* 0x038 */ + u32 semaphorereserve; /* 0x03c */ + u32 gci_indirect_addr; /* 0x040 */ + u32 gci_gpioctl; /* 0x044 */ + u32 gci_gpiostatus; /* 0x048 */ + u32 gci_gpiomask; /* 0x04c */ + u32 eventsummary; /* 0x050 */ + u32 gci_miscctl; /* 0x054 */ + u32 gci_gpiointmask; /* 0x058 */ + u32 gci_gpiowakemask; /* 0x05c */ + u32 gci_input[32]; /* 0x060 */ + u32 gci_event[32]; /* 0x0e0 */ + u32 gci_output[4]; /* 0x160 */ + u32 gci_control_0; /* 0x170 */ + u32 gci_control_1; /* 0x174 */ + u32 gci_intpolreg; /* 0x178 */ + u32 gci_levelintmask; /* 0x17c */ + u32 gci_eventintmask; /* 0x180 */ + u32 wakelevelintmask; /* 0x184 */ + u32 wakeeventintmask; /* 0x188 */ + u32 hwmask; /* 0x18c */ + u32 PAD; + u32 gci_inbandeventintmask; /* 0x194 */ + u32 PAD; + u32 gci_inbandeventstatus; /* 0x19c */ + u32 gci_seciauxtx; /* 0x1a0 */ + u32 gci_seciauxrx; /* 0x1a4 */ + u32 gci_secitx_datatag; /* 0x1a8 */ + u32 gci_secirx_datatag; /* 0x1ac */ + u32 gci_secitx_datamask; /* 0x1b0 */ + u32 gci_seciusef0tx_reg; /* 0x1b4 */ + u32 gci_secif0tx_offset; /* 0x1b8 */ + u32 gci_secif0rx_offset; /* 0x1bc */ + u32 gci_secif1tx_offset; /* 0x1c0 */ + u32 gci_rxfifo_common_ctrl; /* 0x1c4 */ + u32 gci_rxfifoctrl; /* 0x1c8 */ + u32 gci_hw_sema_status; /* 0x1cc */ + u32 gci_seciuartescval; /* 0x1d0 */ + u32 gic_seciuartautobaudctr; /* 0x1d4 */ + u32 gci_secififolevel; /* 0x1d8 */ + u32 gci_seciuartdata; /* 0x1dc */ + u32 gci_secibauddiv; /* 0x1e0 */ + u32 gci_secifcr; /* 0x1e4 */ + u32 gci_secilcr; /* 0x1e8 */ + u32 gci_secimcr; /* 0x1ec */ + u32 gci_secilsr; /* 0x1f0 */ + u32 gci_secimsr; /* 0x1f4 */ + u32 gci_baudadj; /* 0x1f8 */ + u32 gci_inbandintmask; /* 0x1fc */ + u32 gci_chipctrl; /* 0x200 */ + u32 gci_chipsts; /* 0x204 */ + u32 gci_gpioout; /* 0x208 */ + u32 gci_gpioout_read; /* 0x20C */ + u32 gci_mpwaketx; /* 0x210 */ + u32 gci_mpwakedetect; /* 0x214 */ + u32 gci_seciin_ctrl; /* 0x218 */ + u32 gci_seciout_ctrl; /* 0x21C */ + u32 gci_seciin_auxfifo_en; /* 0x220 */ + u32 gci_seciout_txen_txbr; /* 0x224 */ + u32 gci_seciin_rxbrstatus; /* 0x228 */ + u32 gci_seciin_rxerrstatus; /* 0x22C */ + u32 gci_seciin_fcstatus; /* 0x230 */ + u32 gci_seciout_txstatus; /* 0x234 */ + u32 gci_seciout_txbrstatus; /* 0x238 */ + u32 wlan_mem_info; /* 0x23C */ + u32 wlan_bankxinfo; /* 0x240 */ + u32 bt_smem_select; /* 0x244 */ + u32 bt_smem_stby; /* 0x248 */ + u32 bt_smem_status; /* 0x24C */ + u32 wlan_bankxactivepda; /* 0x250 */ + u32 wlan_bankxsleeppda; /* 0x254 */ + u32 wlan_bankxkill; /* 0x258 */ + u32 PAD[41]; + u32 gci_chipid; /* 0x300 */ + u32 PAD[3]; + u32 otpstatus; /* 0x310 */ + u32 otpcontrol; /* 0x314 */ + u32 otpprog; /* 0x318 */ + u32 otplayout; /* 0x31c */ + u32 otplayoutextension; /* 0x320 */ + u32 otpcontrol1; /* 0x324 */ + u32 otpprogdata; /* 0x328 */ + u32 PAD[52]; + u32 otpECCstatus; /* 0x3FC */ + u32 PAD[512]; + u32 lhl_core_capab_adr; /* 0xC00 */ + u32 lhl_main_ctl_adr; /* 0xC04 */ + u32 lhl_pmu_ctl_adr; /* 0xC08 */ + u32 lhl_extlpo_ctl_adr; /* 0xC0C */ + u32 lpo_ctl_adr; /* 0xC10 */ + u32 lhl_lpo2_ctl_adr; /* 0xC14 */ + u32 lhl_osc32k_ctl_adr; /* 0xC18 */ + u32 lhl_clk_status_adr; /* 0xC1C */ + u32 lhl_clk_det_ctl_adr; /* 0xC20 */ + u32 lhl_clk_sel_adr; /* 0xC24 */ + u32 hidoff_cnt_adr[2]; /* 0xC28-0xC2C */ + u32 lhl_autoclk_ctl_adr; /* 0xC30 */ + u32 PAD; + u32 lhl_hibtim_adr; /* 0xC38 */ + u32 lhl_wl_ilp_val_adr; /* 0xC3C */ + u32 lhl_wl_armtim0_intrp_adr; /* 0xC40 */ + u32 lhl_wl_armtim0_st_adr; /* 0xC44 */ + u32 lhl_wl_armtim0_adr; /* 0xC48 */ + u32 PAD[9]; + u32 lhl_wl_mactim0_intrp_adr; /* 0xC70 */ + u32 lhl_wl_mactim0_st_adr; /* 0xC74 */ + u32 lhl_wl_mactim_int0_adr; /* 0xC78 */ + u32 lhl_wl_mactim_frac0_adr; /* 0xC7C */ + u32 lhl_wl_mactim1_intrp_adr; /* 0xC80 */ + u32 lhl_wl_mactim1_st_adr; /* 0xC84 */ + u32 lhl_wl_mactim_int1_adr; /* 0xC88 */ + u32 lhl_wl_mactim_frac1_adr; /* 0xC8C */ + u32 PAD[8]; + u32 gpio_int_en_port_adr[4]; /* 0xCB0-0xCBC */ + u32 gpio_int_st_port_adr[4]; /* 0xCC0-0xCCC */ + u32 gpio_ctrl_iocfg_p_adr[64]; /* 0xCD0-0xDCC */ + u32 gpio_gctrl_iocfg_p0_p39_adr; /* 0xDD0 */ + u32 gpio_gdsctrl_iocfg_p0_p25_p30_p39_adr; /* 0xDD4 */ + u32 gpio_gdsctrl_iocfg_p26_p29_adr; /* 0xDD8 */ + u32 PAD[8]; + u32 lhl_gpio_din0_adr; /* 0xDFC */ + u32 lhl_gpio_din1_adr; /* 0xE00 */ + u32 lhl_wkup_status_adr; /* 0xE04 */ + u32 lhl_ctl_adr; /* 0xE08 */ + u32 lhl_adc_ctl_adr; /* 0xE0C */ + u32 lhl_qdxyz_in_dly_adr; /* 0xE10 */ + u32 lhl_optctl_adr; /* 0xE14 */ + u32 lhl_optct2_adr; /* 0xE18 */ + u32 lhl_scanp_cntr_init_val_adr; /* 0xE1C */ + u32 lhl_opt_togg_val_adr[6]; /* 0xE20-0xE34 */ + u32 lhl_optx_smp_val_adr; /* 0xE38 */ + u32 lhl_opty_smp_val_adr; /* 0xE3C */ + u32 lhl_optz_smp_val_adr; /* 0xE40 */ + u32 lhl_hidoff_keepstate_adr[3]; /* 0xE44-0xE4C */ + u32 lhl_bt_slmboot_ctl0_adr[4]; /* 0xE50-0xE5C */ + u32 lhl_wl_fw_ctl; /* 0xE60 */ + u32 lhl_wl_hw_ctl_adr[2]; /* 0xE64-0xE68 */ + u32 lhl_bt_hw_ctl_adr; /* 0xE6C */ + u32 lhl_top_pwrseq_en_adr; /* 0xE70 */ + u32 lhl_top_pwrdn_ctl_adr; /* 0xE74 */ + u32 lhl_top_pwrup_ctl_adr; /* 0xE78 */ + u32 lhl_top_pwrseq_ctl_adr; /* 0xE7C */ + u32 lhl_top_pwrdn2_ctl_adr; /* 0xE80 */ + u32 lhl_top_pwrup2_ctl_adr; /* 0xE84 */ + u32 wpt_regon_intrp_cfg_adr; /* 0xE88 */ + u32 bt_regon_intrp_cfg_adr; /* 0xE8C */ + u32 wl_regon_intrp_cfg_adr; /* 0xE90 */ + u32 regon_intrp_st_adr; /* 0xE94 */ + u32 regon_intrp_en_adr; /* 0xE98 */ + }; /* chipid */ From bc06d7d92fbb3fa8e3b1179fa73634f5eb950c44 Mon Sep 17 00:00:00 2001 From: David Weng Date: Mon, 28 Jan 2019 01:05:36 -0600 Subject: [PATCH 098/135] brcmfmac: fix 43012 driver reload failure after DS1 exit Device is left in non-responding state when unloading driver in non-DS1 state if it ever enters DS1 at least once. It leaves that state only after a hard reset or power cycle 43012. Signed-off-by: David Weng [Merge from 4.14.77 to 5.4.18] Signed-off-by: Chi-hsien Lin SWWLAN-137253 --- .../net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 12 +++++++----- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 2 -- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index f40970dc3955b..b63521524a02d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1479,11 +1479,6 @@ void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub) addr = CORE_CC_REG(base, pmucontrol_ext); chip->ops->write32(chip->ctx, addr, CY_43012_PMU_CONTROL_EXT_MASK); - - /* PMU watchdog */ - addr = CORE_CC_REG(base, pmuwatchdog); - chip->ops->write32(chip->ctx, addr, - CY_43012_PMU_WATCHDOG_TICK_VAL); break; default: @@ -1576,10 +1571,17 @@ void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) switch (pub->chip) { case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); /* Watchdog res mask */ addr = CORE_CC_REG(base, watchdog_res_mask); chip->ops->write32(chip->ctx, addr, CY_43012_PMU_MIN_RES_MASK); + /* PMU watchdog */ + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_WATCHDOG_TICK_VAL); break; default: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index cd4f5c81e82fb..aa5046c5b017b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4769,8 +4769,6 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) FMAC_ULP_ENTRY_RECV) { brcmf_chip_ulp_reset_lhl_regs(bus->ci); brcmf_chip_reset_pmu_regs(bus->ci); - brcmf_chip_set_default_min_res_mask( - bus->ci); } else { brcmf_chip_set_passive(bus->ci); } From 140ee7e287167dd4d3c0a5ce2e02964e480f0bd7 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Tue, 28 Aug 2018 17:14:20 +0530 Subject: [PATCH 099/135] brcmfmac: reset PMU, backplane & all cores in CYW4373 during rmmod To do a clean reset of the chip during rmmod brcmfmac, program the PmuWatchdogCounter register. When a watchdog reset occurs, the PMU, backplane and all of the cores in the chip are reset. Signed-off-by: Madhan Mohan R Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/chip.c | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index b63521524a02d..8285045a9620a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -222,9 +222,15 @@ struct sbsocramregs { /* PMU CONTROL EXT mask for 43012C0 */ #define CY_43012_PMU_CONTROL_EXT_MASK 0x11 -/* PMU CONTROL EXT mask for 43012C0 */ +/* PMU Watchdog Counter Tick value for 43012C0 */ #define CY_43012_PMU_WATCHDOG_TICK_VAL 0x04 +/* PMU Watchdog Counter Tick value for 4373 */ +#define CY_4373_PMU_WATCHDOG_TICK_VAL 0x04 + +/* Minimum PMU resource mask for 4373 */ +#define CY_4373_PMU_MIN_RES_MASK 0xFCAFF7F + struct brcmf_core_priv { struct brcmf_core pub; u32 wrapbase; @@ -1583,7 +1589,18 @@ void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) chip->ops->write32(chip->ctx, addr, CY_43012_PMU_WATCHDOG_TICK_VAL); break; - + case CY_CC_4373_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_WATCHDOG_TICK_VAL); + mdelay(100); + break; default: break; } From dd8f661a6dd93eba78b9853d74b85344ef44807e Mon Sep 17 00:00:00 2001 From: Able Liao Date: Thu, 20 Feb 2020 15:49:23 +0800 Subject: [PATCH 100/135] brcmfmac: do not disconnect for disassoc frame from unconnected AP Ignore FW event if the event's BSSID is different form the BSSID of the currently connected AP. Check interface state is connected or not, if state is not connected that can ignore link down event. Signed-off-by: Able Liao Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d5a21b6c100c9..e6293809e2970 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6134,7 +6134,12 @@ brcmf_notify_connect_status(struct brcmf_if *ifp, brcmf_net_setcarrier(ifp, true); } else if (brcmf_is_linkdown(e)) { brcmf_dbg(CONN, "Linkdown\n"); - if (!brcmf_is_ibssmode(ifp->vif)) { + if (!brcmf_is_ibssmode(ifp->vif) && + test_bit(BRCMF_VIF_STATUS_CONNECTED, + &ifp->vif->sme_state)) { + if (memcmp(profile->bssid, e->addr, ETH_ALEN)) + return err; + brcmf_bss_connect_done(cfg, ndev, e, false); brcmf_link_down(ifp->vif, brcmf_map_fw_linkdown_reason(e), From ee68f327b77e7fa4c286d8de046b0dc7093601e5 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 6 Mar 2020 04:16:34 -0600 Subject: [PATCH 101/135] brcmfmac: Set pacing shift before transmitting skb to bus Linux 3.6 introduces TSQ which has a per socket threshold for TCP Tx packet to reduce latency. In flow control mode , host driver enqueues skb in hanger and TCP doesn't push new skb frees until host frees the skb when receiving fwstatus event. So set pacing shift 8 to send them as a single large aggregate frame to the bus layer. 43455 TX TCP throughput in FC mode 2 on Linux 5.4.18 sk_pacing_shift : Throughput 10: 190 Mbps 9: 216 Mbps 8: 233 Mbps (Max throughput) 7: 233 Mpbs Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index f290a5f3cdc22..74e69561b31e3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -352,6 +352,9 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, if ((skb->priority == 0) || (skb->priority > 7)) skb->priority = cfg80211_classify8021d(skb, NULL); + /* set pacing shift for packet aggregation */ + sk_pacing_shift_update(skb->sk, 8); + ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) brcmf_txfinalize(ifp, skb, false); From a6c3f1558fc34b2970a4cd0728ca4130d8635a0b Mon Sep 17 00:00:00 2001 From: Pramod Prakash Date: Fri, 2 Aug 2019 12:47:28 +0530 Subject: [PATCH 102/135] brcmfmac: fix 802.1d priority to ac mapping for pcie dongles 802.1d defines 0,3 for BE and 1,2 for BK. In pcie dongles, 0 & 3 are mapped to 0 and 1,2 are mapped to 1. This change corrects this mapping, so that BE & BK are given access precedence accordingly by pcie dongles. Signed-off-by: Pramod Prakash Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index b1727c442377e..e1127d7e086d5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -26,10 +26,10 @@ #define BRCMF_FLOWRING_HASH_STA(fifo, ifidx) (fifo + ifidx * 16) static const u8 brcmf_flowring_prio2fifo[] = { - 1, - 0, 0, 1, + 1, + 0, 2, 2, 3, From 14156a8cb8c2398d5218a919e0a5447545891e53 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 1 Jan 2018 20:12:54 -0600 Subject: [PATCH 103/135] non-upstream: calling skb_orphan before sending skb to SDIO bus Linux 3.6 introduces TSQ which has a per socket threshold for TCP Tx packet to reduce latency. In fcmode 1/2, host driver enqueues skb in hanger and TCP doesn't push new skb frees until host frees the skb when receiving fwstatus event. So using skb_orphan before sending skb to bus will make the skb removing the ownership of socket. With this patch, we got better throughput in fcmode 1/2. We only call skb_orphan when Linux version is less than 4.16.0. Tested 43455 TCP throughput in 20 MHz bandwidth with/without this patch. fcmode 0: 59.5 / 59.6 (Mbps) fcmode 1: 59.3 / 23.4 (Mbps) fcmode 2: 59.6 / 21.5 (Mbps) Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index aa5046c5b017b..70f99081f8815 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -2349,6 +2349,9 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) &prec_out); if (pkt == NULL) break; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + skb_orphan(pkt); +#endif __skb_queue_tail(&pktq, pkt); } spin_unlock_bh(&bus->txq_lock); From b86bd273dac3723951f5f520985ea403601547b6 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Fri, 4 Jan 2019 14:42:04 +0530 Subject: [PATCH 104/135] non-upstream: workaround for 4373 USB WMM 5.2.27 test failure With the addition of skb_orphan in the datapath, though the throughput increases in TX path, it introduces an issue by removing the flowcontrol from upper layer and allowing more data to flow for different access category. This workaround is to disable using skb_orphan when running multi-stream data. This change will not be required in linux 4.15 and later versions. We only use this feature when Linux version is less than 4.16.0. Signed-off-by: Raveendran Somu Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/bus.h | 3 ++ .../broadcom/brcm80211/brcmfmac/fwsignal.c | 46 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 8 +++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index e62c34849b4ca..9e765c26f3c1b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -152,6 +152,9 @@ struct brcmf_bus { const struct brcmf_bus_ops *ops; struct brcmf_bus_msgbuf *msgbuf; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + bool allow_skborphan; +#endif }; /* diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index bed7cac20935c..ce8d2ee006596 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -496,6 +496,9 @@ struct brcmf_fws_info { bool creditmap_received; u8 mode; bool avoid_queueing; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + int fifo_init_credit[BRCMF_FWS_FIFO_COUNT]; +#endif }; /* @@ -1625,9 +1628,13 @@ static int brcmf_fws_notify_credit_map(struct brcmf_if *ifp, fws->fifo_credit_map |= 1 << i; else fws->fifo_credit_map &= ~(1 << i); + WARN_ONCE(fws->fifo_credit[i] < 0, "fifo_credit[%d] is negative(%d)\n", i, fws->fifo_credit[i]); +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + fws->fifo_init_credit[i] = fws->fifo_credit[i]; +#endif } brcmf_fws_schedule_deq(fws); brcmf_fws_unlock(fws); @@ -2206,6 +2213,38 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_unlock(fws); } +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) +static bool brcmf_fws_ismultistream(struct brcmf_fws_info *fws) +{ + bool ret = false; + u8 credit_usage = 0; + + /* Check only for BE, VI and VO traffic */ + u32 delay_map = fws->fifo_delay_map & + ((1 << BRCMF_FWS_FIFO_AC_BE) | + (1 << BRCMF_FWS_FIFO_AC_VI) | + (1 << BRCMF_FWS_FIFO_AC_VO)); + + if (hweight_long(delay_map) > 1) { + ret = true; + } else { + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_BE] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_BE]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VI] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VI]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VO] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VO]) + credit_usage++; + + if (credit_usage > 1) + ret = true; + } + return ret; +} +#endif + static void brcmf_fws_dequeue_worker(struct work_struct *worker) { struct brcmf_fws_info *fws; @@ -2219,6 +2258,13 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) fws = container_of(worker, struct brcmf_fws_info, fws_dequeue_work); drvr = fws->drvr; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + if (brcmf_fws_ismultistream(fws)) + drvr->bus_if->allow_skborphan = false; + else + drvr->bus_if->allow_skborphan = true; +#endif + brcmf_fws_lock(fws); for (fifo = BRCMF_FWS_FIFO_BCMC; fifo >= 0 && !fws->bus_flow_blocked; fifo--) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 95dfcee6444b5..f854143388272 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -638,7 +638,10 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } - skb_orphan(skb); +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + if (devinfo->bus_pub.bus->allow_skborphan) + skb_orphan(skb); +#endif req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, @@ -1286,6 +1289,9 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) bus->ops = &brcmf_usb_bus_ops; bus->proto_type = BRCMF_PROTO_BCDC; bus->always_use_fws_queue = true; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) + bus->allow_skborphan = true; +#endif #ifdef CONFIG_PM bus->wowl_supported = true; #endif From e8eb9fa594ee786f986b9de6182d7c07ff3e3787 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Fri, 13 Mar 2020 03:08:13 -0500 Subject: [PATCH 105/135] brcmfmac: disable command decode in sdio_aos for 4373 AOS is a part of the SDIOD core that becomes active when the rest of SDIOD is sleeping to keep SDIO bus alive responding to reduced set of commands. Transaction between AOS and SDIOD is not protected, and if cmd 52 is received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 70f99081f8815..ff97a4ca33908 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3599,6 +3599,7 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus, static bool brcmf_sdio_aos_no_decode(struct brcmf_sdio *bus) { if (bus->ci->chip == CY_CC_43012_CHIP_ID || + bus->ci->chip == CY_CC_4373_CHIP_ID || bus->ci->chip == BRCM_CC_4354_CHIP_ID || bus->ci->chip == BRCM_CC_4356_CHIP_ID) return true; From 6777477088e5db044c07d9765f1506f8cf183e93 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Fri, 13 Mar 2020 03:12:05 -0500 Subject: [PATCH 106/135] brcmfmac: disable command decode in sdio_aos for 4339 AOS is a part of the SDIOD core that becomes active when the rest of SDIOD is sleeping to keep SDIO bus alive responding to reduced set of commands. Transaction between AOS and SDIOD is not protected, and if cmd 52 is received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index ff97a4ca33908..257a7a3776a40 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3600,6 +3600,7 @@ static bool brcmf_sdio_aos_no_decode(struct brcmf_sdio *bus) { if (bus->ci->chip == CY_CC_43012_CHIP_ID || bus->ci->chip == CY_CC_4373_CHIP_ID || + bus->ci->chip == BRCM_CC_4339_CHIP_ID || bus->ci->chip == BRCM_CC_4354_CHIP_ID || bus->ci->chip == BRCM_CC_4356_CHIP_ID) return true; From 228dbf19179e1c50e381a00308bfd1f15fd0c41d Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Fri, 13 Mar 2020 03:15:29 -0500 Subject: [PATCH 107/135] brcmfmac: disable command decode in sdio_aos for 43455 AOS is a part of the SDIOD core that becomes active when the rest of SDIOD is sleeping to keep SDIO bus alive responding to reduced set of commands. Transaction between AOS and SDIOD is not protected, and if cmd 52 is received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 257a7a3776a40..ff28c39cd6e2d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3601,6 +3601,7 @@ static bool brcmf_sdio_aos_no_decode(struct brcmf_sdio *bus) if (bus->ci->chip == CY_CC_43012_CHIP_ID || bus->ci->chip == CY_CC_4373_CHIP_ID || bus->ci->chip == BRCM_CC_4339_CHIP_ID || + bus->ci->chip == BRCM_CC_4345_CHIP_ID || bus->ci->chip == BRCM_CC_4354_CHIP_ID || bus->ci->chip == BRCM_CC_4356_CHIP_ID) return true; From e1b214573d9f9520f2d7c67e8903cd9cc4a6b6ad Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Thu, 22 Aug 2019 03:39:06 -0500 Subject: [PATCH 108/135] brcmfmac: support the forwarding packet - Support packet forwarding mechanism on fmac driver for some special usages on PCIE. Please ref to JIRA. - Fixed BE/VI priority issue when pumping iperf - Fix for linux coding style Signed-off-by: Jia-Shyr Chuang Signed-off-by: Ting-Ying Li Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 13 ++- .../broadcom/brcm80211/brcmfmac/core.c | 101 +++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 17 +++ .../broadcom/brcm80211/brcmfmac/msgbuf.c | 31 +++++- 4 files changed, 158 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index e6293809e2970..af51a8258af3f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4751,7 +4751,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, err = -EINVAL; goto exit; } - + ifp->isap = false; /* Interface specific setup */ if (dev_role == NL80211_IFTYPE_AP) { if ((brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) && (!mbss)) @@ -4831,7 +4831,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, err); goto exit; } - + ifp->isap = true; brcmf_dbg(TRACE, "AP mode configuration complete\n"); } else if (dev_role == NL80211_IFTYPE_P2P_GO) { err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec); @@ -4863,6 +4863,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, goto exit; } + ifp->isap = true; brcmf_dbg(TRACE, "GO mode configuration complete\n"); } else { WARN_ON(1); @@ -6117,6 +6118,14 @@ brcmf_notify_connect_status(struct brcmf_if *ifp, } if (brcmf_is_apmode(ifp->vif)) { + if (e->event_code == BRCMF_E_ASSOC_IND || + e->event_code == BRCMF_E_REASSOC_IND) { + brcmf_findadd_sta(ifp, e->addr); + } else if ((e->event_code == BRCMF_E_DISASSOC_IND) || + (e->event_code == BRCMF_E_DEAUTH_IND) || + (e->event_code == BRCMF_E_DEAUTH)) { + brcmf_del_sta(ifp, e->addr); + } err = brcmf_notify_connect_status_ap(cfg, ndev, e, data); } else if (brcmf_is_linkup(ifp->vif, e)) { brcmf_dbg(CONN, "Linkup\n"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 74e69561b31e3..603bed90ec296 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -62,6 +62,14 @@ struct wlc_d11rxhdr { s8 rxpwr[4]; } __packed; +#define BRCMF_IF_STA_LIST_LOCK_INIT(ifp) spin_lock_init(&(ifp)->sta_list_lock) +#define BRCMF_IF_STA_LIST_LOCK(ifp, flags) \ + spin_lock_irqsave(&(ifp)->sta_list_lock, (flags)) +#define BRCMF_IF_STA_LIST_UNLOCK(ifp, flags) \ + spin_unlock_irqrestore(&(ifp)->sta_list_lock, (flags)) + +#define BRCMF_STA_NULL ((struct brcmf_sta *)NULL) + char *brcmf_ifname(struct brcmf_if *ifp) { if (!ifp) @@ -822,7 +830,9 @@ struct brcmf_if *brcmf_add_if(struct brcmf_pub *drvr, s32 bsscfgidx, s32 ifidx, init_waitqueue_head(&ifp->pend_8021x_wait); spin_lock_init(&ifp->netif_stop_lock); - + BRCMF_IF_STA_LIST_LOCK_INIT(ifp); + /* Initialize STA info list */ + INIT_LIST_HEAD(&ifp->sta_list); if (mac_addr != NULL) memcpy(ifp->mac_addr, mac_addr, ETH_ALEN); @@ -1670,3 +1680,92 @@ int brcmf_pktfilter_enable(struct net_device *ndev, bool enable) } return ret; } + +/** Find STA with MAC address ea in an interface's STA list. */ +struct brcmf_sta * +brcmf_find_sta(struct brcmf_if *ifp, const u8 *ea) +{ + struct brcmf_sta *sta; + unsigned long flags; + + BRCMF_IF_STA_LIST_LOCK(ifp, flags); + list_for_each_entry(sta, &ifp->sta_list, list) { + if (!memcmp(sta->ea.octet, ea, ETH_ALEN)) { + brcmf_dbg(INFO, "Found STA: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x into sta list\n", + sta->ea.octet[0], sta->ea.octet[1], + sta->ea.octet[2], sta->ea.octet[3], + sta->ea.octet[4], sta->ea.octet[5]); + BRCMF_IF_STA_LIST_UNLOCK(ifp, flags); + return sta; + } + } + BRCMF_IF_STA_LIST_UNLOCK(ifp, flags); + + return BRCMF_STA_NULL; +} + +/** Add STA into the interface's STA list. */ +struct brcmf_sta * +brcmf_add_sta(struct brcmf_if *ifp, const u8 *ea) +{ + struct brcmf_sta *sta; + unsigned long flags; + + sta = kzalloc(sizeof(*sta), GFP_KERNEL); + if (sta == BRCMF_STA_NULL) { + brcmf_err("Alloc failed\n"); + return BRCMF_STA_NULL; + } + memcpy(sta->ea.octet, ea, ETH_ALEN); + brcmf_dbg(INFO, "Add STA: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x into sta list\n", + sta->ea.octet[0], sta->ea.octet[1], + sta->ea.octet[2], sta->ea.octet[3], + sta->ea.octet[4], sta->ea.octet[5]); + + /* link the sta and the dhd interface */ + sta->ifp = ifp; + INIT_LIST_HEAD(&sta->list); + + BRCMF_IF_STA_LIST_LOCK(ifp, flags); + + list_add_tail(&sta->list, &ifp->sta_list); + + BRCMF_IF_STA_LIST_UNLOCK(ifp, flags); + return sta; +} + +/** Delete STA from the interface's STA list. */ +void +brcmf_del_sta(struct brcmf_if *ifp, const u8 *ea) +{ + struct brcmf_sta *sta, *next; + unsigned long flags; + + BRCMF_IF_STA_LIST_LOCK(ifp, flags); + list_for_each_entry_safe(sta, next, &ifp->sta_list, list) { + if (!memcmp(sta->ea.octet, ea, ETH_ALEN)) { + brcmf_dbg(INFO, "del STA: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x from sta list\n", + ea[0], ea[1], ea[2], ea[3], + ea[4], ea[5]); + list_del(&sta->list); + kfree(sta); + } + } + + BRCMF_IF_STA_LIST_UNLOCK(ifp, flags); +} + +/** Add STA if it doesn't exist. Not reentrant. */ +struct brcmf_sta* +brcmf_findadd_sta(struct brcmf_if *ifp, const u8 *ea) +{ + struct brcmf_sta *sta = NULL; + + sta = brcmf_find_sta(ifp, ea); + + if (!sta) { + /* Add entry */ + sta = brcmf_add_sta(ifp, ea); + } + return sta; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 6def862f60084..0852b6607e73d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -188,6 +188,7 @@ struct brcmf_if { struct brcmf_fws_mac_descriptor *fws_desc; int ifidx; s32 bsscfgidx; + bool isap; u8 mac_addr[ETH_ALEN]; u8 netif_stop; spinlock_t netif_stop_lock; @@ -196,6 +197,19 @@ struct brcmf_if { struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES]; u8 ipv6addr_idx; bool fwil_fwerr; + struct list_head sta_list; /* sll of associated stations */ + spinlock_t sta_list_lock; +}; + +struct ether_addr { + u8 octet[ETH_ALEN]; +}; + +/** Per STA params. A list of dhd_sta objects are managed in dhd_if */ +struct brcmf_sta { + void *ifp; /* associated brcm_if */ + struct ether_addr ea; /* stations ethernet mac address */ + struct list_head list; /* link into brcmf_if::sta_list */ }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); @@ -219,4 +233,7 @@ void __exit brcmf_core_exit(void); int brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, bool add); int brcmf_pktfilter_enable(struct net_device *ndev, bool enable); +void brcmf_del_sta(struct brcmf_if *ifp, const u8 *ea); +struct brcmf_sta *brcmf_find_sta(struct brcmf_if *ifp, const u8 *ea); +struct brcmf_sta *brcmf_findadd_sta(struct brcmf_if *ifp, const u8 *ea); #endif /* BRCMFMAC_CORE_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 9f1e8adde8e33..7d8fb32832607 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -1145,7 +1145,8 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_pub *drvr = msgbuf->drvr; struct msgbuf_rx_complete *rx_complete; - struct sk_buff *skb; + struct sk_buff *skb, *cpskb = NULL; + struct ethhdr *eh; u16 data_offset; u16 buflen; u16 flags; @@ -1194,6 +1195,34 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) return; } + eh = (struct ethhdr *)(skb->data); + if (ifp->isap) { + skb_set_network_header(skb, sizeof(struct ethhdr)); + skb->protocol = eh->h_proto; + skb->priority = cfg80211_classify8021d(skb, NULL); + if (is_unicast_ether_addr(eh->h_dest)) { + if (brcmf_find_sta(ifp, eh->h_dest)) { + /* determine the priority */ + if (skb->priority == 0 || skb->priority > 7) { + skb->priority = + cfg80211_classify8021d(skb, + NULL); + } + brcmf_proto_tx_queue_data(ifp->drvr, + ifp->ifidx, skb); + return; + } + } else { + cpskb = pskb_copy(skb, GFP_ATOMIC); + if (cpskb) { + brcmf_proto_tx_queue_data(ifp->drvr, + ifp->ifidx, + cpskb); + } else { + brcmf_err("Unable to do skb copy\n"); + } + } + } skb->protocol = eth_type_trans(skb, ifp->ndev); brcmf_netif_rx(ifp, skb); } From 8c5352d711d6ae61e288930977a90442ade85679 Mon Sep 17 00:00:00 2001 From: Ting-Ying Li Date: Fri, 20 Mar 2020 05:13:47 -0500 Subject: [PATCH 109/135] brcmfmac: add a variable for packet forwarding condition Some firmware does not support disabling "ap_isolate" via iovar. Therefore, get iovar "ap_isolate" value after setting to determine whether fmac driver should turn on host-based packet forwarding. Signed-off-by: Ting-Ying Li Signed-off-by: Chi-hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 13 ++++++++++++- .../net/wireless/broadcom/brcm80211/brcmfmac/core.h | 1 + .../wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index af51a8258af3f..256604b1527b6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5469,7 +5469,7 @@ brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev, { struct brcmf_if *ifp; int ret = 0; - u32 ap_isolate; + u32 ap_isolate, val; brcmf_dbg(TRACE, "Enter\n"); ifp = netdev_priv(dev); @@ -5480,6 +5480,17 @@ brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev, brcmf_err("ap_isolate iovar failed: ret=%d\n", ret); } + /* Get ap_isolate value from firmware to detemine whether fmac */ + /* driver supports packet forwarding. */ + if (brcmf_fil_iovar_int_get(ifp, "ap_isolate", &val) == 0) { + ifp->fmac_pkt_fwd_en = + ((params->ap_isolate == 0) && (val == 1)) ? + true : false; + } else { + brcmf_err("get ap_isolate iovar failed: ret=%d\n", ret); + ifp->fmac_pkt_fwd_en = false; + } + return ret; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 0852b6607e73d..29c931b1f9768 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -199,6 +199,7 @@ struct brcmf_if { bool fwil_fwerr; struct list_head sta_list; /* sll of associated stations */ spinlock_t sta_list_lock; + bool fmac_pkt_fwd_en; }; struct ether_addr { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 7d8fb32832607..373afdcf970d9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -1195,8 +1195,8 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) return; } - eh = (struct ethhdr *)(skb->data); - if (ifp->isap) { + if (ifp->isap && ifp->fmac_pkt_fwd_en) { + eh = (struct ethhdr *)(skb->data); skb_set_network_header(skb, sizeof(struct ethhdr)); skb->protocol = eh->h_proto; skb->priority = cfg80211_classify8021d(skb, NULL); From 149b1c84c05dd336f0b632fb422d0c9699e9adf1 Mon Sep 17 00:00:00 2001 From: Ting-Ying Li Date: Wed, 1 Apr 2020 21:08:51 -0500 Subject: [PATCH 110/135] non-upstream: don't change arp/nd offload in multicast_work Commit 6c219b008815 ("brcmfmac: disable packet filtering in promiscuous mode") adds brcmf_configure_arp_nd_offload() configuration according to promiscuous mode in _brcmf_set_multicast_list(). It will always enable arp/nd offload when the device is not in promiscuous mode without considering the interface role. Enabling ARP offload for STA/SAP concurent mode has caused the AP interface to send ARP responses with incorrect mac address. Remove the arp/nd offload setting line as a workaround for now. Signed-off-by: Ting-Ying Li Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 603bed90ec296..fe083f8e95f34 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -199,7 +199,6 @@ static void _brcmf_set_multicast_list(struct work_struct *work) if (err < 0) bphy_err(drvr, "Setting BRCMF_C_SET_PROMISC failed, %d\n", err); - brcmf_configure_arp_nd_offload(ifp, !cmd_value); } #if IS_ENABLED(CONFIG_IPV6) From 5968325a495bb790b7fc47d230f85ffe22a1d69a Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 6 Apr 2020 13:55:48 -0500 Subject: [PATCH 111/135] cleanup brcmfmac patchset Signed-off-by: Robert Nelson --- .../broadcom/brcm80211/brcmfmac/bus.h | 3 -- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 45 ------------------- .../broadcom/brcm80211/brcmfmac/sdio.c | 3 -- .../broadcom/brcm80211/brcmfmac/usb.c | 7 --- 4 files changed, 58 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 9e765c26f3c1b..e62c34849b4ca 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -152,9 +152,6 @@ struct brcmf_bus { const struct brcmf_bus_ops *ops; struct brcmf_bus_msgbuf *msgbuf; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - bool allow_skborphan; -#endif }; /* diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index ce8d2ee006596..23575351763cf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -496,9 +496,6 @@ struct brcmf_fws_info { bool creditmap_received; u8 mode; bool avoid_queueing; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - int fifo_init_credit[BRCMF_FWS_FIFO_COUNT]; -#endif }; /* @@ -1632,9 +1629,6 @@ static int brcmf_fws_notify_credit_map(struct brcmf_if *ifp, WARN_ONCE(fws->fifo_credit[i] < 0, "fifo_credit[%d] is negative(%d)\n", i, fws->fifo_credit[i]); -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - fws->fifo_init_credit[i] = fws->fifo_credit[i]; -#endif } brcmf_fws_schedule_deq(fws); brcmf_fws_unlock(fws); @@ -2213,38 +2207,6 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_unlock(fws); } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) -static bool brcmf_fws_ismultistream(struct brcmf_fws_info *fws) -{ - bool ret = false; - u8 credit_usage = 0; - - /* Check only for BE, VI and VO traffic */ - u32 delay_map = fws->fifo_delay_map & - ((1 << BRCMF_FWS_FIFO_AC_BE) | - (1 << BRCMF_FWS_FIFO_AC_VI) | - (1 << BRCMF_FWS_FIFO_AC_VO)); - - if (hweight_long(delay_map) > 1) { - ret = true; - } else { - if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_BE] < - fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_BE]) - credit_usage++; - if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VI] < - fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VI]) - credit_usage++; - if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VO] < - fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VO]) - credit_usage++; - - if (credit_usage > 1) - ret = true; - } - return ret; -} -#endif - static void brcmf_fws_dequeue_worker(struct work_struct *worker) { struct brcmf_fws_info *fws; @@ -2258,13 +2220,6 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) fws = container_of(worker, struct brcmf_fws_info, fws_dequeue_work); drvr = fws->drvr; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - if (brcmf_fws_ismultistream(fws)) - drvr->bus_if->allow_skborphan = false; - else - drvr->bus_if->allow_skborphan = true; -#endif - brcmf_fws_lock(fws); for (fifo = BRCMF_FWS_FIFO_BCMC; fifo >= 0 && !fws->bus_flow_blocked; fifo--) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index ff28c39cd6e2d..790d65bafe1b0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -2349,9 +2349,6 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) &prec_out); if (pkt == NULL) break; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - skb_orphan(pkt); -#endif __skb_queue_tail(&pktq, pkt); } spin_unlock_bh(&bus->txq_lock); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index f854143388272..8c61f0ee4ca19 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -638,10 +638,6 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - if (devinfo->bus_pub.bus->allow_skborphan) - skb_orphan(skb); -#endif req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, @@ -1289,9 +1285,6 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) bus->ops = &brcmf_usb_bus_ops; bus->proto_type = BRCMF_PROTO_BCDC; bus->always_use_fws_queue = true; -#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)) - bus->allow_skborphan = true; -#endif #ifdef CONFIG_PM bus->wowl_supported = true; #endif From 035edbbb05e3ddc75a28cc144496e8a062adfcc9 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 22 Apr 2020 10:56:00 -0500 Subject: [PATCH 112/135] ar1021_i2c.c: introduce invert/swap/offsets options Signed-off-by: Robert Nelson --- drivers/input/touchscreen/ar1021_i2c.c | 81 ++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/drivers/input/touchscreen/ar1021_i2c.c b/drivers/input/touchscreen/ar1021_i2c.c index 28644f372bd8e..8d5adf1b7e800 100644 --- a/drivers/input/touchscreen/ar1021_i2c.c +++ b/drivers/input/touchscreen/ar1021_i2c.c @@ -17,6 +17,7 @@ #define AR1021_MAX_X 4095 #define AR1021_MAX_Y 4095 +#define AR1021_MAX_PRESSURE 255 #define AR1021_CMD 0x55 @@ -26,8 +27,29 @@ struct ar1021_i2c { struct i2c_client *client; struct input_dev *input; u8 data[AR1021_TOCUH_PKG_SIZE]; + bool invert_x; + bool invert_y; + bool swap_xy; }; +static bool ar1021_get_prop_u32(struct device *dev, + const char *property, + unsigned int default_value, + unsigned int *value) +{ + u32 val; + int error; + + error = device_property_read_u32(dev, property, &val); + if (error) { + *value = default_value; + return false; + } + + *value = val; + return true; +} + static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) { struct ar1021_i2c *ar1021 = dev_id; @@ -49,9 +71,22 @@ static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) x = ((data[2] & 0x1f) << 7) | (data[1] & 0x7f); y = ((data[4] & 0x1f) << 7) | (data[3] & 0x7f); - input_report_abs(input, ABS_X, x); - input_report_abs(input, ABS_Y, y); + if (ar1021->invert_x) + x = AR1021_MAX_X - x; + + if (ar1021->invert_y) + y = AR1021_MAX_Y - y; + + if (ar1021->swap_xy) { + input_report_abs(input, ABS_X, y); + input_report_abs(input, ABS_Y, x); + } else { + input_report_abs(input, ABS_X, x); + input_report_abs(input, ABS_Y, y); + } + input_report_key(input, BTN_TOUCH, button); + input_report_abs(input, ABS_PRESSURE, AR1021_MAX_PRESSURE); input_sync(input); out: @@ -93,6 +128,8 @@ static int ar1021_i2c_probe(struct i2c_client *client, struct ar1021_i2c *ar1021; struct input_dev *input; int error; + unsigned int offset_x, offset_y; + bool data_present; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { dev_err(&client->dev, "i2c_check_functionality error\n"); @@ -116,10 +153,44 @@ static int ar1021_i2c_probe(struct i2c_client *client, input->open = ar1021_i2c_open; input->close = ar1021_i2c_close; + ar1021->invert_x = device_property_read_bool(&client->dev, "touchscreen-inverted-x"); + ar1021->invert_y = device_property_read_bool(&client->dev, "touchscreen-inverted-y"); + ar1021->swap_xy = device_property_read_bool(&client->dev, "touchscreen-swapped-x-y"); + + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-x", + 0, + &offset_x); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-x: %d\n", offset_x); + + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-y", + 0, + &offset_y); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-y: %d\n", offset_y); + __set_bit(INPUT_PROP_DIRECT, input->propbit); - input_set_capability(input, EV_KEY, BTN_TOUCH); - input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0); - input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0); + //input_set_capability(input, EV_KEY, BTN_TOUCH); + + input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); + input->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); + + if(ar1021->swap_xy) + { + input_set_abs_params(input, ABS_X, 0, AR1021_MAX_Y, 0, 0); + input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_X, 0, 0); + } + else + { + input_set_abs_params(input, ABS_X, offset_x, AR1021_MAX_X-offset_x, 0, 0); + input_set_abs_params(input, ABS_Y, offset_y, AR1021_MAX_Y-offset_y, 0, 0); + } + + input_set_abs_params(input, ABS_PRESSURE, 0, AR1021_MAX_PRESSURE, 0, 0); input_set_drvdata(input, ar1021); From bbd8b48efb1c988388d297a7be7c4afd951100ba Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 31 Aug 2016 10:39:07 -0600 Subject: [PATCH 113/135] Fix remoteproc to work with the PRU GNU Binutils port PRU IRAM addresses need to be masked before being handled to remoteproc. This is due to PRU Binutils' lack of separate address spaces for IRAM and DRAM. Signed-off-by: Dimitar Dimitrov --- drivers/remoteproc/pru_rproc.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index cbcd8a6e420e6..ea2240e411851 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -969,6 +969,14 @@ static void *pru_i_da_to_va(struct pru_rproc *pru, u32 da, int len) u32 offset; void *va = NULL; + /* GNU binutils do not support multiple address spaces. The + * default linker script from the official GNU pru-ld places + * IRAM at an arbitrary high offset, in order to differentiate it + * from DRAM. Hence we need to strip the artificial offset + * from the IRAM address. + */ + da &= ~0xf0000000u; + if (len <= 0) return NULL; From c0f4dc041285ffe418e6890c782ceedcb92f2ec5 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 22 Apr 2020 11:06:16 -0500 Subject: [PATCH 114/135] sound: give us SND_SOC_PCM5102A Signed-off-by: Robert Nelson --- sound/soc/ti/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/ti/Kconfig b/sound/soc/ti/Kconfig index 278f6f8bb2b24..914fd701ad3cf 100644 --- a/sound/soc/ti/Kconfig +++ b/sound/soc/ti/Kconfig @@ -29,6 +29,7 @@ config SND_SOC_DAVINCI_MCASP select SND_SOC_TI_EDMA_PCM select SND_SOC_TI_SDMA_PCM select SND_SOC_TI_UDMA_PCM + select SND_SOC_PCM5102A help Say Y or M here if you want to have support for McASP IP found in various Texas Instruments SoCs like: From 4e63b4e4ebf545f6ad4420fef7edcd16c940ea23 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Sun, 28 Oct 2018 16:14:23 -0500 Subject: [PATCH 115/135] NFM: spi: spidev: allow use of spidev in DT This reverts commit 956b200a846e324322f6211034c734c65a38e550. Signed-off-by: Robert Nelson --- drivers/spi/spidev.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index be503a0e6ef73..bb8191d677c2e 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -733,9 +733,9 @@ static int spidev_probe(struct spi_device *spi) * compatible string, it is a Linux implementation thing * rather than a description of the hardware. */ - WARN(spi->dev.of_node && - of_device_is_compatible(spi->dev.of_node, "spidev"), - "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node); +// WARN(spi->dev.of_node && +// of_device_is_compatible(spi->dev.of_node, "spidev"), +// "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node); spidev_probe_acpi(spi); From fa2661262db4ebaf21ade7ae3f9505f16012a42d Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 28 Oct 2016 14:14:27 -0500 Subject: [PATCH 116/135] HACK: tps65217_pwr_but Signed-off-by: Robert Nelson --- drivers/input/misc/tps65218-pwrbutton.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c index f011447c44fb6..e0c6deacca181 100644 --- a/drivers/input/misc/tps65218-pwrbutton.c +++ b/drivers/input/misc/tps65218-pwrbutton.c @@ -36,7 +36,7 @@ struct tps6521x_data { static const struct tps6521x_data tps65217_data = { .reg_status = TPS65217_REG_STATUS, .pb_mask = TPS65217_STATUS_PB, - .name = "tps65217_pwrbutton", + .name = "tps65217_pwr_but", }; static const struct tps6521x_data tps65218_data = { From 102c90c871e192009eb86651fcb56ac94213cf29 Mon Sep 17 00:00:00 2001 From: Pantelis Antoniou Date: Sat, 6 Dec 2014 20:10:49 +0200 Subject: [PATCH 117/135] ARM: DT: Enable symbols when CONFIG_OF_OVERLAY is used --- arch/arm/boot/Makefile | 4 ++++ arch/arm/boot/dts/Makefile | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index 0b3cd7a33a264..87a8b8db3411b 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -29,6 +29,10 @@ export ZRELADDR INITRD_PHYS PARAMS_PHYS targets := Image zImage xipImage bootpImage uImage +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + ifeq ($(CONFIG_XIP_KERNEL),y) cmd_deflate_xip_data = $(CONFIG_SHELL) -c \ diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 1b7be948a2bc2..959f0abbe6845 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1,4 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 + +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + dtb-$(CONFIG_ARCH_ALPINE) += \ alpine-db.dtb dtb-$(CONFIG_MACH_ARTPEC6) += \ From 683733f3a98cd367f2b32fa7eda786c9f75b2aa6 Mon Sep 17 00:00:00 2001 From: Jay at Control Module Industries Date: Tue, 19 May 2015 13:35:14 -0500 Subject: [PATCH 118/135] cpsw: search for phy I have encountered the same issue(s) on A6A boards. I couldn't find a patch, so I wrote this patch to update the device tree in the davinci_mdio driver in the 3.15.1 tree, it seems to correct it. I would welcome any input on a different approach. https://groups.google.com/d/msg/beagleboard/9mctrG26Mc8/SRlnumt0LoMJ v4.1-rcX: added hack around CONFIG_OF_OVERLAY v4.2-rc3+: added if (of_machine_is_compatible("ti,am335x-bone")) so we do not break dual ethernet am335x devices Signed-off-by: Robert Nelson --- drivers/net/ethernet/ti/davinci_mdio.c | 97 ++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c index 702fdc393da00..e201a20e2e114 100644 --- a/drivers/net/ethernet/ti/davinci_mdio.c +++ b/drivers/net/ethernet/ti/davinci_mdio.c @@ -92,6 +92,10 @@ struct davinci_mdio_data { u32 clk_div; }; +#if IS_ENABLED(CONFIG_OF) +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask); +#endif + static void davinci_mdio_init_clk(struct davinci_mdio_data *data) { u32 mdio_in, div, mdio_out_khz, access_time; @@ -159,6 +163,12 @@ static int davinci_mdio_reset(struct mii_bus *bus) /* restrict mdio bus to live phys only */ dev_info(data->dev, "detected phy mask %x\n", ~phy_mask); phy_mask = ~phy_mask; + + #if IS_ENABLED(CONFIG_OF) + if (of_machine_is_compatible("ti,am335x-bone")) + davinci_mdio_update_dt_from_phymask(phy_mask); + #endif + } else { /* desperately scan all phys */ dev_warn(data->dev, "no live phy, scanning all\n"); @@ -474,6 +484,93 @@ static int davinci_mdio_runtime_resume(struct device *dev) davinci_mdio_enable(data); return 0; } +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask) +{ + int i, len, skip; + u32 addr; + __be32 *old_phy_p, *phy_id_p; + struct property *phy_id_property = NULL; + struct device_node *node_p, *slave_p; + + addr = 0; + + for (i = 0; i < PHY_MAX_ADDR; i++) { + if ((phy_mask & (1 << i)) == 0) { + addr = (u32) i; + break; + } + } + + for_each_compatible_node(node_p, NULL, "ti,cpsw") { + for_each_node_by_name(slave_p, "slave") { + +#if IS_ENABLED(CONFIG_OF_OVERLAY) + skip = 1; + // Hack, the overlay fixup "slave" doesn't have phy-mode... + old_phy_p = (__be32 *) of_get_property(slave_p, "phy-mode", &len); + + if (len != (sizeof(__be32 *) * 1)) + { + skip = 0; + } + + if (skip) { +#endif + + old_phy_p = (__be32 *) of_get_property(slave_p, "phy_id", &len); + + if (len != (sizeof(__be32 *) * 2)) + goto err_out; + + if (old_phy_p) { + + phy_id_property = kzalloc(sizeof(*phy_id_property), GFP_KERNEL); + + if (! phy_id_property) + goto err_out; + + phy_id_property->length = len; + phy_id_property->name = kstrdup("phy_id", GFP_KERNEL); + phy_id_property->value = kzalloc(len, GFP_KERNEL); + + if (! phy_id_property->name) + goto err_out; + + if (! phy_id_property->value) + goto err_out; + + memcpy(phy_id_property->value, old_phy_p, len); + + phy_id_p = (__be32 *) phy_id_property->value + 1; + + *phy_id_p = cpu_to_be32(addr); + + of_update_property(slave_p, phy_id_property); + pr_info("davinci_mdio: dt: updated phy_id[%d] from phy_mask[%x]\n", addr, phy_mask); + + ++addr; + } +#if IS_ENABLED(CONFIG_OF_OVERLAY) + } +#endif + } + } + + return; + +err_out: + + if (phy_id_property) { + if (phy_id_property->name) + kfree(phy_id_property->name); + + if (phy_id_property->value) + kfree(phy_id_property->value); + + if (phy_id_property) + kfree(phy_id_property); + } +} #endif #ifdef CONFIG_PM_SLEEP From 957f5920a09ac4eff11b74ad440c57e5b23da416 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 16 Jul 2015 17:24:57 -0500 Subject: [PATCH 119/135] quiet: 8250_omap.c use pr_info over pr_err Signed-off-by: Robert Nelson --- drivers/tty/serial/8250/8250_omap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 347418050cc2e..427e0863812ab 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1639,10 +1639,10 @@ static int __init omap8250_console_fixup(void) } add_preferred_console("ttyS", idx, options); - pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", + pr_info("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", idx, idx); - pr_err("This ensures that you still see kernel messages. Please\n"); - pr_err("update your kernel commandline.\n"); + pr_info("This ensures that you still see kernel messages. Please\n"); + pr_info("update your kernel commandline.\n"); return 0; } console_initcall(omap8250_console_fixup); From 998b50aee26be27c061651e3ed8fba49b517a444 Mon Sep 17 00:00:00 2001 From: "bigguiness@gmail.com" Date: Wed, 3 Jan 2018 15:34:07 -0800 Subject: [PATCH 120/135] ti_am335x_tsc.c driver ------=_Part_422_1349561576.1515022447432 Content-Type: text/plain; charset="UTF-8" Hello all, The TI touch screen driver does not work _right_ with the libts-bin package in the jessie image. $ cat /etc/dogtag BeagleBoard.org Debian Image 2018-01-01 $ lsb_release -a No LSB modules are available. Distributor ID: Debian Description: Debian GNU/Linux 8.10 (jessie) Release: 8.10 Codename: jessie $ dpkg -l | grep libts-bin ii libts-bin 1.14-1rcnee0~jessie+20171122 armhf touch screen library utilities $ sudo ts_calibrate ts_setup: No such file or directory It is possible to make it work by setting the TSLIB_TSDEVICE environment variable: $ sudo su # export TSLIB_TSDEVICE=/dev/input/event2 # ts_calibrate But, that's a bit of a pain since the environment variable always needs to be set in order to use the touchscreen. It appears that this version of the utilities uses the INPUT_PROP_DIRECT propbit to automatically detect which /dev/input/event device is the touchscreen. It looks like the following is the only change needed to make it work. Unfortunately, I don't have currently have a way to build a custom kernel for the BeagleBone in order to test it. If there is anyone that could I would appreciate it. Regards, Hartley --- drivers/input/touchscreen/ti_am335x_tsc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 83e685557a197..4e0f28e0b548a 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -459,6 +459,7 @@ static int titsc_probe(struct platform_device *pdev) input_dev->name = "ti-tsc"; input_dev->dev.parent = &pdev->dev; + __set_bit(INPUT_PROP_DIRECT, input_dev->propbit); input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); From 4f69a412007373e3d7db86015125b1d1678cd75b Mon Sep 17 00:00:00 2001 From: frost Date: Fri, 12 May 2017 00:22:29 +0800 Subject: [PATCH 121/135] ti_am335x_tsc: correct formula code to calculate pressure; fix touchscreen jitter problem --- drivers/input/touchscreen/ti_am335x_tsc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 4e0f28e0b548a..dd358bff4a55e 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -34,6 +34,7 @@ #define ADCFSM_STEPID 0x10 #define SEQ_SETTLE 275 #define MAX_12BIT ((1 << 12) - 1) +#define PRESSURE_MAX 1000 #define TSC_IRQENB_MASK (IRQENB_FIFO0THRES | IRQENB_EOS | IRQENB_HW_PEN) @@ -234,6 +235,7 @@ static void titsc_read_coordinates(struct titsc *ts_dev, for (i = 0; i < creads; i++) { xvals[i] = titsc_readl(ts_dev, REG_FIFO0); xvals[i] &= 0xfff; + pr_debug("i %d xval %d yval %d z1 %d z2 %d\n", i, xvals[i], yvals[i], *z1, *z2); } /* @@ -312,13 +314,13 @@ static irqreturn_t titsc_irq(int irq, void *dev) * Resistance(touch) = x plate resistance * * x postion/4096 * ((z2 / z1) - 1) */ - z = z1 - z2; + z = z2 - z1; z *= x; z *= ts_dev->x_plate_resistance; - z /= z2; + z /= z1; z = (z + 2047) >> 12; - - if (z <= MAX_12BIT) { + pr_debug("x %d y %d z1 %d z2 %d z %d\n", x, y, z1, z2, z); + if (z <= PRESSURE_MAX) { input_report_abs(input_dev, ABS_X, x); input_report_abs(input_dev, ABS_Y, y); input_report_abs(input_dev, ABS_PRESSURE, z); From 18b0fba4d9ca7d31c5a0b039a5184c933f36b06d Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jan 2018 13:43:56 -0600 Subject: [PATCH 122/135] BeagleBone pinmux helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authors: Pantelis Antoniou Charles Steinkuehler Jason Kridner Robert Nelson Tobias Müller Matthijs van Duin This patch was derived from 19 commits: https://github.com/RobertCNelson/linux-dev/tree/35e301ae8436e9f56f65bf1a7440021eda42f948/patches/drivers/ti/gpio Signed-off-by: Robert Nelson --- drivers/gpio/Kconfig | 14 + drivers/gpio/Makefile | 1 + drivers/gpio/gpio-of-helper.c | 435 ++++++++++++++++++ drivers/gpio/gpiolib-sysfs.c | 35 +- drivers/misc/Kconfig | 1 + drivers/misc/Makefile | 1 + drivers/misc/cape/Kconfig | 5 + drivers/misc/cape/Makefile | 5 + drivers/misc/cape/beaglebone/Kconfig | 10 + drivers/misc/cape/beaglebone/Makefile | 5 + .../misc/cape/beaglebone/bone-pinmux-helper.c | 242 ++++++++++ 11 files changed, 751 insertions(+), 3 deletions(-) create mode 100644 drivers/gpio/gpio-of-helper.c create mode 100644 drivers/misc/cape/Kconfig create mode 100644 drivers/misc/cape/Makefile create mode 100644 drivers/misc/cape/beaglebone/Kconfig create mode 100644 drivers/misc/cape/beaglebone/Makefile create mode 100644 drivers/misc/cape/beaglebone/bone-pinmux-helper.c diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index f9263426af030..7f0fa364b86c9 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -69,6 +69,20 @@ config GPIO_SYSFS ioctl() operations instead. The character device is always available. +config GPIO_OF_HELPER + bool "GPIO OF helper device (EXPERIMENTAL)" + depends on OF_GPIO + help + Say Y here to add an GPIO OF helper driver + + Allows you specify a GPIO helper based on OF + which allows simple export of GPIO functionality + in user-space. + + Features include, value set/get, direction control, + interrupt/value change poll support, event counting + and others. + config GPIO_GENERIC depends on HAS_IOMEM # Only for IOMEM drivers tristate diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index d2fd19c15bae3..6638fc2464a92 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_GPIOLIB) += gpiolib-devprop.o obj-$(CONFIG_OF_GPIO) += gpiolib-of.o obj-$(CONFIG_GPIO_SYSFS) += gpiolib-sysfs.o obj-$(CONFIG_GPIO_ACPI) += gpiolib-acpi.o +obj-$(CONFIG_GPIO_OF_HELPER) += gpio-of-helper.o # Device drivers. Generally keep list sorted alphabetically obj-$(CONFIG_GPIO_GENERIC) += gpio-generic.o diff --git a/drivers/gpio/gpio-of-helper.c b/drivers/gpio/gpio-of-helper.c new file mode 100644 index 0000000000000..83f362f7da5ff --- /dev/null +++ b/drivers/gpio/gpio-of-helper.c @@ -0,0 +1,435 @@ +/* + * GPIO OF based helper + * + * A simple DT based driver to provide access to GPIO functionality + * to user-space via sysfs. + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* fwd decl. */ +struct gpio_of_helper_info; + +enum gpio_type { + GPIO_TYPE_INPUT = 0, + GPIO_TYPE_OUTPUT = 1, +}; + +struct gpio_of_entry { + int id; + struct gpio_of_helper_info *info; + struct device_node *node; + enum gpio_type type; + int gpio; + int irq; + const char *name; + atomic64_t counter; + unsigned int count_flags; +#define COUNT_RISING_EDGE (1 << 0) +#define COUNT_FALLING_EDGE (1 << 1) +}; + +struct gpio_of_helper_info { + struct platform_device *pdev; + struct idr idr; +}; + +static const struct of_device_id gpio_of_helper_of_match[] = { + { + .compatible = "gpio-of-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, gpio_of_helper_of_match); + +static ssize_t gpio_of_helper_show_status(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + char *p, *e; + int id, n; + + p = buf; + e = p + PAGE_SIZE; + n = 0; + idr_for_each_entry(&info->idr, entry, id) { + switch (entry->type) { + case GPIO_TYPE_INPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s %llu\n", + entry->id, entry->name, entry->gpio, "IN", + (unsigned long long) + atomic64_read(&entry->counter)); + break; + case GPIO_TYPE_OUTPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s\n", + entry->id, entry->name, entry->gpio, "OUT"); + break; + } + p += n; + } + + return p - buf; +} + +static DEVICE_ATTR(status, S_IRUGO, + gpio_of_helper_show_status, NULL); + +static irqreturn_t gpio_of_helper_handler(int irq, void *ptr) +{ + struct gpio_of_entry *entry = ptr; + + /* caution - low speed interfaces only! */ + atomic64_inc(&entry->counter); + + return IRQ_HANDLED; +} + +static struct gpio_of_entry * +gpio_of_entry_create(struct gpio_of_helper_info *info, + struct device_node *node) +{ + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + struct gpio_of_entry *entry; + int err, gpio, irq; + unsigned int req_flags, count_flags, irq_flags; + enum gpio_type type; + enum of_gpio_flags gpio_flags; + const char *name; + + /* get the type of the node first */ + if (of_property_read_bool(node, "input")) + type = GPIO_TYPE_INPUT; + else if (of_property_read_bool(node, "output") + || of_property_read_bool(node, "init-low") + || of_property_read_bool(node, "init-high")) + type = GPIO_TYPE_OUTPUT; + else { + dev_err(dev, "Not valid gpio node type\n"); + err = -EINVAL; + goto err_bad_node; + } + + /* get the name */ + if (of_property_read_string(node, "line-name", &name)) + if (of_property_read_string(node, "gpio-name", &name)) + name = node->name; + + err = of_get_named_gpio_flags(node, "gpio", 0, &gpio_flags); + if (IS_ERR_VALUE(err)) { + dev_err(dev, "Failed to get gpio property of '%s'\n", name); + goto err_bad_node; + } + gpio = err; + + req_flags = 0; + count_flags = 0; + + /* set the request flags */ + switch (type) { + case GPIO_TYPE_INPUT: + req_flags = GPIOF_DIR_IN | GPIOF_EXPORT; + if (of_property_read_bool(node, "count-falling-edge")) + count_flags |= COUNT_FALLING_EDGE; + if (of_property_read_bool(node, "count-rising-edge")) + count_flags |= COUNT_RISING_EDGE; + break; + case GPIO_TYPE_OUTPUT: + req_flags = GPIOF_DIR_OUT | GPIOF_EXPORT; + if (of_property_read_bool(node, "init-high")) + req_flags |= GPIOF_OUT_INIT_HIGH; + else if (of_property_read_bool(node, "init-low")) + req_flags |= GPIOF_OUT_INIT_LOW; + break; + } + if (of_property_read_bool(node, "dir-changeable")) + req_flags |= GPIOF_EXPORT_CHANGEABLE; + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_ACTIVE_LOW; + if (gpio_flags & OF_GPIO_SINGLE_ENDED) { + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_OPEN_DRAIN; + else + req_flags |= GPIOF_OPEN_SOURCE; + } + + /* request the gpio */ + err = devm_gpio_request_one(dev, gpio, req_flags, name); + if (err != 0) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + irq = -1; + irq_flags = 0; + + /* counter mode requested - need an interrupt */ + if (count_flags != 0) { + irq = gpio_to_irq(gpio); + if (IS_ERR_VALUE(irq)) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + if (count_flags & COUNT_RISING_EDGE) + irq_flags |= IRQF_TRIGGER_RISING; + if (count_flags & COUNT_FALLING_EDGE) + irq_flags |= IRQF_TRIGGER_FALLING; + } + +// if (!idr_pre_get(&info->idr, GFP_KERNEL)) { +// dev_err(dev, "Failed on idr_pre_get of '%s'\n", name); +// err = -ENOMEM; +// goto err_no_mem; +// } + + idr_preload(GFP_KERNEL); + + entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL); + if (entry == NULL) { + dev_err(dev, "Failed to allocate gpio entry of '%s'\n", name); + err = -ENOMEM; + goto err_no_mem; + } + + entry->id = -1; + entry->info = info; + entry->node = of_node_get(node); /* get node reference */ + entry->type = type; + entry->gpio = gpio; + entry->irq = irq; + entry->name = name; + + /* interrupt enable is last thing done */ + if (irq >= 0) { + atomic64_set(&entry->counter, 0); + entry->count_flags = count_flags; + err = devm_request_irq(dev, irq, gpio_of_helper_handler, + irq_flags, name, entry); + if (err != 0) { + dev_err(dev, "Failed to request irq of '%s'\n", name); + goto err_no_irq; + } + } + + /* all done; insert */ +// err = idr_get_new(&info->idr, entry, &entry->id); +// if (IS_ERR_VALUE(err)) { +// dev_err(dev, "Failed to idr_get_new of '%s'\n", name); +// goto err_fail_idr; +// } + + err = idr_alloc(&info->idr, entry, 0, 0, GFP_NOWAIT); + if (err >= 0) + entry->id = err; + + idr_preload_end(); + + if (err < 0) { + dev_err(dev, "Failed to idr_get_new of '%s'\n", name); + goto err_fail_idr; + } + + dev_dbg(dev, "Allocated GPIO id=%d name='%s'\n", entry->id, name); + + return entry; + +err_fail_idr: + /* nothing to do */ +err_no_irq: + /* release node ref */ + of_node_put(node); + /* nothing else needs to be done, devres handles it */ +err_no_mem: +err_bad_node: + return ERR_PTR(err); +} + +static int gpio_of_entry_destroy(struct gpio_of_entry *entry) +{ + struct gpio_of_helper_info *info = entry->info; + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + + dev_dbg(dev, "Destroying GPIO id=%d\n", entry->id); + + /* remove from the IDR */ + idr_remove(&info->idr, entry->id); + + /* remove node ref */ + of_node_put(entry->node); + + /* free gpio */ + devm_gpio_free(dev, entry->gpio); + + /* gree irq */ + if (entry->irq >= 0) + devm_free_irq(dev, entry->irq, entry); + + /* and free */ + devm_kfree(dev, entry); + + return 0; +} + +static int gpio_of_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct gpio_of_helper_info *info; + struct gpio_of_entry *entry; + struct device_node *pnode = pdev->dev.of_node; + struct device_node *cnode; + struct pinctrl *pinctrl; + int err; + + /* we only support OF */ + if (pnode == NULL) { + dev_err(&pdev->dev, "No platform of_node!\n"); + return -ENODEV; + } + + pinctrl = devm_pinctrl_get_select_default(&pdev->dev); + if (IS_ERR(pinctrl)) { + /* special handling for probe defer */ + if (PTR_ERR(pinctrl) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + dev_warn(&pdev->dev, + "pins are not configured from the driver\n"); + } + + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); + if (info == NULL) { + dev_err(&pdev->dev, "Failed to allocate info\n"); + err = -ENOMEM; + goto err_no_mem; + } + platform_set_drvdata(pdev, info); + info->pdev = pdev; + + idr_init(&info->idr); + + err = device_create_file(dev, &dev_attr_status); + if (err != 0) { + dev_err(dev, "Failed to create status sysfs attribute\n"); + goto err_no_sysfs; + } + + for_each_child_of_node(pnode, cnode) { + + entry = gpio_of_entry_create(info, cnode); + if (IS_ERR_OR_NULL(entry)) { + dev_err(dev, "Failed to create gpio entry\n"); + err = PTR_ERR(entry); + goto err_fail_entry; + } + } + + dev_info(&pdev->dev, "ready\n"); + + return 0; +err_fail_entry: + device_remove_file(&pdev->dev, &dev_attr_status); +err_no_sysfs: +err_no_mem: + return err; +} + +static int gpio_of_helper_remove(struct platform_device *pdev) +{ + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + int id; + + dev_info(&pdev->dev, "removing\n"); + + device_remove_file(&pdev->dev, &dev_attr_status); + + id = 0; + idr_for_each_entry(&info->idr, entry, id) { + /* destroy each and every one */ + gpio_of_entry_destroy(entry); + } + + return 0; +} + +#ifdef CONFIG_PM +//#ifdef CONFIG_PM_RUNTIME +static int gpio_of_helper_runtime_suspend(struct device *dev) +{ + /* place holder */ + return 0; +} + +static int gpio_of_helper_runtime_resume(struct device *dev) +{ + /* place holder */ + return 0; +} +//#endif /* CONFIG_PM_RUNTIME */ + +static struct dev_pm_ops gpio_of_helper_pm_ops = { + SET_RUNTIME_PM_OPS(gpio_of_helper_runtime_suspend, + gpio_of_helper_runtime_resume, NULL) +}; +#define GPIO_OF_HELPER_PM_OPS (&gpio_of_helper_pm_ops) +#else +#define GPIO_OF_HELPER_PM_OPS NULL +#endif /* CONFIG_PM */ + +struct platform_driver gpio_of_helper_driver = { + .probe = gpio_of_helper_probe, + .remove = gpio_of_helper_remove, + .driver = { + .name = "gpio-of-helper", + .owner = THIS_MODULE, + .pm = GPIO_OF_HELPER_PM_OPS, + .of_match_table = gpio_of_helper_of_match, + }, +}; + +module_platform_driver(gpio_of_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou "); +MODULE_DESCRIPTION("GPIO OF Helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:gpio-of-helper"); diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index fbf6b1a0a4fae..ec7aa6f3c56e4 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -37,10 +37,10 @@ static DEFINE_MUTEX(sysfs_lock); /* * /sys/class/gpio/gpioN... only for GPIOs that are exported * /direction - * * MAY BE OMITTED if kernel won't allow direction changes * * is read/write as "in" or "out" * * may also be written as "high" or "low", initializing * output value as specified ("out" implies "low") + * * read-only if kernel won't allow direction changes * /value * * always readable, subject to hardware behavior * * may be writable, as zero/nonzero @@ -53,6 +53,8 @@ static DEFINE_MUTEX(sysfs_lock); * * is read/write as zero/nonzero * * also affects existing and subsequent "falling" and "rising" * /edge configuration + * /label + * * descriptor label */ static ssize_t direction_show(struct device *dev, @@ -83,7 +85,9 @@ static ssize_t direction_store(struct device *dev, mutex_lock(&data->mutex); - if (sysfs_streq(buf, "high")) + if (!data->direction_can_change) + status = -EPERM; + else if (sysfs_streq(buf, "high")) status = gpiod_direction_output_raw(desc, 1); else if (sysfs_streq(buf, "out") || sysfs_streq(buf, "low")) status = gpiod_direction_output_raw(desc, 0); @@ -362,6 +366,23 @@ static ssize_t active_low_store(struct device *dev, } static DEVICE_ATTR_RW(active_low); +static ssize_t label_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gpiod_data *data = dev_get_drvdata(dev); + struct gpio_desc *desc = data->desc; + ssize_t status; + + mutex_lock(&data->mutex); + + status = sprintf(buf, "%s\n", desc->label); + + mutex_unlock(&data->mutex); + + return status; +} +static DEVICE_ATTR_RO(label); + static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -373,12 +394,15 @@ static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, if (attr == &dev_attr_direction.attr) { if (!show_direction) - mode = 0; + mode &= 0444; } else if (attr == &dev_attr_edge.attr) { if (gpiod_to_irq(desc) < 0) mode = 0; if (!show_direction && test_bit(FLAG_IS_OUT, &desc->flags)) mode = 0; + } else if (attr == &dev_attr_value.attr) { + if (!show_direction && !test_bit(FLAG_IS_OUT, &desc->flags)) + mode &= 0444; } return mode; @@ -389,6 +413,7 @@ static struct attribute *gpio_attrs[] = { &dev_attr_edge.attr, &dev_attr_value.attr, &dev_attr_active_low.attr, + &dev_attr_label.attr, NULL, }; @@ -402,6 +427,10 @@ static const struct attribute_group *gpio_groups[] = { NULL }; +/* bwlegh, a second device in the same file... get out of my namespace! */ +#define dev_attr_label dev_attr_chip_label +#define label_show chip_label_show + /* * /sys/class/gpio/gpiochipN/ * /base ... matching gpio_chip.base (N) diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 3589583fc4454..e9958d875c89d 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -488,6 +488,7 @@ source "drivers/misc/mei/Kconfig" source "drivers/misc/vmw_vmci/Kconfig" source "drivers/misc/mic/Kconfig" source "drivers/misc/genwqe/Kconfig" +source "drivers/misc/cape/Kconfig" source "drivers/misc/echo/Kconfig" source "drivers/misc/cxl/Kconfig" source "drivers/misc/ocxl/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 00940aed435f4..48fe4d684684d 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -49,6 +49,7 @@ obj-$(CONFIG_SRAM_EXEC) += sram-exec.o obj-$(CONFIG_SRAM_DMA_HEAP) += sram-dma-heap.o obj-y += mic/ obj-$(CONFIG_GENWQE) += genwqe/ +obj-y += cape/ obj-$(CONFIG_ECHO) += echo/ obj-$(CONFIG_VEXPRESS_SYSCFG) += vexpress-syscfg.o obj-$(CONFIG_CXL_BASE) += cxl/ diff --git a/drivers/misc/cape/Kconfig b/drivers/misc/cape/Kconfig new file mode 100644 index 0000000000000..a2ef85e0415c5 --- /dev/null +++ b/drivers/misc/cape/Kconfig @@ -0,0 +1,5 @@ +# +# Capes +# + +source "drivers/misc/cape/beaglebone/Kconfig" diff --git a/drivers/misc/cape/Makefile b/drivers/misc/cape/Makefile new file mode 100644 index 0000000000000..7c4eb96ab29e4 --- /dev/null +++ b/drivers/misc/cape/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for cape like devices +# + +obj-y += beaglebone/ diff --git a/drivers/misc/cape/beaglebone/Kconfig b/drivers/misc/cape/beaglebone/Kconfig new file mode 100644 index 0000000000000..eeb67828fd829 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Kconfig @@ -0,0 +1,10 @@ +# +# Beaglebone capes +# + +config BEAGLEBONE_PINMUX_HELPER + tristate "Beaglebone Pinmux Helper" + depends on ARCH_OMAP2PLUS && OF + default n + help + Say Y here to include support for the pinmux helper diff --git a/drivers/misc/cape/beaglebone/Makefile b/drivers/misc/cape/beaglebone/Makefile new file mode 100644 index 0000000000000..7f4617a01e262 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for beaglebone capes +# + +obj-$(CONFIG_BEAGLEBONE_PINMUX_HELPER) += bone-pinmux-helper.o diff --git a/drivers/misc/cape/beaglebone/bone-pinmux-helper.c b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c new file mode 100644 index 0000000000000..d81363a77b103 --- /dev/null +++ b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c @@ -0,0 +1,242 @@ +/* + * Pinmux helper driver + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct of_device_id bone_pinmux_helper_of_match[] = { + { + .compatible = "bone-pinmux-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, bone_pinmux_helper_of_match); + +struct pinmux_helper_data { + struct pinctrl *pinctrl; + char *selected_state_name; +}; + +static ssize_t pinmux_helper_show_state(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + const char *name; + + name = data->selected_state_name; + if (name == NULL || strlen(name) == 0) + name = "none"; + return sprintf(buf, "%s\n", name); +} + +static ssize_t pinmux_helper_store_state(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct pinctrl_state *state; + char *state_name; + char *s; + int err; + + /* duplicate (as a null terminated string) */ + state_name = kmalloc(count + 1, GFP_KERNEL); + if (state_name == NULL) + return -ENOMEM; + memcpy(state_name, buf, count); + state_name[count] = '\0'; + + /* and chop off newline */ + s = strchr(state_name, '\n'); + if (s != NULL) + *s = '\0'; + + /* try to select default state at first (if it exists) */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) + dev_err(dev, "Failed to select state %s\n", + state_name); + } else { + dev_err(dev, "Failed to find state %s\n", state_name); + err = PTR_RET(state); + } + + if (err == 0) { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + } + + return err ? err : count; +} + +static DEVICE_ATTR(state, S_IWUSR | S_IRUGO, + pinmux_helper_show_state, pinmux_helper_store_state); + +static struct attribute *pinmux_helper_attributes[] = { + &dev_attr_state.attr, + NULL +}; + +static const struct attribute_group pinmux_helper_attr_group = { + .attrs = pinmux_helper_attributes, +}; + +static int bone_pinmux_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pinmux_helper_data *data; + struct pinctrl_state *state; + char *state_name; + const char *mode_name; + int mode_len; + int err; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (data == NULL) { + dev_err(dev, "Failed to allocate data\n"); + err = -ENOMEM; + goto err_no_mem; + } + + state_name = kmalloc(strlen(PINCTRL_STATE_DEFAULT) + 1, + GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_state_mem; + } + data->selected_state_name = state_name; + strcpy(data->selected_state_name, PINCTRL_STATE_DEFAULT); + + platform_set_drvdata(pdev, data); + + data->pinctrl = devm_pinctrl_get(dev); + if (IS_ERR(data->pinctrl)) { + dev_err(dev, "Failed to get pinctrl\n"); + err = PTR_RET(data->pinctrl); + goto err_no_pinctrl; + } + + /* See if an initial mode is specified in the device tree */ + mode_name = of_get_property(dev->of_node, "mode", &mode_len); + + err = -1; + if (mode_name != NULL ) { + state_name = kmalloc(mode_len + 1, GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_mode_mem; + } + strncpy(state_name, mode_name, mode_len); + + /* try to select requested mode */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_warn(dev, "Unable to select requested mode %s\n", state_name); + kfree(state_name); + } else { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + dev_notice(dev, "Set initial pinmux mode to %s\n", state_name); + } + } + } + + /* try to select default state if mode_name failed */ + if ( err != 0) { + state = pinctrl_lookup_state(data->pinctrl, + data->selected_state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_err(dev, "Failed to select default state\n"); + goto err_no_state; + } + } else { + data->selected_state_name = '\0'; + } + } + + /* Register sysfs hooks */ + err = sysfs_create_group(&dev->kobj, &pinmux_helper_attr_group); + if (err) { + dev_err(dev, "Failed to create sysfs group\n"); + goto err_no_sysfs; + } + + return 0; + +err_no_sysfs: +err_no_state: +err_no_mode_mem: + devm_pinctrl_put(data->pinctrl); +err_no_pinctrl: + devm_kfree(dev, data->selected_state_name); +err_no_state_mem: + devm_kfree(dev, data); +err_no_mem: + return err; +} + +static int bone_pinmux_helper_remove(struct platform_device *pdev) +{ + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + + sysfs_remove_group(&dev->kobj, &pinmux_helper_attr_group); + kfree(data->selected_state_name); + devm_pinctrl_put(data->pinctrl); + devm_kfree(dev, data); + + return 0; +} + +struct platform_driver bone_pinmux_helper_driver = { + .probe = bone_pinmux_helper_probe, + .remove = bone_pinmux_helper_remove, + .driver = { + .name = "bone-pinmux-helper", + .owner = THIS_MODULE, + .of_match_table = bone_pinmux_helper_of_match, + }, +}; + +module_platform_driver(bone_pinmux_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou"); +MODULE_DESCRIPTION("Beaglebone pinmux helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:bone-pinmux-helper"); From ef8a288b2ba4aa4da1de1e3fa2a06d217da7babf Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Sun, 30 Dec 2018 19:44:02 -0600 Subject: [PATCH 123/135] hack: gpiolib: yes we have drivers stomping on each other, we need to find a better way to share gpio... Signed-off-by: Robert Nelson --- drivers/gpio/gpiolib.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index abdf448b11a3d..87ea68f2c80e1 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2687,10 +2687,10 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) { desc_set_label(desc, label ? : "?"); ret = 0; - } else { - kfree_const(label); - ret = -EBUSY; - goto done; +// } else { +// kfree_const(label); +// ret = -EBUSY; +// goto done; } if (chip->request) { From 5b9f34f211bdc62cdc6a0b2fa720002526f3b201 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 27 Jul 2020 15:51:34 -0500 Subject: [PATCH 124/135] Overlays: Port RPi Overlay building Signed-off-by: Robert Nelson --- Makefile | 8 ++++---- arch/arm/boot/dts/Makefile | 11 +++-------- scripts/Makefile.build | 1 - scripts/Makefile.dtbinst | 13 +++++++------ scripts/Makefile.lib | 21 +++++++++++---------- 5 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 544eb010012c9..533e1750de688 100644 --- a/Makefile +++ b/Makefile @@ -1261,10 +1261,11 @@ endif ifneq ($(dtstree),) -%.dtb %.dtbo: dtbdir=$(patsubst %/,%,$(patsubst %./,%,$(dtstree)/$(dir $@))) +%.dtb: include/config/kernel.release scripts_dtc + $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ -%.dtb %.dtbo: include/config/kernel.release scripts_dtc - $(Q)$(MAKE) $(build)=$(dtbdir) $(dtstree)/$@ +%.dtbo: include/config/kernel.release scripts_dtc + $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ PHONY += dtbs dtbs_install dtbs_check dtbs: include/config/kernel.release scripts_dtc @@ -1744,7 +1745,6 @@ clean: $(clean-dirs) \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ -o -name '*.dtb' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ - -o -name '*.dtbo' -o -name '*.dtbo.S' \ -o -name '*.dwo' -o -name '*.lst' \ -o -name '*.su' -o -name '*.mod' -o -name '*.ns_deps' \ -o -name '.*.d' -o -name '.*.tmp' -o -name '*.mod.c' \ diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 959f0abbe6845..a188f73a45299 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1345,12 +1345,7 @@ dtb-$(CONFIG_ARCH_ASPEED) += \ aspeed-bmc-portwell-neptune.dtb \ aspeed-bmc-quanta-q71l.dtb -always += $(dtb-merge-y) +targets += dtbs dtbs_install +targets += $(dtb-y) -$(addprefix $(obj)/,$(dtb-merge-y)): TI_DTBOS - @$(srctree)/scripts/dtb-merge $(srctree) $(objtree) $@ $(objtree)/scripts/dtc/fdtoverlay $(src)/ti - -TI_DTBOS: - $(Q)$(MAKE) $(build)=$(src)/ti - -PHONY += TI_DTBOS +subdir-y := overlays diff --git a/scripts/Makefile.build b/scripts/Makefile.build index cf8aaebbccba0..24a33c01bbf7c 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -452,7 +452,6 @@ intermediate_targets = $(foreach sfx, $(2), \ # %.tab.o <- %.tab.[ch] <- %.y targets += $(call intermediate_targets, .asn1.o, .asn1.c .asn1.h) \ $(call intermediate_targets, .dtb.o, .dtb.S .dtb) \ - $(call intermediate_targets, .dtbo.o, .dtbo.S .dtbo) \ $(call intermediate_targets, .lex.o, .lex.c) \ $(call intermediate_targets, .tab.o, .tab.c .tab.h) diff --git a/scripts/Makefile.dtbinst b/scripts/Makefile.dtbinst index dd74201bac546..da2af04a6d7da 100644 --- a/scripts/Makefile.dtbinst +++ b/scripts/Makefile.dtbinst @@ -20,6 +20,7 @@ include scripts/Kbuild.include include $(src)/Makefile dtbinst-files := $(sort $(dtb-y) $(if $(CONFIG_OF_ALL_DTBS), $(dtb-))) +dtboinst-files := $(sort $(dtbo-y) $(if $(CONFIG_OF_ALL_DTBS), $(dtb-))) dtbinst-dirs := $(subdir-y) $(subdir-m) # Helper targets for Installing DTBs into the boot directory @@ -28,16 +29,16 @@ quiet_cmd_dtb_install = INSTALL $< install-dir = $(patsubst $(dtbinst_root)%,$(INSTALL_DTBS_PATH)%,$(obj)) -$(filter %.dtb,$(dtbinst-files)): %.dtb: $(obj)/%.dtb +$(dtbinst-files): %.dtb: $(obj)/%.dtb + $(call cmd,dtb_install,$(install-dir)) + +$(dtboinst-files): %.dtbo: $(obj)/%.dtbo $(call cmd,dtb_install,$(install-dir)) $(dtbinst-dirs): $(Q)$(MAKE) $(dtbinst)=$(obj)/$@ -dummy: - @echo > /dev/null - -PHONY += $(dtbinst-files) $(dtbinst-dirs) dummy -__dtbs_install: $(dtbinst-files) $(dtbinst-dirs) dummy +PHONY += $(dtbinst-files) $(dtboinst-files) $(dtbinst-dirs) +__dtbs_install: $(dtbinst-files) $(dtboinst-files) $(dtbinst-dirs) .PHONY: $(PHONY) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index df64ee4f052e8..b5f267a86c817 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -63,8 +63,6 @@ extra-$(CONFIG_OF_ALL_DTBS) += $(dtb-) ifneq ($(CHECK_DTBS),) extra-y += $(patsubst %.dtb,%.dt.yaml, $(dtb-y)) extra-$(CONFIG_OF_ALL_DTBS) += $(patsubst %.dtb,%.dt.yaml, $(dtb-)) -extra-y += $(patsubst %.dtbo,%.dt.yaml, $(dtb-y)) -extra-$(CONFIG_OF_ALL_DTBS) += $(patsubst %.dtbo,%.dt.yaml, $(dtb-)) endif # Add subdir path @@ -275,9 +273,6 @@ cmd_dt_S_dtb= \ $(obj)/%.dtb.S: $(obj)/%.dtb FORCE $(call if_changed,dt_S_dtb) -$(obj)/%.dtbo.S: $(obj)/%.dtbo FORCE - $(call if_changed,dt_S_dtb) - quiet_cmd_dtc = DTC $@ cmd_dtc = mkdir -p $(dir ${dtc-tmp}) ; \ $(HOSTCC) -E $(dtc_cpp_flags) -x assembler-with-cpp -o $(dtc-tmp) $< ; \ @@ -289,9 +284,6 @@ cmd_dtc = mkdir -p $(dir ${dtc-tmp}) ; \ $(obj)/%.dtb: $(src)/%.dts $(DTC) FORCE $(call if_changed_dep,dtc,dtb) -$(obj)/%.dtbo: $(src)/%.dtso $(DTC) FORCE - $(call if_changed_dep,dtc,dtb) - DT_CHECKER ?= dt-validate DT_BINDING_DIR := Documentation/devicetree/bindings DT_TMP_SCHEMA := $(objtree)/$(DT_BINDING_DIR)/processed-schema.yaml @@ -307,8 +299,17 @@ endef $(obj)/%.dt.yaml: $(src)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE $(call if_changed_rule,dtc,yaml) -$(obj)/%.dt.yaml: $(src)/%.dtso $(DTC) $(DT_TMP_SCHEMA) FORCE - $(call if_changed_rule,dtc_dt_yaml) +quiet_cmd_dtco = DTCO $@ +cmd_dtco = mkdir -p $(dir ${dtc-tmp}) ; \ + $(CPP) $(dtc_cpp_flags) -x assembler-with-cpp -o $(dtc-tmp) $< ; \ + $(DTC) -@ -H epapr -O dtb -o $@ -b 0 \ + -i $(dir $<) $(DTC_FLAGS) \ + -Wno-interrupts_property \ + -d $(depfile).dtc.tmp $(dtc-tmp) ; \ + cat $(depfile).pre.tmp $(depfile).dtc.tmp > $(depfile) + +$(obj)/%.dtbo: $(src)/%.dts FORCE + $(call if_changed_dep,dtco) dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp) From 58c19fb0bcd5ed87c48cc452c3b6940429829dd8 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 18 Mar 2020 13:10:20 -0500 Subject: [PATCH 125/135] bootup hacks: move mmc early Signed-off-by: Robert Nelson --- drivers/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/Makefile b/drivers/Makefile index 2a6ac4b2e68ce..a9324ad4e945d 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -55,6 +55,9 @@ obj-$(CONFIG_RESET_CONTROLLER) += reset/ obj-y += tty/ obj-y += char/ +# put mmc early as many morden devices use emm/sd card as rootfs storage +obj-y += mmc/ + # iommu/ comes before gpu as gpu are using iommu controllers obj-y += iommu/ @@ -127,7 +130,6 @@ obj-$(CONFIG_EISA) += eisa/ obj-$(CONFIG_PM_OPP) += opp/ obj-$(CONFIG_CPU_FREQ) += cpufreq/ obj-$(CONFIG_CPU_IDLE) += cpuidle/ -obj-y += mmc/ obj-$(CONFIG_MEMSTICK) += memstick/ obj-$(CONFIG_NEW_LEDS) += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ From 8df3586c41793a44346d9b93fde209a6336ae2fd Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 19 Mar 2020 14:00:05 -0500 Subject: [PATCH 126/135] bootup-hacks: xor select neon or arm4regs already default on BBB/X15, just calcuated everybootup Signed-off-by: Robert Nelson --- arch/arm/include/asm/xor.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h index aefddec79286a..4f0732aa787e3 100644 --- a/arch/arm/include/asm/xor.h +++ b/arch/arm/include/asm/xor.h @@ -209,3 +209,9 @@ static struct xor_block_template xor_block_neon = { #else #define NEON_TEMPLATES #endif + +#ifdef CONFIG_KERNEL_MODE_NEON +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_neon) +#else +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_arm4regs) +#endif From 07d679bc3146c1c47e83e66eb649b4e199be1f54 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 6 Mar 2020 16:53:00 -0600 Subject: [PATCH 127/135] builddeb: Install our dtbs under /boot/dtbs/$version Signed-off-by: Robert Nelson --- scripts/package/builddeb | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/package/builddeb b/scripts/package/builddeb index c4c580f547ef5..5db5186a197d1 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -99,6 +99,7 @@ if is_enabled CONFIG_OF_EARLY_FLATTREE; then # Only some architectures with OF support have this target if [ -d "${srctree}/arch/$SRCARCH/boot/dts" ]; then $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install + $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/boot/dtbs/$version" dtbs_install fi fi From c1748d46ab1024b56194b176f3afbdb16a28cab9 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:07:13 -0600 Subject: [PATCH 128/135] enable: Jenkins: http://gfnd.rcn-ee.org:8080 Signed-off-by: Robert Nelson --- .github/FUNDING.yml | 12 ++++++ .github/ISSUE_TEMPLATE/bug_report.md | 20 +++++++++ Jenkinsfile | 11 +++++ README.md | 7 ++++ jenkins_build.sh | 62 ++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 Jenkinsfile create mode 100644 README.md create mode 100755 jenkins_build.sh diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000000..8b20af1e3599a --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: beagleboard # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: beagleboard # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: https://paypal.me/beagleboard # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000000..94be17e338b34 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Describe how to reproduce the bug** +List all the steps needed to reproduce the bug + +**REQUIRED INFORMATION** +Run this command and paste the output here: +``` +sudo /opt/scripts/tools/version.sh +``` diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000000..1e73bc717e366 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,11 @@ +pipeline { + agent any + + stages { + stage('Build') { + steps { + sh '/bin/bash ./jenkins_build.sh' + } + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..a35e1f064aa79 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +| kernel | normal | rt | +|:---------:|:--------:|:-------:| +|4.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4-rt/) | +|4.9 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9-rt/) | +|4.14 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14-rt/) | +|4.19 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19-rt/) | +|5.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4-rt/) | diff --git a/jenkins_build.sh b/jenkins_build.sh new file mode 100755 index 0000000000000..8fba37bce2880 --- /dev/null +++ b/jenkins_build.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +#gcc: + +# +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.05/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.08/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.08-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabihf/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabihf/gcc-linaro-7.3.1-2018.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf.tar.xz +# + +gcc_html_path="https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/" +gcc_filename_prefix="gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf" +gcc_banner="arm-linux-gnueabihf-gcc (Linaro GCC 7.5-2019.12) 7.5.0" +gcc_copyright="2017" +datestamp="2019.12-gcc-arm-linux-gnueabihf" + +# + +if [ ! -d ${gcc_filename_prefix}/ ] ; then + rm -rf ./gcc-* || true + #wget -c ${site}/${version}/${filename} + wget -c http://gfnd.rcn-ee.org/farm/jenkins-dl/${gcc_filename_prefix}.tar.xz + tar xf ${gcc_filename_prefix}.tar.xz +fi + +export CC=`pwd`/${gcc_filename_prefix}/bin/arm-linux-gnueabihf- + +make ARCH=arm clean +make ARCH=arm bb.org_defconfig + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" zImage]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" zImage +if [ ! -f arch/arm/boot/zImage ] ; then + echo "failed: [arch/arm/boot/zImage]" + exit 1 +fi + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" modules]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" modules +if [ ! -f drivers/spi/spidev.ko ] ; then + echo "failed: [drivers/spi/spidev.ko]" + exit 1 +fi + +echo "[make ARCH=arm CROSS_COMPILE=\"${binary}\" dtbs]" +make ARCH=arm CROSS_COMPILE="ccache ${CC}" dtbs +if [ ! -f arch/arm/boot/dts/am335x-boneblack.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-boneblack.dtb]" + exit 1 +else + if [ -f arch/arm/boot/dts/am335x-pocketbeagle.dts ] ; then + if [ ! -f arch/arm/boot/dts/am335x-pocketbeagle.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-pocketbeagle.dtb]" + exit 1 + fi + fi +fi + +make ARCH=arm clean From 01cc45ce0d4d8c20790cc949a8aedaeeb2112cf7 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:27:41 -0600 Subject: [PATCH 129/135] 5.4.70-ti-r20 patchset From 8297e66e0d3c1d2a7b35719a3229a5d1a1655701 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Dec 2020 15:33:45 -0600 Subject: [PATCH 130/135] 5.4.70-ti-r20 5.4.70-ti-r20 bb.org_defconfig 5.4 TI Delta: https://github.com/RobertCNelson/ti-linux-kernel/compare/eb0f4e6c05153446b62028ac84452101b8280253...05b5a584edd4f87bc81874f4d0ad0b7b0c31875b BBDTBS: https://github.com/beagleboard/BeagleBoard-DeviceTrees/commit/e167dbfc27fbf2c339516e6f6ff3f764c693a106 CAN-ISOTP: https://github.com/hartkopp/can-isotp/commit/748f8efe84ab7c27496d8c8165f28892ba0e8868 TI_AMX3_CM3: http://git.ti.com/gitweb/?p=processor-firmware/ti-amx3-cm3-pm-firmware.git;a=commit;h=6a849767df85ce9399494f53fb5c753665396653 WIREGUARD: https://git.zx2c4.com/WireGuard/commit/edad0d6e99e5133b1e8e865d727a25fff6399cb4 Signed-off-by: Robert Nelson --- arch/arm/configs/bb.org_defconfig | 2621 +++++++++++++++++++++++++++++ 1 file changed, 2621 insertions(+) create mode 100644 arch/arm/configs/bb.org_defconfig diff --git a/arch/arm/configs/bb.org_defconfig b/arch/arm/configs/bb.org_defconfig new file mode 100644 index 0000000000000..97f7729f6fdd9 --- /dev/null +++ b/arch/arm/configs/bb.org_defconfig @@ -0,0 +1,2621 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZO=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ_IDLE=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_PREEMPT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_PSI=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=m +CONFIG_MEMCG=y +CONFIG_MEMCG_SWAP=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_RDMA=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_BLK_DEV_INITRD=y +# CONFIG_SYSFS_SYSCALL is not set +CONFIG_BPF_SYSCALL=y +CONFIG_USERFAULTFD=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB_FREELIST_RANDOM=y +CONFIG_SLAB_FREELIST_HARDENED=y +CONFIG_PROFILING=y +CONFIG_OMAP_RESET_CLOCKS=y +CONFIG_SOC_OMAP5=y +CONFIG_SOC_AM33XX=y +CONFIG_SOC_AM43XX=y +CONFIG_SOC_DRA7XX=y +CONFIG_SOC_HAS_OMAP2_SDRC=y +CONFIG_OMAP5_ERRATA_801819=y +CONFIG_ARM_THUMBEE=y +CONFIG_PL310_ERRATA_588369=y +CONFIG_PL310_ERRATA_727915=y +CONFIG_PL310_ERRATA_753970=y +CONFIG_PL310_ERRATA_769419=y +CONFIG_ARM_ERRATA_430973=y +CONFIG_ARM_ERRATA_720789=y +CONFIG_ARM_ERRATA_754327=y +CONFIG_ARM_ERRATA_764369=y +CONFIG_ARM_ERRATA_773022=y +CONFIG_ARM_ERRATA_814220=y +CONFIG_SMP=y +CONFIG_MCPM=y +CONFIG_NR_CPUS=2 +CONFIG_ARM_PSCI=y +CONFIG_HZ_250=y +CONFIG_SECCOMP=y +# CONFIG_ATAGS is not set +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_EFI=y +# CONFIG_DMI is not set +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=m +CONFIG_CPU_FREQ_GOV_USERSPACE=m +CONFIG_CPU_FREQ_GOV_ONDEMAND=m +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m +CONFIG_CPUFREQ_DT=y +# CONFIG_ARM_OMAP2PLUS_CPUFREQ is not set +CONFIG_ARM_TI_CPUFREQ=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_ARM_CPUIDLE=y +CONFIG_HIBERNATION=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y +CONFIG_APM_EMULATION=y +CONFIG_ENERGY_MODEL=y +CONFIG_ARM_CRYPTO=y +CONFIG_CRYPTO_SHA1_ARM_NEON=m +CONFIG_CRYPTO_SHA256_ARM=m +CONFIG_CRYPTO_SHA512_ARM=m +CONFIG_CRYPTO_AES_ARM=m +CONFIG_CRYPTO_AES_ARM_BS=m +CONFIG_CRYPTO_CHACHA20_NEON=m +CONFIG_OPROFILE=m +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_COMPRESS=y +CONFIG_MODULE_COMPRESS_XZ=y +CONFIG_BLK_DEV_ZONED=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_WBT=y +CONFIG_BLK_SED_OPAL=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_KARMA_PARTITION=y +CONFIG_MQ_IOSCHED_KYBER=m +CONFIG_IOSCHED_BFQ=m +CONFIG_BFQ_GROUP_IOSCHED=y +CONFIG_BINFMT_MISC=m +CONFIG_KSM=y +CONFIG_FRONTSWAP=y +CONFIG_ZSWAP=y +CONFIG_ZBUD=y +CONFIG_Z3FOLD=m +CONFIG_ZSMALLOC=m +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=m +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=m +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_XDP_SOCKETS=y +CONFIG_INET=y +CONFIG_WIREGUARD=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_MROUTE_MULTIPLE_TABLES=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_NET_IPVTI=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_XFRM=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_SYNPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_LC=m +CONFIG_IP_VS_WLC=m +CONFIG_IP_VS_FO=m +CONFIG_IP_VS_OVF=m +CONFIG_IP_VS_LBLC=m +CONFIG_IP_VS_LBLCR=m +CONFIG_IP_VS_DH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_MH=m +CONFIG_IP_VS_SED=m +CONFIG_IP_VS_NQ=m +CONFIG_IP_VS_FTP=m +CONFIG_IP_VS_PE_SIP=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_FLOW_TABLE_IPV4=m +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_NF_FLOW_TABLE_IPV6=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_IP_DCCP=m +CONFIG_SCTP_COOKIE_HMAC_SHA1=y +CONFIG_RDS=m +CONFIG_RDS_TCP=m +CONFIG_TIPC=m +CONFIG_ATM=m +CONFIG_ATM_CLIP=m +CONFIG_ATM_LANE=m +CONFIG_ATM_MPOA=m +CONFIG_ATM_BR2684=m +CONFIG_L2TP=m +CONFIG_L2TP_DEBUGFS=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m +CONFIG_ATALK=m +CONFIG_DEV_APPLETALK=m +CONFIG_IPDDP=m +CONFIG_IPDDP_ENCAP=y +CONFIG_PHONET=m +CONFIG_6LOWPAN=m +CONFIG_6LOWPAN_GHC_EXT_HDR_HOP=m +CONFIG_6LOWPAN_GHC_UDP=m +CONFIG_6LOWPAN_GHC_ICMPV6=m +CONFIG_6LOWPAN_GHC_EXT_HDR_DEST=m +CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m +CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m +CONFIG_IEEE802154=m +CONFIG_IEEE802154_6LOWPAN=m +CONFIG_MAC802154=m +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_ATM=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_CANID=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_DCB=y +CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set +CONFIG_BATMAN_ADV_NC=y +CONFIG_BATMAN_ADV_DEBUGFS=y +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_MPLS_GSO=y +CONFIG_MPLS_ROUTING=m +CONFIG_MPLS_IPTUNNEL=m +CONFIG_NET_SWITCHDEV=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +CONFIG_BPF_STREAM_PARSER=y +CONFIG_NET_PKTGEN=m +CONFIG_NET_DROP_MONITOR=y +CONFIG_HAMRADIO=y +CONFIG_AX25=m +CONFIG_NETROM=m +CONFIG_ROSE=m +CONFIG_MKISS=m +CONFIG_6PACK=m +CONFIG_BPQETHER=m +CONFIG_BAYCOM_SER_FDX=m +CONFIG_BAYCOM_SER_HDX=m +CONFIG_YAM=m +CONFIG_CAN=m +CONFIG_CAN_VCAN=m +CONFIG_CAN_SLCAN=m +CONFIG_CAN_C_CAN=m +CONFIG_CAN_C_CAN_PLATFORM=m +CONFIG_CAN_HI311X=m +CONFIG_CAN_MCP251X=m +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB2=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CAN_UCAN=m +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_6LOWPAN=m +CONFIG_BT_LEDS=y +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTSDIO=m +CONFIG_BT_MRVL=m +CONFIG_BT_MRVL_SDIO=m +CONFIG_BT_ATH3K=m +CONFIG_BT_WILINK=m +CONFIG_BT_MTKUART=m +CONFIG_AF_RXRPC_IPV6=y +CONFIG_RXKAD=y +CONFIG_CFG80211=m +# CONFIG_CFG80211_DEFAULT_PS is not set +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_MAC80211_MESH=y +CONFIG_RFKILL=y +CONFIG_RFKILL_INPUT=y +CONFIG_NET_9P=m +CONFIG_NET_9P_VIRTIO=m +CONFIG_NFC=m +CONFIG_NFC_DIGITAL=m +CONFIG_NFC_NCI=m +CONFIG_NFC_NCI_SPI=m +CONFIG_NFC_NCI_UART=m +CONFIG_NFC_TRF7970A=m +CONFIG_NFC_SIM=m +CONFIG_NFC_PORT100=m +CONFIG_NFC_PN533_USB=m +CONFIG_NFC_PN533_I2C=m +CONFIG_NFC_ST_NCI_I2C=m +CONFIG_NFC_ST_NCI_SPI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m +CONFIG_NFC_ST95HF=m +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_EXTRA_FIRMWARE="am335x-pm-firmware.elf am335x-bone-scale-data.bin am335x-evm-scale-data.bin am43x-evm-scale-data.bin" +CONFIG_EXTRA_FIRMWARE_DIR="firmware" +CONFIG_OMAP_OCP2SCP=y +CONFIG_SIMPLE_PM_BUS=y +CONFIG_CONNECTOR=y +CONFIG_GNSS=m +CONFIG_GNSS_MTK_SERIAL=m +CONFIG_GNSS_SIRF_SERIAL=m +CONFIG_GNSS_UBX_SERIAL=m +CONFIG_MTD=y +CONFIG_MTD_AR7_PARTS=m +CONFIG_MTD_OF_PARTS=m +CONFIG_MTD_BLOCK=m +CONFIG_MTD_BLOCK_RO=m +CONFIG_RFD_FTL=m +CONFIG_SSFDC=m +CONFIG_MTD_OOPS=m +CONFIG_MTD_SWAP=m +CONFIG_MTD_PHYSMAP=m +CONFIG_MTD_PLATRAM=m +CONFIG_MTD_DATAFLASH=m +CONFIG_MTD_SST25L=m +CONFIG_MTD_ONENAND=y +CONFIG_MTD_ONENAND_VERIFY_WRITE=y +CONFIG_MTD_ONENAND_2X_PROGRAM=y +CONFIG_MTD_RAW_NAND=y +CONFIG_MTD_NAND_ECC_SW_BCH=y +CONFIG_MTD_NAND_OMAP2=m +CONFIG_MTD_NAND_NANDSIM=m +CONFIG_MTD_LPDDR=m +CONFIG_MTD_SPI_NOR=m +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_BLOCK=y +CONFIG_OF_OVERLAY=y +CONFIG_BLK_DEV_NULL_BLK=m +CONFIG_ZRAM=m +CONFIG_ZRAM_WRITEBACK=y +CONFIG_ZRAM_MEMORY_TRACKING=y +CONFIG_BLK_DEV_LOOP=m +CONFIG_BLK_DEV_DRBD=m +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=m +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_ATA_OVER_ETH=m +CONFIG_VIRTIO_BLK=m +CONFIG_BLK_DEV_RBD=m +CONFIG_AD525X_DPOT=m +CONFIG_AD525X_DPOT_I2C=m +CONFIG_AD525X_DPOT_SPI=m +CONFIG_ICS932S401=m +CONFIG_APDS9802ALS=m +CONFIG_ISL29003=m +CONFIG_ISL29020=m +CONFIG_SENSORS_TSL2550=m +CONFIG_SENSORS_BH1770=m +CONFIG_SENSORS_APDS990X=m +CONFIG_HMC6352=m +CONFIG_DS1682=m +CONFIG_SRAM=y +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_EEPROM_LEGACY=m +CONFIG_EEPROM_MAX6875=m +CONFIG_EEPROM_93XX46=m +CONFIG_TI_ST=m +CONFIG_BEAGLEBONE_PINMUX_HELPER=y +CONFIG_RAID_ATTRS=m +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=m +CONFIG_BLK_DEV_SR=m +CONFIG_CHR_DEV_SG=m +CONFIG_CHR_DEV_SCH=m +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=m +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=m +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=m +CONFIG_SCSI_UFSHCD=m +CONFIG_LIBFC=m +CONFIG_LIBFCOE=m +CONFIG_SCSI_VIRTIO=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=m +CONFIG_SCSI_DH_HP_SW=m +CONFIG_SCSI_DH_EMC=m +CONFIG_SCSI_DH_ALUA=m +CONFIG_ATA=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=m +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_MD_CLUSTER=m +CONFIG_BCACHE=m +CONFIG_BLK_DEV_DM=m +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_DM_ZONED=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_TCM_FC=m +CONFIG_ISCSI_TARGET=m +CONFIG_NETDEVICES=y +CONFIG_BONDING=m +CONFIG_DUMMY=m +CONFIG_EQUALIZER=m +CONFIG_IFB=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_IPVLAN=m +CONFIG_IPVTAP=m +CONFIG_VXLAN=m +CONFIG_GENEVE=m +CONFIG_GTP=m +CONFIG_MACSEC=m +CONFIG_NETCONSOLE=m +CONFIG_NETCONSOLE_DYNAMIC=y +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_VIRTIO_NET=m +CONFIG_NLMON=m +CONFIG_NET_VRF=m +CONFIG_ATM_DUMMY=m +# CONFIG_NET_VENDOR_ALACRITECH is not set +# CONFIG_NET_VENDOR_AMAZON is not set +# CONFIG_NET_VENDOR_AQUANTIA is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +# CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_VENDOR_CADENCE is not set +# CONFIG_NET_VENDOR_CAVIUM is not set +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_CORTINA is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_GOOGLE is not set +# CONFIG_NET_VENDOR_HISILICON is not set +# CONFIG_NET_VENDOR_HUAWEI is not set +# CONFIG_NET_VENDOR_INTEL is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MELLANOX is not set +CONFIG_ENC28J60=y +CONFIG_ENCX24J600=y +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_NETRONOME is not set +# CONFIG_NET_VENDOR_NI is not set +# CONFIG_NET_VENDOR_PENSANDO is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set +CONFIG_SMC91X=m +CONFIG_SMC911X=m +CONFIG_SMSC911X=m +# CONFIG_NET_VENDOR_SOCIONEXT is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +CONFIG_TI_CPSW_PHY_SEL=y +CONFIG_TI_CPSW=y +CONFIG_TI_CPTS=y +CONFIG_TI_ICSS_IEP=m +CONFIG_TI_PRUETH_ECAP=m +# CONFIG_NET_VENDOR_VIA is not set +CONFIG_WIZNET_W5100=y +CONFIG_WIZNET_W5100_SPI=y +# CONFIG_NET_VENDOR_XILINX is not set +CONFIG_MDIO_BCM_UNIMAC=m +CONFIG_LED_TRIGGER_PHY=y +CONFIG_AT803X_PHY=y +CONFIG_MICREL_PHY=y +CONFIG_MICROCHIP_PHY=y +CONFIG_MICROSEMI_PHY=y +CONFIG_SMSC_PHY=y +CONFIG_VITESSE_PHY=y +CONFIG_PPP=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOATM=m +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_SLIP=m +CONFIG_SLIP_COMPRESSED=y +CONFIG_SLIP_SMART=y +CONFIG_SLIP_MODE_SLIP6=y +CONFIG_USB_CATC=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_LAN78XX=m +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_NCM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_SR9800=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_NET1080=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_CDC_SUBSET=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_ZAURUS=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_HSO=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_CDC_PHONET=m +CONFIG_USB_IPHETH=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_USB_NET_CH9200=m +CONFIG_ATH9K=m +CONFIG_ATH9K_CHANNEL_CONTEXT=y +CONFIG_ATH9K_HTC=m +CONFIG_CARL9170=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_SDIO=m +CONFIG_ATH6KL_USB=m +CONFIG_AR5523=m +CONFIG_ATH10K=m +CONFIG_ATH10K_USB=m +CONFIG_AT76C50X_USB=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_HOSTAP=m +CONFIG_HOSTAP_FIRMWARE=y +CONFIG_P54_COMMON=m +CONFIG_P54_USB=m +CONFIG_LIBERTAS=m +CONFIG_LIBERTAS_USB=m +CONFIG_LIBERTAS_SDIO=m +CONFIG_LIBERTAS_MESH=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_MT76x0U=m +CONFIG_MT76x2U=m +CONFIG_RT2X00=m +CONFIG_RT2500USB=m +CONFIG_RT73USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RTL8187=m +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RSI_91X=m +# CONFIG_RSI_SDIO is not set +CONFIG_WL1251=m +CONFIG_WL1251_SPI=m +CONFIG_WL1251_SDIO=m +CONFIG_WL12XX=m +CONFIG_WL18XX=m +CONFIG_WLCORE_SPI=m +CONFIG_WLCORE_SDIO=m +CONFIG_ZD1211RW=m +CONFIG_MAC80211_HWSIM=m +CONFIG_USB_NET_RNDIS_WLAN=m +CONFIG_IEEE802154_AT86RF230=m +CONFIG_IEEE802154_MRF24J40=m +CONFIG_IEEE802154_CC2520=m +CONFIG_IEEE802154_ATUSB=m +CONFIG_IEEE802154_ADF7242=m +CONFIG_IEEE802154_CA8210=m +CONFIG_IEEE802154_MCR20A=m +CONFIG_IEEE802154_HWSIM=m +CONFIG_INPUT_SPARSEKMAP=m +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=m +CONFIG_KEYBOARD_ADP5588=m +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_QT2160=m +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_LM8323=m +CONFIG_KEYBOARD_MAX7359=m +CONFIG_KEYBOARD_STOWAWAY=m +# CONFIG_MOUSE_PS2 is not set +CONFIG_MOUSE_APPLETOUCH=m +CONFIG_MOUSE_ELAN_I2C=m +CONFIG_MOUSE_SYNAPTICS_I2C=m +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_IFORCE_232=m +CONFIG_JOYSTICK_WARRIOR=m +CONFIG_JOYSTICK_MAGELLAN=m +CONFIG_JOYSTICK_SPACEORB=m +CONFIG_JOYSTICK_SPACEBALL=m +CONFIG_JOYSTICK_STINGER=m +CONFIG_JOYSTICK_TWIDJOY=m +CONFIG_JOYSTICK_ZHENHUA=m +CONFIG_JOYSTICK_AS5011=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_JOYSTICK_PSXPAD_SPI=y +CONFIG_JOYSTICK_PSXPAD_SPI_FF=y +CONFIG_JOYSTICK_PXRC=m +CONFIG_JOYSTICK_FSIA6B=m +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_GTCO=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_TABLET_SERIAL_WACOM4=m +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_ADS7846=m +CONFIG_TOUCHSCREEN_AD7877=m +CONFIG_TOUCHSCREEN_AD7879=m +CONFIG_TOUCHSCREEN_AD7879_I2C=m +CONFIG_TOUCHSCREEN_AR1021_I2C=y +CONFIG_TOUCHSCREEN_ATMEL_MXT=m +CONFIG_TOUCHSCREEN_DYNAPRO=m +CONFIG_TOUCHSCREEN_HAMPSHIRE=m +CONFIG_TOUCHSCREEN_FUJITSU=m +CONFIG_TOUCHSCREEN_GOODIX=m +CONFIG_TOUCHSCREEN_GUNZE=m +CONFIG_TOUCHSCREEN_ELO=m +CONFIG_TOUCHSCREEN_WACOM_W8001=m +CONFIG_TOUCHSCREEN_MCS5000=m +CONFIG_TOUCHSCREEN_MTOUCH=m +CONFIG_TOUCHSCREEN_INEXIO=m +CONFIG_TOUCHSCREEN_MK712=m +CONFIG_TOUCHSCREEN_PENMOUNT=m +CONFIG_TOUCHSCREEN_EDT_FT5X06=y +CONFIG_TOUCHSCREEN_TOUCHRIGHT=m +CONFIG_TOUCHSCREEN_TOUCHWIN=m +CONFIG_TOUCHSCREEN_TI_AM335X_TSC=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_TSC2005=m +CONFIG_TOUCHSCREEN_TSC2007=m +CONFIG_TOUCHSCREEN_SILEAD=y +CONFIG_TOUCHSCREEN_STMPE=y +CONFIG_TOUCHSCREEN_TPS6507X=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_MMA8450=m +CONFIG_INPUT_ATI_REMOTE2=m +CONFIG_INPUT_KEYSPAN_REMOTE=m +CONFIG_INPUT_POWERMATE=m +CONFIG_INPUT_YEALINK=m +CONFIG_INPUT_CM109=m +CONFIG_INPUT_TPS65218_PWRBUTTON=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_PALMAS_PWRBUTTON=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_N_GSM=m +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_DMA is not set +CONFIG_SERIAL_8250_NR_UARTS=6 +CONFIG_SERIAL_8250_RUNTIME_UARTS=6 +CONFIG_SERIAL_8250_OMAP=y +CONFIG_SERIAL_8250_PRUSS=m +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_MAX3100=m +CONFIG_SERIAL_MAX310X=m +CONFIG_SERIAL_PRU_SUART=m +CONFIG_SERIAL_DEV_BUS=y +CONFIG_TTY_PRINTK=m +CONFIG_VIRTIO_CONSOLE=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=m +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS_I2C_ATMEL=y +CONFIG_RANDOM_TRUST_BOOTLOADER=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_ARB_GPIO_CHALLENGE=m +CONFIG_I2C_MUX_GPIO=y +CONFIG_I2C_MUX_PCA954x=m +CONFIG_I2C_MUX_PINCTRL=y +CONFIG_I2C_GPIO=y +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_ROBOTFUZZ_OSIF=m +CONFIG_I2C_TAOS_EVM=m +CONFIG_I2C_TINY_USB=m +CONFIG_I2C_SLAVE=y +CONFIG_SPI=y +CONFIG_SPI_GPIO=y +CONFIG_SPI_OMAP24XX=y +CONFIG_SPI_TI_QSPI=y +CONFIG_SPI_SPIDEV=m +CONFIG_SPI_SLAVE=y +CONFIG_SPI_SLAVE_TIME=m +CONFIG_SPI_SLAVE_SYSTEM_CONTROL=m +CONFIG_PPS_CLIENT_LDISC=m +CONFIG_PPS_CLIENT_GPIO=m +CONFIG_PINCTRL_MCP23S08=m +CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_PALMAS=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_OF_HELPER=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_SYSCON=y +CONFIG_GPIO_ADP5588=m +CONFIG_GPIO_ADNP=m +CONFIG_GPIO_MAX7300=m +CONFIG_GPIO_MAX732X=m +CONFIG_GPIO_PCA953X=y +CONFIG_GPIO_PCA953X_IRQ=y +CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_TPIC2810=m +CONFIG_GPIO_PALMAS=y +CONFIG_GPIO_STMPE=y +CONFIG_GPIO_TPS65218=y +CONFIG_GPIO_74X164=m +CONFIG_GPIO_MAX3191X=m +CONFIG_GPIO_MAX7301=m +CONFIG_GPIO_MC33880=m +CONFIG_GPIO_PISOSR=m +CONFIG_GPIO_XRA1403=m +CONFIG_W1=y +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_MASTER_DS2482=m +CONFIG_W1_MASTER_GPIO=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_DS28E17=m +CONFIG_POWER_AVS=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_GPIO=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_POWER_RESET_RESTART=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_POWER_RESET_SYSCON_POWEROFF=y +CONFIG_GENERIC_ADC_BATTERY=m +CONFIG_CHARGER_GPIO=m +CONFIG_SENSORS_AD7314=m +CONFIG_SENSORS_AD7414=m +CONFIG_SENSORS_AD7418=m +CONFIG_SENSORS_ADM1021=m +CONFIG_SENSORS_ADM1025=m +CONFIG_SENSORS_ADM1026=m +CONFIG_SENSORS_ADM1029=m +CONFIG_SENSORS_ADM1031=m +CONFIG_SENSORS_ADM9240=m +CONFIG_SENSORS_ADT7310=m +CONFIG_SENSORS_ADT7410=m +CONFIG_SENSORS_ADT7411=m +CONFIG_SENSORS_ADT7462=m +CONFIG_SENSORS_ADT7470=m +CONFIG_SENSORS_ADT7475=m +CONFIG_SENSORS_AS370=m +CONFIG_SENSORS_ASC7621=m +CONFIG_SENSORS_ASPEED=m +CONFIG_SENSORS_ATXP1=m +CONFIG_SENSORS_DS620=m +CONFIG_SENSORS_DS1621=m +CONFIG_SENSORS_F71805F=m +CONFIG_SENSORS_F71882FG=m +CONFIG_SENSORS_F75375S=m +CONFIG_SENSORS_FTSTEUTATES=m +CONFIG_SENSORS_GL518SM=m +CONFIG_SENSORS_GL520SM=m +CONFIG_SENSORS_G760A=m +CONFIG_SENSORS_G762=m +CONFIG_SENSORS_GPIO_FAN=y +CONFIG_SENSORS_HIH6130=m +CONFIG_SENSORS_IIO_HWMON=m +CONFIG_SENSORS_IT87=m +CONFIG_SENSORS_JC42=m +CONFIG_SENSORS_POWR1220=m +CONFIG_SENSORS_LINEAGE=m +CONFIG_SENSORS_LTC2945=m +CONFIG_SENSORS_LTC2990=m +CONFIG_SENSORS_LTC4151=m +CONFIG_SENSORS_LTC4215=m +CONFIG_SENSORS_LTC4222=m +CONFIG_SENSORS_LTC4245=m +CONFIG_SENSORS_LTC4260=m +CONFIG_SENSORS_LTC4261=m +CONFIG_SENSORS_MAX1111=m +CONFIG_SENSORS_MAX16065=m +CONFIG_SENSORS_MAX1619=m +CONFIG_SENSORS_MAX1668=m +CONFIG_SENSORS_MAX197=m +CONFIG_SENSORS_MAX31722=m +CONFIG_SENSORS_MAX6621=m +CONFIG_SENSORS_MAX6639=m +CONFIG_SENSORS_MAX6642=m +CONFIG_SENSORS_MAX6650=m +CONFIG_SENSORS_MAX6697=m +CONFIG_SENSORS_MAX31790=m +CONFIG_SENSORS_MCP3021=m +CONFIG_SENSORS_TC654=m +CONFIG_SENSORS_ADCXX=m +CONFIG_SENSORS_LM63=m +CONFIG_SENSORS_LM70=m +CONFIG_SENSORS_LM73=m +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM77=m +CONFIG_SENSORS_LM78=m +CONFIG_SENSORS_LM80=m +CONFIG_SENSORS_LM83=m +CONFIG_SENSORS_LM85=m +CONFIG_SENSORS_LM87=m +CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_LM92=m +CONFIG_SENSORS_LM93=m +CONFIG_SENSORS_LM95234=m +CONFIG_SENSORS_LM95241=m +CONFIG_SENSORS_LM95245=m +CONFIG_SENSORS_PC87360=m +CONFIG_SENSORS_PC87427=m +CONFIG_SENSORS_NTC_THERMISTOR=m +CONFIG_SENSORS_NCT6683=m +CONFIG_SENSORS_NCT6775=m +CONFIG_SENSORS_NCT7802=m +CONFIG_SENSORS_NCT7904=m +CONFIG_SENSORS_NPCM7XX=m +CONFIG_SENSORS_PCF8591=m +CONFIG_PMBUS=m +CONFIG_SENSORS_ADM1275=m +CONFIG_SENSORS_IBM_CFFPS=m +CONFIG_SENSORS_INSPUR_IPSPS=m +CONFIG_SENSORS_IR35221=m +CONFIG_SENSORS_IR38064=m +CONFIG_SENSORS_IRPS5401=m +CONFIG_SENSORS_ISL68137=m +CONFIG_SENSORS_LM25066=m +CONFIG_SENSORS_LTC2978=m +CONFIG_SENSORS_LTC2978_REGULATOR=y +CONFIG_SENSORS_LTC3815=m +CONFIG_SENSORS_MAX16064=m +CONFIG_SENSORS_MAX20751=m +CONFIG_SENSORS_MAX31785=m +CONFIG_SENSORS_MAX34440=m +CONFIG_SENSORS_MAX8688=m +CONFIG_SENSORS_PXE1610=m +CONFIG_SENSORS_TPS40422=m +CONFIG_SENSORS_TPS53679=m +CONFIG_SENSORS_UCD9000=m +CONFIG_SENSORS_UCD9200=m +CONFIG_SENSORS_ZL6100=m +CONFIG_SENSORS_PWM_FAN=m +CONFIG_SENSORS_SHT15=m +CONFIG_SENSORS_SHT21=m +CONFIG_SENSORS_SHT3x=m +CONFIG_SENSORS_SHTC1=m +CONFIG_SENSORS_DME1737=m +CONFIG_SENSORS_EMC1403=m +CONFIG_SENSORS_EMC2103=m +CONFIG_SENSORS_EMC6W201=m +CONFIG_SENSORS_SMSC47M1=m +CONFIG_SENSORS_SMSC47M192=m +CONFIG_SENSORS_SMSC47B397=m +CONFIG_SENSORS_SCH5627=m +CONFIG_SENSORS_SCH5636=m +CONFIG_SENSORS_STTS751=m +CONFIG_SENSORS_SMM665=m +CONFIG_SENSORS_ADC128D818=m +CONFIG_SENSORS_ADS7828=m +CONFIG_SENSORS_ADS7871=m +CONFIG_SENSORS_AMC6821=m +CONFIG_SENSORS_INA209=m +CONFIG_SENSORS_INA2XX=m +CONFIG_SENSORS_INA3221=m +CONFIG_SENSORS_TC74=m +CONFIG_SENSORS_THMC50=m +CONFIG_SENSORS_TMP102=y +CONFIG_SENSORS_TMP103=m +CONFIG_SENSORS_TMP108=m +CONFIG_SENSORS_TMP401=m +CONFIG_SENSORS_TMP421=m +CONFIG_SENSORS_VT1211=m +CONFIG_SENSORS_W83773G=m +CONFIG_SENSORS_W83781D=m +CONFIG_SENSORS_W83791D=m +CONFIG_SENSORS_W83792D=m +CONFIG_SENSORS_W83793=m +CONFIG_SENSORS_W83795=m +CONFIG_SENSORS_W83L785TS=m +CONFIG_SENSORS_W83L786NG=m +CONFIG_SENSORS_W83627HF=m +CONFIG_SENSORS_W83627EHF=m +CONFIG_THERMAL_STATISTICS=y +CONFIG_THERMAL_GOV_FAIR_SHARE=y +CONFIG_THERMAL_GOV_BANG_BANG=y +CONFIG_CPU_THERMAL=y +CONFIG_CLOCK_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y +CONFIG_TI_THERMAL=y +CONFIG_OMAP5_THERMAL=y +CONFIG_DRA752_THERMAL=y +CONFIG_GENERIC_ADC_THERMAL=m +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV=y +CONFIG_WATCHDOG_PRETIMEOUT_GOV_PANIC=m +CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP=y +CONFIG_SOFT_WATCHDOG=y +CONFIG_OMAP_WATCHDOG=y +CONFIG_MFD_STMPE=y +CONFIG_MFD_TI_AM335X_TSCADC=y +CONFIG_MFD_PALMAS=y +CONFIG_MFD_TPS65217=y +CONFIG_MFD_TPS65218=y +CONFIG_MFD_WL1273_CORE=m +CONFIG_REGULATOR_USERSPACE_CONSUMER=y +CONFIG_REGULATOR_GPIO=y +CONFIG_REGULATOR_PALMAS=y +CONFIG_REGULATOR_PBIAS=y +CONFIG_REGULATOR_PWM=y +CONFIG_REGULATOR_TI_ABB=y +CONFIG_REGULATOR_TPS65217=y +CONFIG_REGULATOR_TPS65218=y +CONFIG_RC_CORE=m +CONFIG_LIRC=y +CONFIG_RC_DECODERS=y +CONFIG_IR_NEC_DECODER=m +CONFIG_IR_RC5_DECODER=m +CONFIG_IR_RC6_DECODER=m +CONFIG_IR_JVC_DECODER=m +CONFIG_IR_SONY_DECODER=m +CONFIG_IR_SANYO_DECODER=m +CONFIG_IR_SHARP_DECODER=m +CONFIG_IR_MCE_KBD_DECODER=m +CONFIG_IR_XMP_DECODER=m +CONFIG_IR_IMON_DECODER=m +CONFIG_RC_DEVICES=y +CONFIG_RC_ATI_REMOTE=m +CONFIG_IR_IMON=m +CONFIG_IR_IMON_RAW=m +CONFIG_IR_MCEUSB=m +CONFIG_IR_REDRAT3=m +CONFIG_IR_STREAMZAP=m +CONFIG_IR_IGORPLUGUSB=m +CONFIG_IR_IGUANA=m +CONFIG_IR_TTUSBIR=m +CONFIG_RC_LOOPBACK=m +CONFIG_IR_GPIO_CIR=m +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y +CONFIG_MEDIA_RADIO_SUPPORT=y +CONFIG_MEDIA_SDR_SUPPORT=y +CONFIG_MEDIA_CEC_SUPPORT=y +CONFIG_VIDEO_V4L2_SUBDEV_API=y +CONFIG_DVB_DYNAMIC_MINORS=y +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_VIDEO_CLASS=m +CONFIG_USB_M5602=m +CONFIG_USB_STV06XX=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_PWC=m +CONFIG_VIDEO_CPIA2=m +CONFIG_USB_ZR364XX=m +CONFIG_USB_STKWEBCAM=m +CONFIG_USB_S2255=m +CONFIG_VIDEO_USBTV=m +CONFIG_VIDEO_PVRUSB2=m +CONFIG_VIDEO_HDPVR=m +CONFIG_VIDEO_USBVISION=m +CONFIG_VIDEO_STK1160_COMMON=m +CONFIG_VIDEO_GO7007=m +CONFIG_VIDEO_GO7007_USB=m +CONFIG_VIDEO_GO7007_USB_S2250_BOARD=m +CONFIG_VIDEO_AU0828=m +CONFIG_VIDEO_AU0828_RC=y +CONFIG_VIDEO_CX231XX=m +CONFIG_VIDEO_CX231XX_ALSA=m +CONFIG_VIDEO_CX231XX_DVB=m +CONFIG_DVB_USB=m +CONFIG_DVB_USB_A800=m +CONFIG_DVB_USB_DIBUSB_MB=m +CONFIG_DVB_USB_DIBUSB_MB_FAULTY=y +CONFIG_DVB_USB_DIBUSB_MC=m +CONFIG_DVB_USB_DIB0700=m +CONFIG_DVB_USB_UMT_010=m +CONFIG_DVB_USB_CXUSB=m +CONFIG_DVB_USB_M920X=m +CONFIG_DVB_USB_DIGITV=m +CONFIG_DVB_USB_VP7045=m +CONFIG_DVB_USB_VP702X=m +CONFIG_DVB_USB_GP8PSK=m +CONFIG_DVB_USB_NOVA_T_USB2=m +CONFIG_DVB_USB_TTUSB2=m +CONFIG_DVB_USB_DTT200U=m +CONFIG_DVB_USB_OPERA1=m +CONFIG_DVB_USB_AF9005=m +CONFIG_DVB_USB_AF9005_REMOTE=m +CONFIG_DVB_USB_PCTV452E=m +CONFIG_DVB_USB_DW2102=m +CONFIG_DVB_USB_CINERGY_T2=m +CONFIG_DVB_USB_DTV5100=m +CONFIG_DVB_USB_AZ6027=m +CONFIG_DVB_USB_TECHNISAT_USB2=m +CONFIG_DVB_USB_V2=m +CONFIG_DVB_USB_AF9015=m +CONFIG_DVB_USB_AF9035=m +CONFIG_DVB_USB_ANYSEE=m +CONFIG_DVB_USB_AU6610=m +CONFIG_DVB_USB_AZ6007=m +CONFIG_DVB_USB_CE6230=m +CONFIG_DVB_USB_EC168=m +CONFIG_DVB_USB_GL861=m +CONFIG_DVB_USB_LME2510=m +CONFIG_DVB_USB_MXL111SF=m +CONFIG_DVB_USB_RTL28XXU=m +CONFIG_DVB_USB_DVBSKY=m +CONFIG_DVB_USB_ZD1301=m +CONFIG_SMS_USB_DRV=m +CONFIG_DVB_B2C2_FLEXCOP_USB=m +CONFIG_DVB_AS102=m +CONFIG_VIDEO_EM28XX=m +CONFIG_VIDEO_EM28XX_V4L2=m +CONFIG_VIDEO_EM28XX_ALSA=m +CONFIG_VIDEO_EM28XX_DVB=m +CONFIG_USB_AIRSPY=m +CONFIG_USB_HACKRF=m +CONFIG_USB_MSI2500=m +CONFIG_USB_PULSE8_CEC=m +CONFIG_USB_RAINSHADOW_CEC=m +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_V4L_MEM2MEM_DRIVERS=y +CONFIG_SMS_SDIO_DRV=m +CONFIG_RADIO_SI470X=y +CONFIG_USB_SI470X=m +CONFIG_RADIO_SI4713=m +CONFIG_I2C_SI4713=m +CONFIG_USB_MR800=m +CONFIG_RADIO_SHARK=m +CONFIG_RADIO_SHARK2=m +CONFIG_USB_KEENE=m +CONFIG_USB_RAREMONO=m +CONFIG_USB_MA901=m +CONFIG_RADIO_WL128X=m +CONFIG_MEDIA_SUBDRV_AUTOSELECT=y +CONFIG_VIDEO_TVAUDIO=m +CONFIG_VIDEO_TDA7432=m +CONFIG_VIDEO_TDA9840=m +CONFIG_VIDEO_TEA6415C=m +CONFIG_VIDEO_TEA6420=m +CONFIG_VIDEO_CS3308=m +CONFIG_VIDEO_CS5345=m +CONFIG_VIDEO_TLV320AIC23B=m +CONFIG_VIDEO_WM8739=m +CONFIG_VIDEO_VP27SMPX=m +CONFIG_VIDEO_SAA6588=m +CONFIG_VIDEO_BT819=m +CONFIG_VIDEO_BT856=m +CONFIG_VIDEO_KS0127=m +CONFIG_VIDEO_SAA7110=m +CONFIG_VIDEO_VPX3220=m +CONFIG_VIDEO_SAA717X=m +CONFIG_VIDEO_SAA7127=m +CONFIG_VIDEO_SAA7185=m +CONFIG_VIDEO_ADV7170=m +CONFIG_VIDEO_ADV7175=m +CONFIG_VIDEO_OV7670=m +CONFIG_VIDEO_SR030PC30=m +CONFIG_VIDEO_NOON010PC30=m +CONFIG_VIDEO_UPD64031A=m +CONFIG_VIDEO_UPD64083=m +CONFIG_VIDEO_SAA6752HS=m +CONFIG_VIDEO_M52790=m +CONFIG_MEDIA_TUNER_MT2131=m +CONFIG_MEDIA_TUNER_M88RS6000T=m +CONFIG_MEDIA_TUNER_MXL301RF=m +CONFIG_MEDIA_TUNER_QM1D1B0004=m +CONFIG_DVB_STV0910=m +CONFIG_DVB_STV6111=m +CONFIG_DVB_MXL5XX=m +CONFIG_DVB_CX24110=m +CONFIG_DVB_ZL10036=m +CONFIG_DVB_TDA8083=m +CONFIG_DVB_TDA8261=m +CONFIG_DVB_VES1X93=m +CONFIG_DVB_TUA6100=m +CONFIG_DVB_CX24117=m +CONFIG_DVB_MB86A16=m +CONFIG_DVB_SP8870=m +CONFIG_DVB_SP887X=m +CONFIG_DVB_CX22700=m +CONFIG_DVB_L64781=m +CONFIG_DVB_STV0367=m +CONFIG_DVB_VES1820=m +CONFIG_DVB_TDA10021=m +CONFIG_DVB_OR51211=m +CONFIG_DVB_OR51132=m +CONFIG_DVB_LNBH25=m +CONFIG_DVB_ISL6405=m +CONFIG_DVB_TDA665x=m +CONFIG_DVB_HORUS3A=m +CONFIG_DVB_ASCOT2E=m +CONFIG_DVB_HELENE=m +CONFIG_DVB_CXD2099=m +CONFIG_DVB_DUMMY_FE=m +CONFIG_DRM=y +CONFIG_DRM_DP_AUX_CHARDEV=y +CONFIG_DRM_LOAD_EDID_FIRMWARE=y +CONFIG_DRM_I2C_NXP_TDA998X=y +CONFIG_DRM_VGEM=m +CONFIG_DRM_UDL=m +CONFIG_DRM_OMAP=y +CONFIG_OMAP5_DSS_HDMI=y +CONFIG_DRM_OMAP_ENCODER_TPD12S015=y +CONFIG_DRM_OMAP_CONNECTOR_HDMI=y +CONFIG_DRM_TILCDC=y +CONFIG_DRM_PANEL_SIMPLE=y +CONFIG_DRM_PANEL_SONY_ACX565AKM=m +CONFIG_DRM_PANEL_TPO_TD028TTEC1=m +CONFIG_DRM_PANEL_TPO_TD043MTEA1=m +CONFIG_DRM_SII902X=y +CONFIG_DRM_TI_TFP410=y +CONFIG_DRM_I2C_ADV7511=y +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_ETNAVIV=y +CONFIG_TINYDRM_HX8357D=m +CONFIG_TINYDRM_ILI9225=m +CONFIG_TINYDRM_ILI9341=m +CONFIG_TINYDRM_MI0283QT=m +CONFIG_TINYDRM_REPAPER=m +CONFIG_TINYDRM_ST7586=m +CONFIG_TINYDRM_ST7735R=m +CONFIG_DRM_LEGACY=y +CONFIG_FIRMWARE_EDID=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m +CONFIG_FB_SIMPLE=y +CONFIG_FB_SSD1307=y +# CONFIG_BACKLIGHT_GENERIC is not set +CONFIG_BACKLIGHT_PWM=y +CONFIG_BACKLIGHT_GPIO=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_SOUND=y +# CONFIG_SOUND_OSS_CORE_PRECLAIM is not set +CONFIG_SND=y +CONFIG_SND_OSSEMUL=y +CONFIG_SND_MIXER_OSS=m +CONFIG_SND_PCM_OSS=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_DYNAMIC_MINORS=y +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_DUMMY=m +CONFIG_SND_ALOOP=m +CONFIG_SND_VIRMIDI=m +CONFIG_SND_MTPAV=m +CONFIG_SND_SERIAL_U16550=m +CONFIG_SND_MPU401=m +CONFIG_SND_HDA_PREALLOC_SIZE=2048 +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_BCD2000=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_SOC=y +CONFIG_SND_SOC_DAVINCI_MCASP=y +CONFIG_SND_SOC_OMAP_DMIC=y +CONFIG_SND_SOC_OMAP_MCBSP=y +CONFIG_SND_SOC_OMAP_MCPDM=y +CONFIG_SND_SOC_OMAP_HDMI=y +CONFIG_SND_SOC_ADAU1701=m +CONFIG_SND_SOC_ADAU7002=m +CONFIG_SND_SOC_AK4554=m +CONFIG_SND_SOC_CS4265=m +CONFIG_SND_SOC_CS4271_I2C=m +CONFIG_SND_SOC_PCM512x_I2C=m +CONFIG_SND_SOC_SPDIF=m +CONFIG_SND_SOC_TLV320AIC23_I2C=m +CONFIG_SND_SOC_TLV320AIC31XX=m +CONFIG_SND_SOC_TLV320AIC3X=m +CONFIG_SND_SOC_TS3A227E=m +CONFIG_SND_SOC_WM8753=m +CONFIG_SND_SOC_WM8804_I2C=m +CONFIG_SND_SOC_WM8903=m +CONFIG_SND_SOC_TPA6130A2=m +CONFIG_SND_SIMPLE_CARD=y +CONFIG_SND_AUDIO_GRAPH_CARD=y +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=m +CONFIG_HID_ASUS=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_MACALLY=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CP2112=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EZKEY=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_HOLTEK=m +CONFIG_HOLTEK_FF=y +CONFIG_HID_GT683R=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_WALTOP=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_GYRATION=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_REDRAGON=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PICOLCD_CIR=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_RETRODE=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SONY=m +CONFIG_SONY_FF=y +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_RMI=m +CONFIG_HID_GREENASIA=m +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_WACOM=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_ZEROPLUS_FF=y +CONFIG_HID_ZYDACRON=m +CONFIG_HID_SENSOR_HUB=m +CONFIG_HID_SENSOR_CUSTOM_SENSOR=m +CONFIG_HID_ALPS=m +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_I2C_HID=y +CONFIG_USB_LED_TRIG=y +CONFIG_USB_ULPI_BUS=m +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_DYNAMIC_MINORS=y +CONFIG_USB_OTG=y +CONFIG_USB_LEDS_TRIGGER_USBPORT=m +CONFIG_USB_MON=m +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_PRINTER=m +CONFIG_USB_TMC=m +CONFIG_USB_STORAGE=y +CONFIG_USB_STORAGE_REALTEK=y +CONFIG_USB_STORAGE_DATAFAB=y +CONFIG_USB_STORAGE_FREECOM=y +CONFIG_USB_STORAGE_ISD200=y +CONFIG_USB_STORAGE_USBAT=y +CONFIG_USB_STORAGE_SDDR09=y +CONFIG_USB_STORAGE_SDDR55=y +CONFIG_USB_STORAGE_JUMPSHOT=y +CONFIG_USB_STORAGE_ALAUDA=y +CONFIG_USB_STORAGE_ONETOUCH=y +CONFIG_USB_STORAGE_KARMA=y +CONFIG_USB_STORAGE_CYPRESS_ATACB=y +CONFIG_USB_STORAGE_ENE_UB6250=y +CONFIG_USB_UAS=y +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBIP_VHCI_HC_PORTS=15 +CONFIG_USBIP_VHCI_NR_HCS=8 +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VUDC=m +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_DSPS=y +CONFIG_MUSB_PIO_ONLY=y +CONFIG_USB_DWC3=y +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_METRO=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_MXUPORT=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_XIRCOM=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SERIAL_WISHBONE=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_DEBUG=m +CONFIG_USB_EMI62=m +CONFIG_USB_EMI26=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_LCD=m +CONFIG_USB_CYPRESS_CY7C63=m +CONFIG_USB_CYTHERM=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_FTDI_ELAN=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_SISUSBVGA_CON=y +CONFIG_USB_LD=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_TEST=m +CONFIG_USB_EHSET_TEST_FIXTURE=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_YUREX=m +CONFIG_USB_HSIC_USB3503=m +CONFIG_USB_CHAOSKEY=m +CONFIG_NOP_USB_XCEIV=y +CONFIG_AM335X_PHY_USB=y +CONFIG_USB_GPIO_VBUS=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_VBUS_DRAW=500 +CONFIG_USB_DUMMY_HCD=m +CONFIG_USB_CONFIGFS=m +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_OBEX=y +CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y +CONFIG_USB_CONFIGFS_ECM_SUBSET=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y +CONFIG_USB_CONFIGFS_PHONET=y +CONFIG_USB_CONFIGFS_MASS_STORAGE=y +CONFIG_USB_CONFIGFS_F_LB_SS=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_UAC1=y +CONFIG_USB_CONFIGFS_F_UAC2=y +CONFIG_USB_CONFIGFS_F_MIDI=y +CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y +CONFIG_USB_CONFIGFS_F_PRINTER=y +CONFIG_USB_ZERO=m +CONFIG_USB_AUDIO=m +CONFIG_USB_ETH=m +CONFIG_USB_G_NCM=m +CONFIG_USB_GADGETFS=m +CONFIG_USB_FUNCTIONFS=m +CONFIG_USB_FUNCTIONFS_ETH=y +CONFIG_USB_FUNCTIONFS_RNDIS=y +CONFIG_USB_FUNCTIONFS_GENERIC=y +CONFIG_USB_MASS_STORAGE=m +CONFIG_USB_G_SERIAL=m +CONFIG_USB_MIDI_GADGET=m +CONFIG_USB_G_PRINTER=m +CONFIG_USB_CDC_COMPOSITE=m +CONFIG_USB_G_NOKIA=m +CONFIG_USB_G_ACM_MS=m +CONFIG_USB_G_MULTI=m +CONFIG_USB_G_HID=m +CONFIG_USB_G_DBGP=m +CONFIG_TYPEC=y +CONFIG_TYPEC_HD3SS3220=y +CONFIG_USB_ROLE_SWITCH=y +CONFIG_MMC=y +CONFIG_PWRSEQ_SD8787=m +CONFIG_MMC_BLOCK_MINORS=256 +CONFIG_SDIO_UART=m +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_OMAP_HS=y +CONFIG_MMC_SPI=y +CONFIG_MMC_VUB300=m +CONFIG_MMC_USHC=m +CONFIG_MMC_SDHCI_OMAP=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_BRIGHTNESS_HW_CHANGED=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_LP3944=m +CONFIG_LEDS_LP5523=m +CONFIG_LEDS_PCA955X=m +CONFIG_LEDS_DAC124S085=m +CONFIG_LEDS_PWM=m +CONFIG_LEDS_REGULATOR=m +CONFIG_LEDS_BD2802=m +CONFIG_LEDS_LT3593=m +CONFIG_LEDS_TCA6507=m +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_DISK=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_ACTIVITY=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_TRANSIENT=m +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_LEDS_TRIGGER_NETDEV=y +CONFIG_ACCESSIBILITY=y +CONFIG_A11Y_BRAILLE_CONSOLE=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_ABB5ZES3=y +CONFIG_RTC_DRV_ABEOZ9=y +CONFIG_RTC_DRV_ABX80X=y +CONFIG_RTC_DRV_DS1307=y +CONFIG_RTC_DRV_DS1374=y +CONFIG_RTC_DRV_DS1374_WDT=y +CONFIG_RTC_DRV_DS1672=y +CONFIG_RTC_DRV_HYM8563=y +CONFIG_RTC_DRV_MAX6900=y +CONFIG_RTC_DRV_RS5C372=y +CONFIG_RTC_DRV_ISL1208=y +CONFIG_RTC_DRV_ISL12022=y +CONFIG_RTC_DRV_ISL12026=y +CONFIG_RTC_DRV_X1205=y +CONFIG_RTC_DRV_PCF8523=y +CONFIG_RTC_DRV_PCF85063=y +CONFIG_RTC_DRV_PCF85363=y +CONFIG_RTC_DRV_PCF8563=y +CONFIG_RTC_DRV_PCF8583=y +CONFIG_RTC_DRV_M41T80=y +CONFIG_RTC_DRV_M41T80_WDT=y +CONFIG_RTC_DRV_BQ32K=y +CONFIG_RTC_DRV_PALMAS=y +CONFIG_RTC_DRV_S35390A=y +CONFIG_RTC_DRV_FM3130=y +CONFIG_RTC_DRV_RX8010=y +CONFIG_RTC_DRV_RX8581=y +CONFIG_RTC_DRV_RX8025=y +CONFIG_RTC_DRV_EM3027=y +CONFIG_RTC_DRV_RV8803=y +CONFIG_RTC_DRV_M41T93=y +CONFIG_RTC_DRV_M41T94=y +CONFIG_RTC_DRV_DS1302=y +CONFIG_RTC_DRV_DS1305=y +CONFIG_RTC_DRV_DS1343=y +CONFIG_RTC_DRV_DS1347=y +CONFIG_RTC_DRV_DS1390=y +CONFIG_RTC_DRV_MAX6916=y +CONFIG_RTC_DRV_R9701=y +CONFIG_RTC_DRV_RX4581=y +CONFIG_RTC_DRV_RX6110=y +CONFIG_RTC_DRV_RS5C348=y +CONFIG_RTC_DRV_MAX6902=y +CONFIG_RTC_DRV_PCF2123=y +CONFIG_RTC_DRV_MCP795=y +CONFIG_RTC_DRV_DS3232=y +CONFIG_RTC_DRV_PCF2127=y +CONFIG_RTC_DRV_RV3029C2=y +CONFIG_RTC_DRV_DS1286=m +CONFIG_RTC_DRV_DS1511=m +CONFIG_RTC_DRV_DS1553=m +CONFIG_RTC_DRV_DS1685_FAMILY=m +CONFIG_RTC_DRV_DS1742=m +CONFIG_RTC_DRV_DS2404=m +CONFIG_RTC_DRV_STK17TA8=m +CONFIG_RTC_DRV_M48T86=m +CONFIG_RTC_DRV_M48T35=m +CONFIG_RTC_DRV_M48T59=m +CONFIG_RTC_DRV_MSM6242=m +CONFIG_RTC_DRV_BQ4802=m +CONFIG_RTC_DRV_RP5C01=m +CONFIG_RTC_DRV_V3020=m +CONFIG_RTC_DRV_OMAP=y +CONFIG_RTC_DRV_HID_SENSOR_TIME=m +CONFIG_DMADEVICES=y +CONFIG_TI_CPPI41=y +CONFIG_ASYNC_TX_DMA=y +CONFIG_AUXDISPLAY=y +CONFIG_HD44780=m +CONFIG_IMG_ASCII_LCD=m +CONFIG_HT16K33=m +CONFIG_UIO=m +CONFIG_VIRT_DRIVERS=y +CONFIG_VIRTIO_BALLOON=m +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=m +CONFIG_GREYBUS=m +CONFIG_GREYBUS_ES2=m +CONFIG_STAGING=y +CONFIG_RTLLIB=m +CONFIG_RTL8723BS=m +CONFIG_R8712U=m +CONFIG_R8188EU=m +CONFIG_ADIS16203=m +CONFIG_ADIS16240=m +CONFIG_AD7816=m +CONFIG_AD7192=m +CONFIG_AD7280=m +CONFIG_ADT7316=m +CONFIG_ADT7316_I2C=m +CONFIG_AD7150=m +CONFIG_AD7746=m +CONFIG_AD9832=m +CONFIG_AD9834=m +CONFIG_AD5933=m +CONFIG_ADE7854=m +CONFIG_AD2S1210=m +CONFIG_FB_TFT=m +CONFIG_FB_TFT_AGM1264K_FL=m +CONFIG_FB_TFT_BD663474=m +CONFIG_FB_TFT_HX8340BN=m +CONFIG_FB_TFT_HX8347D=m +CONFIG_FB_TFT_HX8353D=m +CONFIG_FB_TFT_HX8357D=m +CONFIG_FB_TFT_ILI9163=m +CONFIG_FB_TFT_ILI9320=m +CONFIG_FB_TFT_ILI9325=m +CONFIG_FB_TFT_ILI9340=m +CONFIG_FB_TFT_ILI9341=m +CONFIG_FB_TFT_ILI9481=m +CONFIG_FB_TFT_ILI9486=m +CONFIG_FB_TFT_PCD8544=m +CONFIG_FB_TFT_RA8875=m +CONFIG_FB_TFT_S6D02A1=m +CONFIG_FB_TFT_S6D1121=m +CONFIG_FB_TFT_SH1106=m +CONFIG_FB_TFT_SSD1289=m +CONFIG_FB_TFT_SSD1305=m +CONFIG_FB_TFT_SSD1306=m +CONFIG_FB_TFT_SSD1331=m +CONFIG_FB_TFT_SSD1351=m +CONFIG_FB_TFT_ST7735R=m +CONFIG_FB_TFT_ST7789V=m +CONFIG_FB_TFT_TINYLCD=m +CONFIG_FB_TFT_TLS8204=m +CONFIG_FB_TFT_UC1611=m +CONFIG_FB_TFT_UC1701=m +CONFIG_FB_TFT_UPD161704=m +CONFIG_FB_TFT_WATTEROTT=m +CONFIG_GREYBUS_AUDIO=m +CONFIG_GREYBUS_BOOTROM=m +CONFIG_GREYBUS_FIRMWARE=m +CONFIG_GREYBUS_HID=m +CONFIG_GREYBUS_LIGHT=m +CONFIG_GREYBUS_LOG=m +CONFIG_GREYBUS_LOOPBACK=m +CONFIG_GREYBUS_POWER=m +CONFIG_GREYBUS_RAW=m +CONFIG_GREYBUS_VIBRATOR=m +CONFIG_GREYBUS_BRIDGED_PHY=m +CONFIG_GREYBUS_GPIO=m +CONFIG_GREYBUS_I2C=m +CONFIG_GREYBUS_PWM=m +CONFIG_GREYBUS_SDIO=m +CONFIG_GREYBUS_SPI=m +CONFIG_GREYBUS_UART=m +CONFIG_GREYBUS_USB=m +CONFIG_USB_WUSB_CBAF=m +CONFIG_STAGING_EXFAT_FS=m +CONFIG_COMMON_CLK_PALMAS=y +CONFIG_COMMON_CLK_TI_ADPLL=y +CONFIG_HWSPINLOCK=y +CONFIG_HWSPINLOCK_OMAP=y +CONFIG_OMAP_IOMMU=y +CONFIG_REMOTEPROC=y +CONFIG_OMAP_REMOTEPROC=y +CONFIG_OMAP_REMOTEPROC_WATCHDOG=y +CONFIG_WKUP_M3_RPROC=y +CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_VIRTIO=y +CONFIG_RPMSG_PRU=y +CONFIG_SOC_TI=y +CONFIG_AMX3_PM=y +CONFIG_WKUP_M3_IPC=y +CONFIG_TI_PRUSS=y +CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y +CONFIG_DEVFREQ_GOV_PERFORMANCE=y +CONFIG_DEVFREQ_GOV_POWERSAVE=y +CONFIG_DEVFREQ_GOV_USERSPACE=y +CONFIG_DEVFREQ_GOV_PASSIVE=y +CONFIG_PM_DEVFREQ_EVENT=y +CONFIG_EXTCON_GPIO=y +CONFIG_EXTCON_PALMAS=y +CONFIG_EXTCON_USB_GPIO=y +CONFIG_TI_EMIF=y +CONFIG_TI_EMIF_SRAM=y +CONFIG_IIO=y +CONFIG_IIO_SW_DEVICE=m +CONFIG_IIO_SW_TRIGGER=m +CONFIG_ADIS16201=m +CONFIG_ADIS16209=m +CONFIG_ADXL345_I2C=m +CONFIG_ADXL345_SPI=m +CONFIG_ADXL372_SPI=m +CONFIG_ADXL372_I2C=m +CONFIG_BMA180=m +CONFIG_BMA220=m +CONFIG_BMC150_ACCEL=m +CONFIG_DA280=m +CONFIG_DA311=m +CONFIG_DMARD06=m +CONFIG_DMARD09=m +CONFIG_DMARD10=m +CONFIG_HID_SENSOR_ACCEL_3D=m +CONFIG_IIO_ST_ACCEL_3AXIS=m +CONFIG_KXSD9=m +CONFIG_KXCJK1013=m +CONFIG_MC3230=m +CONFIG_MMA7455_I2C=m +CONFIG_MMA7455_SPI=m +CONFIG_MMA7660=m +CONFIG_MMA8452=m +CONFIG_MMA9551=m +CONFIG_MMA9553=m +CONFIG_MXC4005=m +CONFIG_MXC6255=m +CONFIG_SCA3000=m +CONFIG_STK8312=m +CONFIG_STK8BA50=m +CONFIG_AD7124=m +CONFIG_AD7266=m +CONFIG_AD7291=m +CONFIG_AD7298=m +CONFIG_AD7476=m +CONFIG_AD7606_IFACE_SPI=m +CONFIG_AD7766=m +CONFIG_AD7768_1=m +CONFIG_AD7780=m +CONFIG_AD7791=m +CONFIG_AD7793=m +CONFIG_AD7887=m +CONFIG_AD7923=m +CONFIG_AD7949=m +CONFIG_AD799X=m +CONFIG_CC10001_ADC=m +CONFIG_ENVELOPE_DETECTOR=m +CONFIG_HI8435=m +CONFIG_HX711=m +CONFIG_INA2XX_ADC=m +CONFIG_LTC2471=m +CONFIG_LTC2485=m +CONFIG_LTC2497=m +CONFIG_MAX1027=m +CONFIG_MAX11100=m +CONFIG_MAX1118=m +CONFIG_MAX1363=m +CONFIG_MAX9611=m +CONFIG_MCP320X=m +CONFIG_MCP3422=m +CONFIG_MCP3911=m +CONFIG_NAU7802=m +CONFIG_PALMAS_GPADC=m +CONFIG_SD_ADC_MODULATOR=m +CONFIG_STMPE_ADC=m +CONFIG_TI_ADC081C=m +CONFIG_TI_ADC0832=m +CONFIG_TI_ADC084S021=m +CONFIG_TI_ADC12138=m +CONFIG_TI_ADC108S102=m +CONFIG_TI_ADC128S052=m +CONFIG_TI_ADC161S626=m +CONFIG_TI_ADS1015=m +CONFIG_TI_ADS7950=m +CONFIG_TI_ADS8344=m +CONFIG_TI_ADS8688=m +CONFIG_TI_ADS124S08=m +CONFIG_TI_AM335X_ADC=y +CONFIG_TI_TLC4541=m +CONFIG_IIO_RESCALE=m +CONFIG_AD8366=m +CONFIG_ATLAS_PH_SENSOR=m +CONFIG_BME680=m +CONFIG_CCS811=m +CONFIG_IAQCORE=m +CONFIG_PMS7003=m +CONFIG_SENSIRION_SGP30=m +CONFIG_SPS30=m +CONFIG_VZ89X=m +CONFIG_AD5064=m +CONFIG_AD5360=m +CONFIG_AD5380=m +CONFIG_AD5421=m +CONFIG_AD5446=m +CONFIG_AD5449=m +CONFIG_AD5592R=m +CONFIG_AD5593R=m +CONFIG_AD5504=m +CONFIG_AD5624R_SPI=m +CONFIG_LTC1660=m +CONFIG_LTC2632=m +CONFIG_AD5686_SPI=m +CONFIG_AD5696_I2C=m +CONFIG_AD5755=m +CONFIG_AD5758=m +CONFIG_AD5761=m +CONFIG_AD5764=m +CONFIG_AD5791=m +CONFIG_AD7303=m +CONFIG_AD8801=m +CONFIG_DPOT_DAC=m +CONFIG_DS4424=m +CONFIG_M62332=m +CONFIG_MAX517=m +CONFIG_MAX5821=m +CONFIG_MCP4725=m +CONFIG_MCP4922=m +CONFIG_TI_DAC082S085=m +CONFIG_TI_DAC5571=m +CONFIG_TI_DAC7311=m +CONFIG_TI_DAC7612=m +CONFIG_AD9523=m +CONFIG_ADF4350=m +CONFIG_ADF4371=m +CONFIG_ADIS16080=m +CONFIG_ADIS16130=m +CONFIG_ADIS16136=m +CONFIG_ADIS16260=m +CONFIG_ADXRS450=m +CONFIG_BMG160=m +CONFIG_FXAS21002C=m +CONFIG_HID_SENSOR_GYRO_3D=m +CONFIG_MPU3050_I2C=m +CONFIG_IIO_ST_GYRO_3AXIS=m +CONFIG_ITG3200=m +CONFIG_AFE4403=m +CONFIG_AFE4404=m +CONFIG_MAX30100=m +CONFIG_MAX30102=m +CONFIG_AM2315=m +CONFIG_DHT11=m +CONFIG_HDC100X=m +CONFIG_HID_SENSOR_HUMIDITY=m +CONFIG_HTS221=m +CONFIG_HTU21=m +CONFIG_SI7005=m +CONFIG_SI7020=m +CONFIG_ADIS16400=m +CONFIG_ADIS16460=m +CONFIG_ADIS16480=m +CONFIG_BMI160_I2C=m +CONFIG_BMI160_SPI=m +CONFIG_KMX61=m +CONFIG_INV_MPU6050_I2C=m +CONFIG_INV_MPU6050_SPI=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_ADJD_S311=m +CONFIG_AL3320A=m +CONFIG_APDS9300=m +CONFIG_APDS9960=m +CONFIG_BH1750=m +CONFIG_BH1780=m +CONFIG_CM32181=m +CONFIG_CM3232=m +CONFIG_CM3323=m +CONFIG_CM3605=m +CONFIG_CM36651=m +CONFIG_GP2AP020A00F=m +CONFIG_SENSORS_ISL29018=m +CONFIG_SENSORS_ISL29028=m +CONFIG_ISL29125=m +CONFIG_HID_SENSOR_ALS=m +CONFIG_HID_SENSOR_PROX=m +CONFIG_JSA1212=m +CONFIG_RPR0521=m +CONFIG_LTR501=m +CONFIG_LV0104CS=m +CONFIG_MAX44000=m +CONFIG_MAX44009=m +CONFIG_NOA1305=m +CONFIG_OPT3001=m +CONFIG_PA12203001=m +CONFIG_SI1133=m +CONFIG_SI1145=m +CONFIG_STK3310=m +CONFIG_ST_UVIS25=m +CONFIG_TCS3414=m +CONFIG_TCS3472=m +CONFIG_SENSORS_TSL2563=m +CONFIG_TSL2583=m +CONFIG_TSL2772=m +CONFIG_TSL4531=m +CONFIG_US5182D=m +CONFIG_VCNL4000=m +CONFIG_VCNL4035=m +CONFIG_VEML6070=m +CONFIG_VL6180=m +CONFIG_ZOPT2201=m +CONFIG_AK8974=m +CONFIG_AK09911=m +CONFIG_BMC150_MAGN_I2C=m +CONFIG_BMC150_MAGN_SPI=m +CONFIG_MAG3110=m +CONFIG_HID_SENSOR_MAGNETOMETER_3D=m +CONFIG_MMC35240=m +CONFIG_IIO_ST_MAGN_3AXIS=m +CONFIG_SENSORS_HMC5843_I2C=m +CONFIG_SENSORS_HMC5843_SPI=m +CONFIG_SENSORS_RM3100_I2C=m +CONFIG_SENSORS_RM3100_SPI=m +CONFIG_HID_SENSOR_INCLINOMETER_3D=m +CONFIG_HID_SENSOR_DEVICE_ROTATION=m +CONFIG_IIO_HRTIMER_TRIGGER=m +CONFIG_IIO_INTERRUPT_TRIGGER=m +CONFIG_IIO_TIGHTLOOP_TRIGGER=m +CONFIG_IIO_SYSFS_TRIGGER=m +CONFIG_AD5272=m +CONFIG_DS1803=m +CONFIG_MAX5432=m +CONFIG_MAX5481=m +CONFIG_MAX5487=m +CONFIG_MCP4018=m +CONFIG_MCP4131=m +CONFIG_MCP4531=m +CONFIG_MCP41010=m +CONFIG_TPL0102=m +CONFIG_LMP91000=m +CONFIG_ABP060MG=m +CONFIG_BMP280=m +CONFIG_DPS310=m +CONFIG_HID_SENSOR_PRESS=m +CONFIG_HP03=m +CONFIG_MPL115_I2C=m +CONFIG_MPL115_SPI=m +CONFIG_MPL3115=m +CONFIG_MS5611=m +CONFIG_MS5611_I2C=m +CONFIG_MS5611_SPI=m +CONFIG_MS5637=m +CONFIG_IIO_ST_PRESS=m +CONFIG_T5403=m +CONFIG_HP206C=m +CONFIG_ZPA2326=m +CONFIG_AS3935=m +CONFIG_ISL29501=m +CONFIG_LIDAR_LITE_V2=m +CONFIG_MB1232=m +CONFIG_RFD77402=m +CONFIG_SRF04=m +CONFIG_SX9500=m +CONFIG_SRF08=m +CONFIG_VL53L0X_I2C=m +CONFIG_AD2S90=m +CONFIG_AD2S1200=m +CONFIG_MAXIM_THERMOCOUPLE=m +CONFIG_HID_SENSOR_TEMP=m +CONFIG_MLX90614=m +CONFIG_MLX90632=m +CONFIG_TMP006=m +CONFIG_TMP007=m +CONFIG_TSYS01=m +CONFIG_TSYS02D=m +CONFIG_MAX31856=m +CONFIG_PWM_OMAP_DMTIMER=y +CONFIG_PWM_PCA9685=y +CONFIG_PWM_STMPE=y +CONFIG_PWM_TIECAP=y +CONFIG_PWM_TIEHRPWM=y +CONFIG_TI_PRUSS_INTC=y +CONFIG_RESET_TI_SYSCON=y +CONFIG_OMAP_USB2=y +CONFIG_TI_PIPE3=y +CONFIG_RAS=y +CONFIG_FPGA=m +CONFIG_ALTERA_PR_IP_CORE=m +CONFIG_ALTERA_PR_IP_CORE_PLAT=m +CONFIG_FPGA_MGR_ALTERA_PS_SPI=m +CONFIG_FPGA_MGR_XILINX_SPI=m +CONFIG_FPGA_MGR_ICE40_SPI=m +CONFIG_FPGA_MGR_MACHXO2_SPI=m +CONFIG_FPGA_BRIDGE=m +CONFIG_ALTERA_FREEZE_BRIDGE=m +CONFIG_XILINX_PR_DECOUPLER=m +CONFIG_FPGA_REGION=m +CONFIG_OF_FPGA_REGION=m +CONFIG_COUNTER=y +CONFIG_VALIDATE_FS_PARSER=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_JFS_FS=m +CONFIG_JFS_POSIX_ACL=y +CONFIG_JFS_SECURITY=y +CONFIG_GFS2_FS=m +CONFIG_GFS2_FS_LOCKING_DLM=y +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_F2FS_FS=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_FS_ENCRYPTION=y +CONFIG_FS_VERITY=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_QFMT_V2=m +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=m +CONFIG_VIRTIO_FS=m +CONFIG_OVERLAY_FS=y +CONFIG_FSCACHE=m +CONFIG_FSCACHE_STATS=y +CONFIG_CACHEFILES=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=m +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_IOCHARSET="ascii" +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=m +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_CONFIGFS_FS=y +CONFIG_ECRYPT_FS=m +CONFIG_ECRYPT_FS_MESSAGING=y +CONFIG_JFFS2_FS=m +CONFIG_JFFS2_SUMMARY=y +CONFIG_JFFS2_FS_XATTR=y +CONFIG_JFFS2_COMPRESSION_OPTIONS=y +CONFIG_JFFS2_LZO=y +CONFIG_UBIFS_FS=y +CONFIG_UBIFS_FS_ADVANCED_COMPR=y +CONFIG_SQUASHFS=m +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_ROMFS_FS=m +CONFIG_ROMFS_BACKED_BY_BOTH=y +CONFIG_EROFS_FS=m +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_SUNRPC_DEBUG=y +CONFIG_CEPH_FS=m +CONFIG_CEPH_FSCACHE=y +CONFIG_CEPH_FS_POSIX_ACL=y +CONFIG_CIFS=m +CONFIG_CIFS_WEAK_PW_HASH=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_FSCACHE=y +CONFIG_CODA_FS=m +CONFIG_AFS_FS=m +CONFIG_AFS_FSCACHE=y +CONFIG_9P_FS=m +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_DLM=m +CONFIG_DLM_DEBUG=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_HARDENED_USERCOPY=y +# CONFIG_HARDENED_USERCOPY_FALLBACK is not set +CONFIG_FORTIFY_SOURCE=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_TOMOYO=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_SECURITY_YAMA=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y +CONFIG_DEFAULT_SECURITY_APPARMOR=y +CONFIG_CRYPTO_USER=m +CONFIG_CRYPTO_PCRYPT=m +CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_DH=y +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_RMD128=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_RMD256=m +CONFIG_CRYPTO_RMD320=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_TGR192=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_SALSA20=m +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_LZ4=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ANSI_CPRNG=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_DEV_OMAP=y +CONFIG_CRYPTO_DEV_OMAP_SHAM=y +CONFIG_CRYPTO_DEV_OMAP_AES=y +CONFIG_CRYPTO_DEV_OMAP_DES=y +CONFIG_PKCS8_PRIVATE_KEY_PARSER=m +# CONFIG_RAID6_PQ_BENCHMARK is not set +CONFIG_CORDIC=m +# CONFIG_XZ_DEC_X86 is not set +# CONFIG_XZ_DEC_POWERPC is not set +# CONFIG_XZ_DEC_IA64 is not set +# CONFIG_XZ_DEC_SPARC is not set +CONFIG_CMA_SIZE_MBYTES=48 +CONFIG_IRQ_POLL=y +CONFIG_PRINTK_TIME=y +CONFIG_BOOT_PRINTK_DELAY=y +CONFIG_DYNAMIC_DEBUG=y +CONFIG_STRIP_ASM_SYMS=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6 +CONFIG_DETECT_HUNG_TASK=y +CONFIG_SCHEDSTATS=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_NOTIFIER_ERROR_INJECTION=m +CONFIG_FTRACE_SYSCALLS=y +CONFIG_TRACER_SNAPSHOT=y +CONFIG_STACK_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +# CONFIG_RUNTIME_TESTING_MENU is not set +CONFIG_BUG_ON_DATA_CORRUPTION=y +# CONFIG_STRICT_DEVMEM is not set From c87f1b1ade428075ff92fbe7db7bfe5de8d76c61 Mon Sep 17 00:00:00 2001 From: Vaishnav M A Date: Mon, 20 Jul 2020 17:41:57 +0000 Subject: [PATCH 131/135] RFC : mikroBUS driver for add-on boards change log: - v2: remove debug interfaces for adding and deleting mikrobus ports, add mikrobus ports via device-tree, handle pinmux, consider extended usage of mikrobus pins from normal configuration Signed-off-by: Vaishnav M A Rebased-by: Jason Kridner --- MAINTAINERS | 6 + drivers/misc/Kconfig | 1 + drivers/misc/Makefile | 1 + drivers/misc/mikrobus/Kconfig | 16 + drivers/misc/mikrobus/Makefile | 7 + drivers/misc/mikrobus/mikrobus_core.c | 694 ++++++++++++++++++++++ drivers/misc/mikrobus/mikrobus_core.h | 190 ++++++ drivers/misc/mikrobus/mikrobus_manifest.c | 440 ++++++++++++++ drivers/misc/mikrobus/mikrobus_manifest.h | 20 + drivers/misc/mikrobus/mikrobus_port.c | 179 ++++++ include/linux/greybus/greybus_manifest.h | 47 ++ 11 files changed, 1601 insertions(+) create mode 100644 drivers/misc/mikrobus/Kconfig create mode 100644 drivers/misc/mikrobus/Makefile create mode 100644 drivers/misc/mikrobus/mikrobus_core.c create mode 100644 drivers/misc/mikrobus/mikrobus_core.h create mode 100644 drivers/misc/mikrobus/mikrobus_manifest.c create mode 100644 drivers/misc/mikrobus/mikrobus_manifest.h create mode 100644 drivers/misc/mikrobus/mikrobus_port.c diff --git a/MAINTAINERS b/MAINTAINERS index fa9ca33042786..dc3be20872d44 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10866,6 +10866,12 @@ M: Oliver Neukum S: Maintained F: drivers/usb/image/microtek.* +MIKROBUS ADDON BOARD DRIVER +M: Vaishnav M A +S: Maintained +W: https://elinux.org/Mikrobus +F: drivers/misc/mikrobus/ + MIPS M: Ralf Baechle M: Paul Burton diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index e9958d875c89d..0d4340fe84422 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -494,4 +494,5 @@ source "drivers/misc/cxl/Kconfig" source "drivers/misc/ocxl/Kconfig" source "drivers/misc/cardreader/Kconfig" source "drivers/misc/habanalabs/Kconfig" +source "drivers/misc/mikrobus/Kconfig" endmenu diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 48fe4d684684d..67785f032798d 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o obj-$(CONFIG_PCH_PHUB) += pch_phub.o obj-y += ti-st/ obj-y += lis3lv02d/ +obj-y += mikrobus/ obj-$(CONFIG_ALTERA_STAPL) +=altera-stapl/ obj-$(CONFIG_INTEL_MEI) += mei/ obj-$(CONFIG_VMWARE_VMCI) += vmw_vmci/ diff --git a/drivers/misc/mikrobus/Kconfig b/drivers/misc/mikrobus/Kconfig new file mode 100644 index 0000000000000..5f42bc4e94104 --- /dev/null +++ b/drivers/misc/mikrobus/Kconfig @@ -0,0 +1,16 @@ +menuconfig MIKROBUS + tristate "Module for instantiating devices on mikroBUS ports" + help + This option enables the mikroBUS driver. mikroBUS is an add-on + board socket standard that offers maximum expandability with + the smallest number of pins. The mikroBUS driver instantiates + devices on a mikroBUS port described by identifying data present + in an add-on board resident EEPROM, more details on the mikroBUS + driver support and discussion can be found in this eLinux wiki : + elinux.org/Mikrobus + + + Say Y here to enable support for this driver. + + To compile this code as a module, chose M here: the module + will be called mikrobus.ko diff --git a/drivers/misc/mikrobus/Makefile b/drivers/misc/mikrobus/Makefile new file mode 100644 index 0000000000000..af72565103100 --- /dev/null +++ b/drivers/misc/mikrobus/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# mikroBUS Core + +mikrobus-y := mikrobus_core.o mikrobus_manifest.o +mikrobus_port-y := mikrobus_port.o +obj-$(CONFIG_MIKROBUS) += mikrobus.o +obj-$(CONFIG_MIKROBUS) += mikrobus_port.o \ No newline at end of file diff --git a/drivers/misc/mikrobus/mikrobus_core.c b/drivers/misc/mikrobus/mikrobus_core.c new file mode 100644 index 0000000000000..f1ccd4b4d5a2a --- /dev/null +++ b/drivers/misc/mikrobus/mikrobus_core.c @@ -0,0 +1,694 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * mikroBUS driver for instantiating add-on + * board devices with an identifier EEPROM + * + * Copyright 2020 Vaishnav M A, BeagleBoard.org Foundation. + */ + +#define pr_fmt(fmt) "mikrobus:%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mikrobus_core.h" +#include "mikrobus_manifest.h" + +static DEFINE_MUTEX(core_lock); +static DEFINE_IDR(mikrobus_port_idr); +static struct class_compat *mikrobus_port_compat_class; +int __mikrobus_first_dynamic_bus_num; +static bool is_registered; + +const char *MIKROBUS_PINCTRL_STR[] = {"pwm", "uart", "i2c", "spi"}; + +struct bus_type mikrobus_bus_type = { + .name = "mikrobus", +}; +EXPORT_SYMBOL_GPL(mikrobus_bus_type); + +static int mikrobus_port_scan_eeprom(struct mikrobus_port *port) +{ + struct addon_board_info *board; + int manifest_size; + char header[12]; + int retval; + char *buf; + + retval = nvmem_device_read(port->eeprom, 0, 12, header); + if (retval != 12) { + dev_err(&port->dev, "failed to fetch manifest header %d\n", + retval); + return -EINVAL; + } + manifest_size = mikrobus_manifest_header_validate(header, 12); + if (manifest_size < 0) { + dev_err(&port->dev, "invalid manifest size %d\n", + manifest_size); + return -EINVAL; + } + buf = kzalloc(manifest_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + retval = nvmem_device_read(port->eeprom, 0, manifest_size, buf); + if (retval != manifest_size) { + dev_err(&port->dev, "failed to fetch manifest %d\n", retval); + retval = -EINVAL; + goto err_free_buf; + } + board = kzalloc(sizeof(*board), GFP_KERNEL); + if (!board) { + retval = -ENOMEM; + goto err_free_buf; + } + INIT_LIST_HEAD(&board->manifest_descs); + INIT_LIST_HEAD(&board->devices); + retval = mikrobus_manifest_parse(board, buf, manifest_size); + if (!retval) { + dev_err(&port->dev, "failed to parse manifest, size %d\n", + manifest_size); + retval = -EINVAL; + goto err_free_board; + } + retval = mikrobus_board_register(port, board); + if (retval) { + dev_err(&port->dev, "failed to register board %s\n", + board->name); + goto err_free_board; + } + kfree(buf); + return 0; +err_free_board: + kfree(board); +err_free_buf: + kfree(buf); + return retval; +} + +static ssize_t name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%s\n", to_mikrobus_port(dev)->name); +} +static DEVICE_ATTR_RO(name); + +static ssize_t new_device_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mikrobus_port *port = to_mikrobus_port(dev); + struct addon_board_info *board; + int retval; + + if (port->board) { + dev_err(dev, "already has board registered\n"); + return -EBUSY; + } + + board = kzalloc(sizeof(*board), GFP_KERNEL); + if (!board) + return -ENOMEM; + INIT_LIST_HEAD(&board->manifest_descs); + INIT_LIST_HEAD(&board->devices); + retval = mikrobus_manifest_parse(board, (void *)buf, count); + if (!retval) { + dev_err(dev, "failed to parse manifest\n"); + retval = -EINVAL; + goto err_free_board; + } + retval = mikrobus_board_register(port, board); + if (retval) { + dev_err(dev, "failed to register board %s\n", board->name); + retval = -EINVAL; + goto err_free_board; + } + return count; +err_free_board: + kfree(board); + return retval; +} +static DEVICE_ATTR_WO(new_device); + +static ssize_t rescan_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mikrobus_port *port = to_mikrobus_port(dev); + unsigned long id; + int retval; + + if (kstrtoul(buf, 0, &id)) { + dev_err(dev, "cannot parse trigger\n"); + return -EINVAL; + } + if (port->board) { + dev_err(dev, "already has board registered\n"); + return -EBUSY; + } + retval = mikrobus_port_scan_eeprom(port); + if (retval) { + dev_err(dev, "board register from manifest failed\n"); + return -EINVAL; + } + return count; +} +static DEVICE_ATTR_WO(rescan); + +static ssize_t delete_device_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct mikrobus_port *port = to_mikrobus_port(dev); + unsigned long id; + + if (kstrtoul(buf, 0, &id)) { + dev_err(dev, "cannot parse board id"); + return -EINVAL; + } + if (!port->board) { + dev_err(dev, "does not have registered boards"); + return -ENODEV; + } + mikrobus_board_unregister(port, port->board); + return count; +} +static DEVICE_ATTR_IGNORE_LOCKDEP(delete_device, 0200, NULL, delete_device_store); + +static struct attribute *mikrobus_port_attrs[] = { + &dev_attr_new_device.attr, &dev_attr_rescan.attr, + &dev_attr_delete_device.attr, &dev_attr_name.attr, NULL}; +ATTRIBUTE_GROUPS(mikrobus_port); + +static void mikrobus_port_release(struct device *dev) +{ + struct mikrobus_port *port = to_mikrobus_port(dev); + + mutex_lock(&core_lock); + idr_remove(&mikrobus_port_idr, port->id); + mutex_unlock(&core_lock); + kfree(port); +} + +struct device_type mikrobus_port_type = { + .groups = mikrobus_port_groups, + .release = mikrobus_port_release, +}; +EXPORT_SYMBOL_GPL(mikrobus_port_type); + + + +int mikrobus_port_pinctrl_select(struct mikrobus_port *port) +{ + struct pinctrl_state *state; + int retval; + int i; + + for (i = 0; i < MIKROBUS_NUM_PINCTRL_STATE; i++) { + state = pinctrl_lookup_state(port->pinctrl, + port->pinctrl_selected[i]); + if (!IS_ERR(state)) { + retval = pinctrl_select_state(port->pinctrl, state); + pr_info("setting pinctrl %s\n", + port->pinctrl_selected[i]); + if (retval != 0) { + dev_err(&port->dev, "failed to select state %s\n", + port->pinctrl_selected[i]); + return retval; + } + } else { + dev_err(&port->dev, "failed to find state %s\n", + port->pinctrl_selected[i]); + return PTR_ERR(state); + } + } + + return retval; +} +EXPORT_SYMBOL_GPL(mikrobus_port_pinctrl_select); + +static int mikrobus_port_pinctrl_setup(struct mikrobus_port *port, struct addon_board_info *board) +{ + int retval; + int i; + + for (i = 0; i < MIKROBUS_NUM_PINCTRL_STATE; i++) { + switch (i) { + case MIKROBUS_PINCTRL_PWM: + if (board->pin_state[MIKROBUS_PIN_PWM] == MIKROBUS_STATE_PWM) + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], PINCTRL_STATE_DEFAULT); + else + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], MIKROBUS_PINCTRL_STATE_GPIO); + break; + case MIKROBUS_PINCTRL_UART: + if (board->pin_state[MIKROBUS_PIN_RX] == MIKROBUS_STATE_UART) + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], PINCTRL_STATE_DEFAULT); + else + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], MIKROBUS_PINCTRL_STATE_GPIO); + break; + case MIKROBUS_PINCTRL_I2C: + if (board->pin_state[MIKROBUS_PIN_SCL] == MIKROBUS_STATE_I2C) + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], PINCTRL_STATE_DEFAULT); + else + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], MIKROBUS_PINCTRL_STATE_GPIO); + break; + case MIKROBUS_PINCTRL_SPI: + if (board->pin_state[MIKROBUS_PIN_MOSI] == MIKROBUS_STATE_SPI) + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], PINCTRL_STATE_DEFAULT); + else + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], MIKROBUS_PINCTRL_STATE_GPIO); + break; + } + } + + retval = mikrobus_port_pinctrl_select(port); + if (retval) + dev_err(&port->dev, "failed to select pinctrl states [%d]", retval); + return retval; +} + +static int mikrobus_irq_get(struct mikrobus_port *port, int irqno, + int irq_type) +{ + int irq; + + if (irqno > port->gpios->ndescs - 1) { + dev_err(&port->dev, "GPIO %d does not exist", irqno); + return -ENODEV; + } + + irq = gpiod_to_irq(port->gpios->desc[irqno]); + if (irq < 0) { + dev_err(&port->dev, "could not get irq %d", irqno); + return -EINVAL; + } + irq_set_irq_type(irq, irq_type); + return irq; +} + +static int mikrobus_gpio_setup(struct gpio_desc *gpio, int gpio_state) +{ + int retval; + + switch (gpio_state) { + case MIKROBUS_STATE_INPUT: + retval = gpiod_direction_input(gpio); + break; + case MIKROBUS_STATE_OUTPUT_HIGH: + retval = gpiod_direction_output(gpio, 1); + break; + case MIKROBUS_STATE_OUTPUT_LOW: + retval = gpiod_direction_output(gpio, 0); + break; + case MIKROBUS_STATE_PWM: + case MIKROBUS_STATE_SPI: + case MIKROBUS_STATE_I2C: + default: + return 0; + } + return retval; +} + +static char *mikrobus_gpio_chip_name_get(struct mikrobus_port *port, int gpio) +{ + char *name; + struct gpio_chip *gpiochip; + + if (gpio > port->gpios->ndescs - 1) + return NULL; + + gpiochip = gpiod_to_chip(port->gpios->desc[gpio]); + name = kmemdup(gpiochip->label, MIKROBUS_NAME_SIZE, GFP_KERNEL); + return name; +} + +static int mikrobus_gpio_hwnum_get(struct mikrobus_port *port, int gpio) +{ + int hwnum; + struct gpio_chip *gpiochip; + + if (gpio > port->gpios->ndescs - 1) + return -ENODEV; + + gpiochip = gpiod_to_chip(port->gpios->desc[gpio]); + hwnum = desc_to_gpio(port->gpios->desc[gpio]) - gpiochip->base; + return hwnum; +} + +static void mikrobus_board_device_release_all(struct addon_board_info *info) +{ + struct board_device_info *dev; + struct board_device_info *next; + + list_for_each_entry_safe(dev, next, &info->devices, links) { + list_del(&dev->links); + kfree(dev); + } +} + +static int mikrobus_device_register(struct mikrobus_port *port, + struct board_device_info *dev, char *board_name) +{ + struct i2c_board_info *i2c; + struct spi_board_info *spi; + struct platform_device *pdev; + struct gpiod_lookup_table *lookup; + char devname[MIKROBUS_NAME_SIZE]; + int i; + + dev_info(&port->dev, "registering device : %s", dev->drv_name); + + if (dev->gpio_lookup) { + lookup = dev->gpio_lookup; + if (dev->protocol == GREYBUS_PROTOCOL_SPI) { + snprintf(devname, sizeof(devname), "%s.%u", + dev_name(&port->spi_mstr->dev), + port->chip_select[dev->reg]); + lookup->dev_id = kmemdup(devname, MIKROBUS_NAME_SIZE, GFP_KERNEL); + } else if (dev->protocol == GREYBUS_PROTOCOL_RAW) { + snprintf(devname, sizeof(devname), "%s.%u", + dev->drv_name, dev->reg); + lookup->dev_id = kmemdup(devname, MIKROBUS_NAME_SIZE, GFP_KERNEL); + } else + lookup->dev_id = dev->drv_name; + dev_info(&port->dev, " adding lookup table : %s\n", + lookup->dev_id); + for (i = 0; i < dev->num_gpio_resources; i++) { + lookup->table[i].key = + mikrobus_gpio_chip_name_get(port, + lookup->table[i].chip_hwnum); + lookup->table[i].chip_hwnum = + mikrobus_gpio_hwnum_get(port, + lookup->table[i].chip_hwnum); + lookup->table[i].flags = GPIO_ACTIVE_HIGH; + } + gpiod_add_lookup_table(lookup); + } + switch (dev->protocol) { + case GREYBUS_PROTOCOL_SPI: + spi = kzalloc(sizeof(*spi), GFP_KERNEL); + if (!spi) + return -ENOMEM; + strncpy(spi->modalias, dev->drv_name, sizeof(spi->modalias) - 1); + if (dev->irq) + spi->irq = mikrobus_irq_get(port, dev->irq, dev->irq_type); + if (dev->properties) + spi->properties = dev->properties; + spi->chip_select = port->chip_select[dev->reg]; + spi->max_speed_hz = dev->max_speed_hz; + spi->mode = dev->mode; + dev->dev_client = (void *) spi_new_device(port->spi_mstr, spi); + break; + case GREYBUS_PROTOCOL_I2C: + i2c = kzalloc(sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return -ENOMEM; + strncpy(i2c->type, dev->drv_name, sizeof(i2c->type) - 1); + if (dev->irq) + i2c->irq = mikrobus_irq_get(port, dev->irq, dev->irq_type); + if (dev->properties) + i2c->properties = dev->properties; + i2c->addr = dev->reg; + dev->dev_client = (void *) i2c_new_client_device(port->i2c_adap, i2c); + break; + case GREYBUS_PROTOCOL_RAW: + pdev = platform_device_alloc(dev->drv_name, 0); + if (!pdev) + return -ENOMEM; + if (dev->properties) + platform_device_add_properties(pdev, dev->properties); + dev->dev_client = pdev; + platform_device_add(dev->dev_client); + break; + case GREYBUS_PROTOCOL_UART: + dev_info(&port->dev, "serdev devices support not yet added"); + break; + default: + return -EINVAL; + } + return 0; +} + +static void mikrobus_device_unregister(struct mikrobus_port *port, + struct board_device_info *dev, char *board_name) +{ + dev_info(&port->dev, "removing device %s\n", dev->drv_name); + if (dev->gpio_lookup) { + gpiod_remove_lookup_table(dev->gpio_lookup); + kfree(dev->gpio_lookup); + } + kfree(dev->properties); + switch (dev->protocol) { + case GREYBUS_PROTOCOL_SPI: + spi_unregister_device((struct spi_device *)dev->dev_client); + break; + case GREYBUS_PROTOCOL_I2C: + i2c_unregister_device((struct i2c_client *)dev->dev_client); + break; + case GREYBUS_PROTOCOL_RAW: + platform_device_unregister((struct platform_device *)dev->dev_client); + break; + case GREYBUS_PROTOCOL_UART: + dev_err(&port->dev, "serdev devices support not yet added"); + break; + } +} + +int mikrobus_board_register(struct mikrobus_port *port, struct addon_board_info *board) +{ + struct board_device_info *devinfo; + struct board_device_info *next; + int retval; + int i; + + if (WARN_ON(list_empty(&board->devices))) + return false; + if (port->pinctrl) { + retval = mikrobus_port_pinctrl_setup(port, board); + if (retval) + dev_err(&port->dev, "failed to setup pinctrl state [%d]", retval); + + } + if (port->gpios) { + for (i = 0; i < port->gpios->ndescs; i++) { + retval = mikrobus_gpio_setup(port->gpios->desc[i], board->pin_state[i]); + if (retval) + dev_err(&port->dev, "failed to setup gpio %d, state %d", + i, board->pin_state[i]); + } + } + list_for_each_entry_safe(devinfo, next, &board->devices, links) + mikrobus_device_register(port, devinfo, board->name); + port->board = board; + return 0; +} +EXPORT_SYMBOL_GPL(mikrobus_board_register); + +void mikrobus_board_unregister(struct mikrobus_port *port, struct addon_board_info *board) +{ + struct board_device_info *devinfo; + struct board_device_info *next; + + if (WARN_ON(list_empty(&board->devices))) + return; + port->board = NULL; + list_for_each_entry_safe(devinfo, next, &board->devices, links) + mikrobus_device_unregister(port, devinfo, board->name); + mikrobus_board_device_release_all(board); + kfree(board); + port->board = NULL; +} +EXPORT_SYMBOL_GPL(mikrobus_board_unregister); + +static struct i2c_board_info mikrobus_eeprom_info = { + I2C_BOARD_INFO("24c32", 0x57), +}; + +static int mikrobus_port_eeprom_probe(struct mikrobus_port *port) +{ + struct i2c_client *eeprom_client; + struct nvmem_device *eeprom; + char dev_name[MIKROBUS_NAME_SIZE]; + + eeprom_client = i2c_new_client_device(port->i2c_adap, &mikrobus_eeprom_info); + if (!IS_ERR(eeprom_client)) { + pr_info(" mikrobus port %d default eeprom is probed at %02x\n", port->id, + eeprom_client->addr); + snprintf(dev_name, sizeof(dev_name), "%d-%04x0", port->i2c_adap->nr, + eeprom_client->addr); + eeprom = nvmem_device_get(&eeprom_client->dev, dev_name); + if (IS_ERR(eeprom)) { + pr_err(" mikrobus port %d eeprom nvmem device probe failed\n", port->id); + i2c_unregister_device(eeprom_client); + port->eeprom = NULL; + return 0; + } + } else { + pr_info(" mikrobus port %d default eeprom probe failed\n", port->id); + return 0; + } + port->eeprom = eeprom; + port->eeprom_client = eeprom_client; + return 0; +} + +int mikrobus_port_register(struct mikrobus_port *port) +{ + struct device *dev = &port->dev; + int retval; + int id; + + if (WARN_ON(!is_registered)) + return -EAGAIN; + + if (dev->of_node) { + id = of_alias_get_id(dev->of_node, "mikrobus"); + if (id >= 0) { + port->id = id; + mutex_lock(&core_lock); + id = idr_alloc(&mikrobus_port_idr, port, port->id, port->id + 1, + GFP_KERNEL); + mutex_unlock(&core_lock); + if (WARN(id < 0, "couldn't get idr")) + return id == -ENOSPC ? -EBUSY : id; + } + } else { + mutex_lock(&core_lock); + id = idr_alloc(&mikrobus_port_idr, port, __mikrobus_first_dynamic_bus_num, 0, + GFP_KERNEL); + mutex_unlock(&core_lock); + if (id < 0) + return id; + port->id = id; + } + port->dev.bus = &mikrobus_bus_type; + port->dev.type = &mikrobus_port_type; + strncpy(port->name, "mikrobus-port", sizeof(port->name) - 1); + dev_set_name(&port->dev, "mikrobus-%d", port->id); + pr_info("registering port mikrobus-%d ", port->id); + retval = device_register(&port->dev); + if (retval) { + pr_err("port '%d': can't register device (%d)", port->id, retval); + put_device(&port->dev); + return retval; + } + retval = class_compat_create_link(mikrobus_port_compat_class, &port->dev, + port->dev.parent); + if (retval) + dev_warn(&port->dev, "failed to create compatibility class link\n"); + if (!port->eeprom) { + dev_info(&port->dev, "mikrobus port %d eeprom empty probing default eeprom\n", + port->id); + retval = mikrobus_port_eeprom_probe(port); + } + if (port->eeprom) { + retval = mikrobus_port_scan_eeprom(port); + if (retval) { + dev_warn(&port->dev, "failed to register board from manifest\n"); + return 0; + } + } + return retval; +} +EXPORT_SYMBOL_GPL(mikrobus_port_register); + +void mikrobus_port_delete(struct mikrobus_port *port) +{ + struct mikrobus_port *found; + + mutex_lock(&core_lock); + found = idr_find(&mikrobus_port_idr, port->id); + mutex_unlock(&core_lock); + if (found != port) { + pr_err("port [%s] not registered", port->name); + return; + } + if (port->board) { + dev_err(&port->dev, "attempting to delete port with registered boards, port [%s]\n", + port->name); + return; + } + + if (port->eeprom) { + nvmem_device_put(port->eeprom); + i2c_unregister_device(port->eeprom_client); + } + + class_compat_remove_link(mikrobus_port_compat_class, &port->dev, + port->dev.parent); + device_unregister(&port->dev); + mutex_lock(&core_lock); + idr_remove(&mikrobus_port_idr, port->id); + mutex_unlock(&core_lock); + memset(&port->dev, 0, sizeof(port->dev)); +} +EXPORT_SYMBOL_GPL(mikrobus_port_delete); + +static int __init mikrobus_init(void) +{ + int retval; + + retval = bus_register(&mikrobus_bus_type); + if (retval) { + pr_err("bus_register failed (%d)\n", retval); + return retval; + } + mikrobus_port_compat_class = class_compat_register("mikrobus-port"); + if (!mikrobus_port_compat_class) { + pr_err("class_compat register failed (%d)\n", retval); + retval = -ENOMEM; + goto class_err; + } + retval = of_alias_get_highest_id("mikrobus"); + if (retval >= __mikrobus_first_dynamic_bus_num) + __mikrobus_first_dynamic_bus_num = retval + 1; + + is_registered = true; + return 0; + +class_err: + bus_unregister(&mikrobus_bus_type); + idr_destroy(&mikrobus_port_idr); + is_registered = false; + return retval; +} +subsys_initcall(mikrobus_init); + +static void __exit mikrobus_exit(void) +{ + bus_unregister(&mikrobus_bus_type); + class_compat_unregister(mikrobus_port_compat_class); + idr_destroy(&mikrobus_port_idr); +} +module_exit(mikrobus_exit); + +MODULE_AUTHOR("Vaishnav M A "); +MODULE_DESCRIPTION("mikroBUS main module"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/misc/mikrobus/mikrobus_core.h b/drivers/misc/mikrobus/mikrobus_core.h new file mode 100644 index 0000000000000..03894ad6d48f3 --- /dev/null +++ b/drivers/misc/mikrobus/mikrobus_core.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * mikroBUS Driver for instantiating add-on + * board devices with an identifier EEPROM + * + * Copyright 2020 Vaishnav M A, BeagleBoard.org Foundation. + */ + +#ifndef __MIKROBUS_H +#define __MIKROBUS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MIKROBUS_VERSION_MAJOR 0x00 +#define MIKROBUS_VERSION_MINOR 0x03 + +#define MIKROBUS_NAME_SIZE 40 +#define MIKROBUS_PINCTRL_NAME_SIZE 20 + +#define MIKROBUS_NUM_PINCTRL_STATE 4 +#define MIKROBUS_NUM_CS 2 + +#define MIKROBUS_PINCTRL_PWM 0 +#define MIKROBUS_PINCTRL_UART 1 +#define MIKROBUS_PINCTRL_I2C 2 +#define MIKROBUS_PINCTRL_SPI 3 + +#define MIKROBUS_PINCTRL_STATE_GPIO "gpio" + +extern struct bus_type mikrobus_bus_type; +extern struct device_type mikrobus_port_type; +extern const char *MIKROBUS_PINCTRL_STR[]; + +enum mikrobus_property_type { + MIKROBUS_PROPERTY_TYPE_MIKROBUS = 0x00, + MIKROBUS_PROPERTY_TYPE_LINK = 0x01, + MIKROBUS_PROPERTY_TYPE_GPIO = 0x02, + MIKROBUS_PROPERTY_TYPE_U8 = 0x03, + MIKROBUS_PROPERTY_TYPE_U16 = 0x04, + MIKROBUS_PROPERTY_TYPE_U32 = 0x05, + MIKROBUS_PROPERTY_TYPE_U64 = 0x06, +}; + +enum mikrobus_pin { + MIKROBUS_PIN_PWM = 0x00, + MIKROBUS_PIN_INT = 0x01, + MIKROBUS_PIN_RX = 0x02, + MIKROBUS_PIN_TX = 0x03, + MIKROBUS_PIN_SCL = 0x04, + MIKROBUS_PIN_SDA = 0x05, + MIKROBUS_PIN_MOSI = 0x06, + MIKROBUS_PIN_MISO = 0x07, + MIKROBUS_PIN_SCK = 0x08, + MIKROBUS_PIN_CS = 0x09, + MIKROBUS_PIN_RST = 0x0A, + MIKROBUS_PIN_AN = 0x0B, + MIKROBUS_PORT_PIN_COUNT = 0x0C, +}; + +enum mikrobus_pin_state { + MIKROBUS_STATE_INPUT = 0x01, + MIKROBUS_STATE_OUTPUT_HIGH = 0x02, + MIKROBUS_STATE_OUTPUT_LOW = 0x03, + MIKROBUS_STATE_PWM = 0x04, + MIKROBUS_STATE_SPI = 0x05, + MIKROBUS_STATE_I2C = 0x06, + MIKROBUS_STATE_UART = 0x07, +}; + +/* + * board_device_info describes a single device on a mikrobus add-on + * board, an add-on board can present one or more device to the host + * + * @gpio_lookup: used to provide the GPIO lookup table for + * passing the named GPIOs to device drivers. + * @properties: used to provide the property_entry to pass named + * properties to device drivers, applicable only when driver uses + * device_property_read_* calls to fetch the properties. + * @num_gpio_resources: number of named gpio resources for the device, + * used mainly for gpiod_lookup_table memory allocation. + * @num_properties: number of custom properties for the device, + * used mainly for property_entry memory allocation. + * @protocol: used to know the type of the device and it should + * contain one of the values defined under 'enum greybus_class_type' + * under linux/greybus/greybus_manifest.h + * @reg: I2C address for the device, for devices on the SPI bus + * this field is the chip select address relative to the mikrobus + * port:0->device chip select connected to CS pin on mikroBUS port + * 1->device chip select connected to RST Pin on mikroBUS port + * @mode: SPI mode + * @max_speed_hz: SPI max speed(Hz) + * @drv_name: device_id to match with the driver + * @irq_type: type of IRQ trigger , match with defines in linux/interrupt.h + * @irq: irq number relative to the mikrobus port should contain one of the + * values defined under 'enum mikrobus_pin' + * @id: device id starting from 1 + */ +struct board_device_info { + struct gpiod_lookup_table *gpio_lookup; + struct property_entry *properties; + struct list_head links; + unsigned short num_gpio_resources; + unsigned short num_properties; + unsigned short protocol; + unsigned short reg; + unsigned int mode; + void *dev_client; + u32 max_speed_hz; + char *drv_name; + int irq_type; + int irq; + int id; +}; + +/* + * addon_board_info describes a mikrobus add-on device the add-on + * board, an add-on board can present one or more device to the host + * + * @manifest_descs: list of manifest descriptors + * @devices: list of devices on the board + * @pin_state: the state of each pin on the mikrobus port required + * for the add-on board should contain one of the values defined under + * 'enum mikrobus_pin_state' restrictions are as per mikrobus standard + * specifications. + * @name: add-on board name + */ +struct addon_board_info { + struct list_head manifest_descs; + struct list_head devices; + u8 pin_state[MIKROBUS_PORT_PIN_COUNT]; + char *name; +}; + +/* + * mikrobus_port describes the peripherals mapped to a + * mikrobus port. + * + * @eeprom_client: i2c_client corresponding to the eeprom + * on the add-on board. + * @board: pointer to the attached add-on board. + * @i2c_adap: I2C adapter attached to the mikrobus port. + * @spi_mstr: SPI master attached to the mikrobus port. + * @eeprom: nvmem_device for the eeprom on the add-on board. + * @pwm: pwm_device attached to the mikrobus port PWM pin. + * @pinctrl_selected: current pinctrl_selected state. + * @chip_select: chip select number mapped to the SPI + * CS pin on the mikrobus port and the RST pin on the mikrobus + * port + * @id: port id starting from 1 + */ +struct mikrobus_port { + struct i2c_client *eeprom_client; + struct addon_board_info *board; + struct i2c_adapter *i2c_adap; + struct spi_master *spi_mstr; + struct nvmem_device *eeprom; + struct gpio_descs *gpios; + struct pwm_device *pwm; + struct pinctrl *pinctrl; + struct module *owner; + struct device dev; + char name[MIKROBUS_NAME_SIZE]; + char *pinctrl_selected[MIKROBUS_NUM_PINCTRL_STATE]; + unsigned int chip_select[MIKROBUS_NUM_CS]; + int id; +}; +#define to_mikrobus_port(d) container_of(d, struct mikrobus_port, dev) + +void mikrobus_board_unregister(struct mikrobus_port *port, + struct addon_board_info *board); +int mikrobus_board_register(struct mikrobus_port *port, + struct addon_board_info *board); +int mikrobus_port_register(struct mikrobus_port *port); +int mikrobus_port_pinctrl_select(struct mikrobus_port *port); +void mikrobus_port_delete(struct mikrobus_port *port); + +#endif /* __MIKROBUS_H */ diff --git a/drivers/misc/mikrobus/mikrobus_manifest.c b/drivers/misc/mikrobus/mikrobus_manifest.c new file mode 100644 index 0000000000000..629720b4c0fa5 --- /dev/null +++ b/drivers/misc/mikrobus/mikrobus_manifest.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * mikroBUS manifest parsing, an + * extension to Greybus Manifest Parsing + * under drivers/greybus/manifest.c + * + * Copyright 2014-2015 Google Inc. + * Copyright 2014-2015 Linaro Ltd. + */ + +#define pr_fmt(fmt) "mikrobus_manifest:%s: " fmt, __func__ + +#include +#include +#include +#include + +#include "mikrobus_manifest.h" + +struct manifest_desc { + struct list_head links; + size_t size; + void *data; + enum greybus_descriptor_type type; +}; + +static void manifest_descriptor_release_all(struct addon_board_info *board) +{ + struct manifest_desc *descriptor; + struct manifest_desc *next; + + list_for_each_entry_safe(descriptor, next, &board->manifest_descs, links) { + list_del(&descriptor->links); + kfree(descriptor); + } +} + +static int board_descriptor_add(struct addon_board_info *board, struct greybus_descriptor *desc, + size_t size) +{ + struct greybus_descriptor_header *desc_header = &desc->header; + struct manifest_desc *descriptor; + size_t desc_size; + size_t expected_size; + + if (size < sizeof(*desc_header)) { + pr_err("short descriptor (%zu < %zu)", size, sizeof(*desc_header)); + return -EINVAL; + } + desc_size = le16_to_cpu(desc_header->size); + if (desc_size > size) { + pr_err("incorrect descriptor size (%zu != %zu)", size, desc_size); + return -EINVAL; + } + expected_size = sizeof(*desc_header); + switch (desc_header->type) { + case GREYBUS_TYPE_STRING: + expected_size += sizeof(struct greybus_descriptor_string); + expected_size += desc->string.length; + expected_size = ALIGN(expected_size, 4); + break; + case GREYBUS_TYPE_PROPERTY: + expected_size += sizeof(struct greybus_descriptor_property); + expected_size += desc->property.length; + expected_size = ALIGN(expected_size, 4); + break; + case GREYBUS_TYPE_DEVICE: + expected_size += sizeof(struct greybus_descriptor_device); + break; + case GREYBUS_TYPE_MIKROBUS: + expected_size += sizeof(struct greybus_descriptor_mikrobus); + break; + case GREYBUS_TYPE_INTERFACE: + expected_size += sizeof(struct greybus_descriptor_interface); + break; + case GREYBUS_TYPE_CPORT: + expected_size += sizeof(struct greybus_descriptor_cport); + break; + case GREYBUS_TYPE_BUNDLE: + expected_size += sizeof(struct greybus_descriptor_bundle); + break; + case GREYBUS_TYPE_INVALID: + default: + pr_err("invalid descriptor type %d", desc_header->type); + return -EINVAL; + } + + descriptor = kzalloc(sizeof(*descriptor), GFP_KERNEL); + if (!descriptor) + return -ENOMEM; + descriptor->size = desc_size; + descriptor->data = (char *)desc + sizeof(*desc_header); + descriptor->type = desc_header->type; + list_add_tail(&descriptor->links, &board->manifest_descs); + return desc_size; +} + +static char *mikrobus_string_get(struct addon_board_info *board, u8 string_id) +{ + struct greybus_descriptor_string *desc_string; + struct manifest_desc *descriptor; + bool found = false; + char *string; + + if (!string_id) + return NULL; + + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type != GREYBUS_TYPE_STRING) + continue; + desc_string = descriptor->data; + if (desc_string->id == string_id) { + found = true; + break; + } + } + if (!found) + return ERR_PTR(-ENOENT); + string = kmemdup(&desc_string->string, desc_string->length + 1, GFP_KERNEL); + if (!string) + return ERR_PTR(-ENOMEM); + string[desc_string->length] = '\0'; + return string; +} + +static void mikrobus_state_get(struct addon_board_info *board) +{ + struct greybus_descriptor_mikrobus *mikrobus; + struct greybus_descriptor_interface *interface; + struct manifest_desc *descriptor; + bool found = false; + int i; + + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type == GREYBUS_TYPE_MIKROBUS) { + mikrobus = descriptor->data; + found = true; + break; + } + } + if (!found) { + pr_err("mikrobus descriptor not found"); + return; + } + for (i = 0; i < MIKROBUS_PORT_PIN_COUNT; i++) + board->pin_state[i] = mikrobus->pin_state[i]; + + found = false; + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type == GREYBUS_TYPE_INTERFACE) { + interface = descriptor->data; + found = true; + break; + } + } + if (!found) { + pr_err("interface descriptor not found"); + return; + } + board->name = mikrobus_string_get(board, interface->product_stringid); +} + +static struct property_entry * +mikrobus_property_entry_get(struct addon_board_info *board, u8 *prop_link, + int num_properties) +{ + struct greybus_descriptor_property *desc_property; + struct manifest_desc *descriptor; + struct property_entry *properties; + bool found = false; + char *prop_name; + u64 *val_u64; + u32 *val_u32; + u16 *val_u16; + u8 *val_u8; + int i; + + properties = kcalloc(num_properties, sizeof(*properties), GFP_KERNEL); + if (!properties) + return ERR_PTR(-ENOMEM); + for (i = 0; i < num_properties; i++) { + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type != GREYBUS_TYPE_PROPERTY) + continue; + desc_property = descriptor->data; + if (desc_property->id == prop_link[i]) { + found = true; + break; + } + } + if (!found) { + kfree(properties); + return ERR_PTR(-ENOENT); + } + prop_name = mikrobus_string_get(board, desc_property->propname_stringid); + if (!prop_name) { + kfree(properties); + return ERR_PTR(-ENOENT); + } + switch (desc_property->type) { + case MIKROBUS_PROPERTY_TYPE_U8: + val_u8 = kmemdup(&desc_property->value, + (desc_property->length) * sizeof(u8), + GFP_KERNEL); + if (desc_property->length == 1) + properties[i] = PROPERTY_ENTRY_U8(prop_name, *val_u8); + else + properties[i] = PROPERTY_ENTRY_U8_ARRAY_LEN(prop_name, + (void *)desc_property->value, desc_property->length); + break; + case MIKROBUS_PROPERTY_TYPE_U16: + val_u16 = kmemdup(&desc_property->value, + (desc_property->length) * sizeof(u16), GFP_KERNEL); + if (desc_property->length == 1) + properties[i] = PROPERTY_ENTRY_U16(prop_name, *val_u16); + else + properties[i] = PROPERTY_ENTRY_U16_ARRAY_LEN(prop_name, + (void *)desc_property->value, desc_property->length); + break; + case MIKROBUS_PROPERTY_TYPE_U32: + val_u32 = kmemdup(&desc_property->value, + (desc_property->length) * sizeof(u32), GFP_KERNEL); + if (desc_property->length == 1) + properties[i] = PROPERTY_ENTRY_U32(prop_name, *val_u32); + else + properties[i] = PROPERTY_ENTRY_U32_ARRAY_LEN(prop_name, + (void *)desc_property->value, desc_property->length); + break; + case MIKROBUS_PROPERTY_TYPE_U64: + val_u64 = kmemdup(&desc_property->value, + (desc_property->length) * sizeof(u64), GFP_KERNEL); + if (desc_property->length == 1) + properties[i] = PROPERTY_ENTRY_U64(prop_name, *val_u64); + else + properties[i] = PROPERTY_ENTRY_U64_ARRAY_LEN(prop_name, + (void *)desc_property->value, desc_property->length); + break; + default: + kfree(properties); + return ERR_PTR(-EINVAL); + } + } + return properties; +} + +static u8 *mikrobus_property_link_get(struct addon_board_info *board, u8 prop_id, + struct board_device_info *board_dev, u8 prop_type) +{ + struct greybus_descriptor_property *desc_property; + struct manifest_desc *descriptor; + bool found = false; + u8 *val_u8; + + if (!prop_id) + return NULL; + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type != GREYBUS_TYPE_PROPERTY) + continue; + desc_property = descriptor->data; + if (desc_property->id == prop_id && desc_property->type == prop_type) { + found = true; + break; + } + } + if (!found) + return ERR_PTR(-ENOENT); + val_u8 = kmemdup(&desc_property->value, desc_property->length, GFP_KERNEL); + if (prop_type == MIKROBUS_PROPERTY_TYPE_GPIO) + board_dev->num_gpio_resources = desc_property->length; + else if (prop_type == MIKROBUS_PROPERTY_TYPE_LINK) + board_dev->num_properties = desc_property->length; + return val_u8; +} + +static int mikrobus_manifest_attach_device(struct addon_board_info *board, + struct greybus_descriptor_device *dev_desc) +{ + struct board_device_info *board_dev; + struct gpiod_lookup_table *lookup; + struct greybus_descriptor_property *desc_property; + struct manifest_desc *descriptor; + u8 *gpio_desc_link; + u8 *prop_link; + int retval; + int i; + + board_dev = kzalloc(sizeof(*board_dev), GFP_KERNEL); + if (!board_dev) + return -ENOMEM; + board_dev->id = dev_desc->id; + board_dev->drv_name = mikrobus_string_get(board, dev_desc->driver_stringid); + if (!board_dev->drv_name) { + retval = -ENOENT; + goto err_free_board_dev; + } + board_dev->protocol = dev_desc->protocol; + board_dev->reg = dev_desc->reg; + board_dev->irq = dev_desc->irq; + board_dev->irq_type = dev_desc->irq_type; + board_dev->max_speed_hz = le32_to_cpu(dev_desc->max_speed_hz); + board_dev->mode = dev_desc->mode; + pr_info("parsed device %d, driver=%s", board_dev->id, board_dev->drv_name); + + if (dev_desc->prop_link > 0) { + prop_link = mikrobus_property_link_get(board, dev_desc->prop_link, + board_dev, MIKROBUS_PROPERTY_TYPE_LINK); + if (!prop_link) { + retval = -ENOENT; + goto err_free_board_dev; + } + pr_info("device %d, number of properties=%d", board_dev->id, + board_dev->num_properties); + board_dev->properties = mikrobus_property_entry_get(board, prop_link, + board_dev->num_properties); + } + + if (dev_desc->gpio_link > 0) { + gpio_desc_link = mikrobus_property_link_get(board, dev_desc->gpio_link, board_dev, + MIKROBUS_PROPERTY_TYPE_GPIO); + if (!gpio_desc_link) { + retval = -ENOENT; + goto err_free_board_dev; + } + pr_info("device %d, number of gpio resource=%d", board_dev->id, + board_dev->num_gpio_resources); + lookup = kzalloc(struct_size(lookup, table, board_dev->num_gpio_resources), + GFP_KERNEL); + if (!lookup) { + retval = -ENOMEM; + goto err_free_board_dev; + } + for (i = 0; i < board_dev->num_gpio_resources; i++) { + list_for_each_entry(descriptor, &board->manifest_descs, links) { + if (descriptor->type != GREYBUS_TYPE_PROPERTY) + continue; + desc_property = descriptor->data; + if (desc_property->id == gpio_desc_link[i]) { + lookup->table[i].chip_hwnum = *desc_property->value; + lookup->table[i].con_id = + mikrobus_string_get(board, + desc_property->propname_stringid); + break; + } + } + } + board_dev->gpio_lookup = lookup; + } + list_add_tail(&board_dev->links, &board->devices); + return 0; +err_free_board_dev: + kfree(board_dev); + return retval; +} + +static int mikrobus_manifest_parse_devices(struct addon_board_info *board) +{ + struct greybus_descriptor_device *desc_device; + struct manifest_desc *desc, *next; + int retval; + int devcount = 0; + + list_for_each_entry_safe(desc, next, &board->manifest_descs, links) { + if (desc->type != GREYBUS_TYPE_DEVICE) + continue; + desc_device = desc->data; + retval = mikrobus_manifest_attach_device(board, desc_device); + devcount++; + } + return devcount; +} + +int mikrobus_manifest_parse(struct addon_board_info *board, void *data, + size_t size) +{ + struct greybus_manifest_header *header; + struct greybus_manifest *manifest; + struct greybus_descriptor *desc; + u16 manifest_size; + int dev_count; + int desc_size; + + if (size < sizeof(*header)) { + pr_err("short manifest (%zu < %zu)", size, sizeof(*header)); + return -EINVAL; + } + + manifest = data; + header = &manifest->header; + manifest_size = le16_to_cpu(header->size); + + if (manifest_size != size) { + pr_err("invalid manifest size(%zu < %zu)", size, manifest_size); + return -EINVAL; + } + + if (header->version_major > MIKROBUS_VERSION_MAJOR) { + pr_err("manifest version too new (%u.%u > %u.%u)", + header->version_major, header->version_minor, + MIKROBUS_VERSION_MAJOR, MIKROBUS_VERSION_MINOR); + return -EINVAL; + } + + desc = manifest->descriptors; + size -= sizeof(*header); + while (size) { + desc_size = board_descriptor_add(board, desc, size); + if (desc_size < 0) { + pr_err("invalid manifest descriptor, size: %zu", desc_size); + return -EINVAL; + } + desc = (void *)desc + desc_size; + size -= desc_size; + } + mikrobus_state_get(board); + dev_count = mikrobus_manifest_parse_devices(board); + pr_info(" %s manifest parsed with %d devices", board->name, dev_count); + manifest_descriptor_release_all(board); + return true; +} + +size_t mikrobus_manifest_header_validate(void *data, size_t size) +{ + struct greybus_manifest_header *header; + u16 manifest_size; + + if (size < sizeof(*header)) { + pr_err("short manifest (%zu < %zu)", size, sizeof(*header)); + return -EINVAL; + } + header = data; + manifest_size = le16_to_cpu(header->size); + if (header->version_major > MIKROBUS_VERSION_MAJOR) { + pr_err("manifest version too new (%u.%u > %u.%u)", + header->version_major, header->version_minor, + MIKROBUS_VERSION_MAJOR, MIKROBUS_VERSION_MINOR); + return -EINVAL; + } + return manifest_size; +} + diff --git a/drivers/misc/mikrobus/mikrobus_manifest.h b/drivers/misc/mikrobus/mikrobus_manifest.h new file mode 100644 index 0000000000000..36b64b2093f52 --- /dev/null +++ b/drivers/misc/mikrobus/mikrobus_manifest.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * mikroBUS manifest definition + * extension to Greybus Manifest Definition + * + * Copyright 2014-2015 Google Inc. + * Copyright 2014-2015 Linaro Ltd. + * + * Released under the GPLv2 and BSD licenses. + */ + +#ifndef __MIKROBUS_MANIFEST_H +#define __MIKROBUS_MANIFEST_H + +#include "mikrobus_core.h" + +int mikrobus_manifest_parse(struct addon_board_info *info, void *data, size_t size); +size_t mikrobus_manifest_header_validate(void *data, size_t size); + +#endif /* __MIKROBUS_MANIFEST_H */ diff --git a/drivers/misc/mikrobus/mikrobus_port.c b/drivers/misc/mikrobus/mikrobus_port.c new file mode 100644 index 0000000000000..d54520b98053a --- /dev/null +++ b/drivers/misc/mikrobus/mikrobus_port.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * mikroBUS driver for adding mikrobus port from device tree + * + * Copyright 2020 Vaishnav M A, BeagleBoard.org Foundation. + */ +#define pr_fmt(fmt) "mikrobus_port:%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mikrobus_core.h" + +static int mikrobus_port_probe_pinctrl_setup(struct mikrobus_port *port) +{ + struct pinctrl_state *state; + struct device *dev = port->dev.parent; + int retval, i; + + state = pinctrl_lookup_state(port->pinctrl, PINCTRL_STATE_DEFAULT); + if (!IS_ERR(state)) { + retval = pinctrl_select_state(port->pinctrl, state); + if (retval != 0) { + dev_err(dev, "Failed to select state %s\n", + PINCTRL_STATE_DEFAULT); + return retval; + } + } else { + dev_err(dev, "failed to find state %s\n", + PINCTRL_STATE_DEFAULT); + return PTR_ERR(state); + } + + for (i = 0; i < MIKROBUS_NUM_PINCTRL_STATE; i++) { + port->pinctrl_selected[i] = + kmalloc(MIKROBUS_PINCTRL_NAME_SIZE, GFP_KERNEL); + sprintf(port->pinctrl_selected[i], "%s_%s", + MIKROBUS_PINCTRL_STR[i], PINCTRL_STATE_DEFAULT); + } + + retval = mikrobus_port_pinctrl_select(port); + if (retval) + dev_err(dev, "failed to select pinctrl states [%d]", retval); + return retval; +} + +static int mikrobus_port_probe(struct platform_device *pdev) +{ + struct mikrobus_port *port; + struct device *dev = &pdev->dev; + struct device_node *i2c_adap_np; + int retval; + u32 val; + + port = kzalloc(sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + i2c_adap_np = of_parse_phandle(dev->of_node, "i2c-adapter", 0); + if (!i2c_adap_np) { + dev_err(dev, "cannot parse i2c-adapter\n"); + retval = -ENODEV; + goto err_port; + } + port->i2c_adap = of_find_i2c_adapter_by_node(i2c_adap_np); + of_node_put(i2c_adap_np); + retval = device_property_read_u32(dev, "spi-master", &val); + if (retval) { + dev_err(dev, "failed to get spi-master [%d]\n", retval); + goto err_port; + } + port->spi_mstr = spi_busnum_to_master(val); + retval = device_property_read_u32_array(dev, "spi-cs", + port->chip_select, 2); + if (retval) { + dev_err(dev, "failed to get spi-cs [%d]\n", retval); + goto err_port; + } + //port->pwm = devm_pwm_get(dev, NULL); + //if (IS_ERR(port->pwm)) { + // retval = PTR_ERR(port->pwm); + // if (retval != -EPROBE_DEFER) + // dev_err(dev, "failed to request PWM device [%d]\n", + // retval); + // goto err_port; + //} + port->gpios = gpiod_get_array(dev, "mikrobus", GPIOD_OUT_LOW); + if (IS_ERR(port->gpios)) { + retval = PTR_ERR(port->gpios); + dev_err(dev, "failed to get gpio array [%d]\n", retval); + goto err_port; + } + port->pinctrl = devm_pinctrl_get(dev); + if (IS_ERR(port->pinctrl)) { + retval = PTR_ERR(port->pinctrl); + dev_err(dev, "failed to get pinctrl [%d]\n", retval); + goto err_port; + } + port->dev.parent = dev; + port->dev.of_node = pdev->dev.of_node; + + retval = mikrobus_port_probe_pinctrl_setup(port); + if (retval) { + dev_err(dev, "failed to setup pinctrl [%d]\n", retval); + goto err_port; + } + + retval = mikrobus_port_register(port); + if (retval) { + pr_err("port : can't register port [%d]\n", retval); + goto err_port; + } + platform_set_drvdata(pdev, port); + return 0; +err_port: + kfree(port); + return retval; +} + +static int mikrobus_port_remove(struct platform_device *pdev) +{ + struct mikrobus_port *port = platform_get_drvdata(pdev); + + mikrobus_port_delete(port); + return 0; +} + +static const struct of_device_id mikrobus_port_of_match[] = { + {.compatible = "linux,mikrobus"}, + {}, +}; +MODULE_DEVICE_TABLE(of, mikrobus_port_of_match); + +static struct platform_driver mikrobus_port_driver = { + .probe = mikrobus_port_probe, + .remove = mikrobus_port_remove, + .driver = { + .name = "mikrobus", + .of_match_table = of_match_ptr(mikrobus_port_of_match), + }, +}; + +static int __init +mikrobus_port_init_driver(void) +{ + int retval; + + retval = platform_driver_register(&mikrobus_port_driver); + if (retval) + pr_err("driver register failed [%d]\n", retval); + return retval; +} +subsys_initcall(mikrobus_port_init_driver); + +static void __exit mikrobus_port_exit_driver(void) +{ + platform_driver_unregister(&mikrobus_port_driver); +} +module_exit(mikrobus_port_exit_driver); + +MODULE_AUTHOR("Vaishnav M A "); +MODULE_DESCRIPTION("mikroBUS port module"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/greybus/greybus_manifest.h b/include/linux/greybus/greybus_manifest.h index 6e62fe4787122..821661ea7f01c 100644 --- a/include/linux/greybus/greybus_manifest.h +++ b/include/linux/greybus/greybus_manifest.h @@ -23,6 +23,9 @@ enum greybus_descriptor_type { GREYBUS_TYPE_STRING = 0x02, GREYBUS_TYPE_BUNDLE = 0x03, GREYBUS_TYPE_CPORT = 0x04, + GREYBUS_TYPE_MIKROBUS = 0x05, + GREYBUS_TYPE_PROPERTY = 0x06, + GREYBUS_TYPE_DEVICE = 0x07, }; enum greybus_protocol { @@ -151,6 +154,47 @@ struct greybus_descriptor_cport { __u8 protocol_id; /* enum greybus_protocol */ } __packed; +/* + * A mikrobus descriptor is used to describe the details + * about the bus ocnfiguration for the add-on board + * connected to the mikrobus port. + */ +struct greybus_descriptor_mikrobus { + __u8 pin_state[12]; +} __packed; + +/* + * A property descriptor is used to pass named properties + * to device drivers through the unified device properties + * interface under linux/property.h + */ +struct greybus_descriptor_property { + __u8 length; + __u8 id; + __u8 propname_stringid; + __u8 type; + __u8 value[0]; +} __packed; + +/* + * A device descriptor is used to describe the + * details required by a add-on board device + * driver. + */ +struct greybus_descriptor_device { + __u8 id; + __u8 driver_stringid; + __u8 protocol; + __u8 reg; + __le32 max_speed_hz; + __u8 irq; + __u8 irq_type; + __u8 mode; + __u8 prop_link; + __u8 gpio_link; + __u8 pad[3]; +} __packed; + struct greybus_descriptor_header { __le16 size; __u8 type; /* enum greybus_descriptor_type */ @@ -164,6 +208,9 @@ struct greybus_descriptor { struct greybus_descriptor_interface interface; struct greybus_descriptor_bundle bundle; struct greybus_descriptor_cport cport; + struct greybus_descriptor_mikrobus mikrobus; + struct greybus_descriptor_property property; + struct greybus_descriptor_device device; }; } __packed; From ccae755dd2ead728d2d5f0f67d4392fc69b88178 Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Thu, 27 Aug 2020 02:00:14 -0400 Subject: [PATCH 132/135] mikroBUS build fixes --- drivers/misc/mikrobus/Makefile | 5 ++--- drivers/misc/mikrobus/mikrobus_core.c | 2 ++ drivers/misc/mikrobus/mikrobus_manifest.c | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/misc/mikrobus/Makefile b/drivers/misc/mikrobus/Makefile index af72565103100..05a6ec3a013d7 100644 --- a/drivers/misc/mikrobus/Makefile +++ b/drivers/misc/mikrobus/Makefile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # mikroBUS Core -mikrobus-y := mikrobus_core.o mikrobus_manifest.o -mikrobus_port-y := mikrobus_port.o obj-$(CONFIG_MIKROBUS) += mikrobus.o -obj-$(CONFIG_MIKROBUS) += mikrobus_port.o \ No newline at end of file +obj-$(CONFIG_MIKROBUS) += mikrobus_port.o +mikrobus-y := mikrobus_core.o mikrobus_manifest.o diff --git a/drivers/misc/mikrobus/mikrobus_core.c b/drivers/misc/mikrobus/mikrobus_core.c index f1ccd4b4d5a2a..7ac51c74d0fbe 100644 --- a/drivers/misc/mikrobus/mikrobus_core.c +++ b/drivers/misc/mikrobus/mikrobus_core.c @@ -689,6 +689,8 @@ static void __exit mikrobus_exit(void) } module_exit(mikrobus_exit); +EXPORT_SYMBOL(MIKROBUS_PINCTRL_STR); + MODULE_AUTHOR("Vaishnav M A "); MODULE_DESCRIPTION("mikroBUS main module"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/misc/mikrobus/mikrobus_manifest.c b/drivers/misc/mikrobus/mikrobus_manifest.c index 629720b4c0fa5..077d61e006b00 100644 --- a/drivers/misc/mikrobus/mikrobus_manifest.c +++ b/drivers/misc/mikrobus/mikrobus_manifest.c @@ -389,7 +389,7 @@ int mikrobus_manifest_parse(struct addon_board_info *board, void *data, manifest_size = le16_to_cpu(header->size); if (manifest_size != size) { - pr_err("invalid manifest size(%zu < %zu)", size, manifest_size); + pr_err("invalid manifest size(%zu < %u)", size, manifest_size); return -EINVAL; } @@ -405,7 +405,7 @@ int mikrobus_manifest_parse(struct addon_board_info *board, void *data, while (size) { desc_size = board_descriptor_add(board, desc, size); if (desc_size < 0) { - pr_err("invalid manifest descriptor, size: %zu", desc_size); + pr_err("invalid manifest descriptor, size: %u", desc_size); return -EINVAL; } desc = (void *)desc + desc_size; From e0707fed305cb4b35402a4b97a8e0e239e2e14ed Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Sun, 3 Jan 2021 17:19:55 -0500 Subject: [PATCH 133/135] Add hm3301 driver from Seeed From https://github.com/Seeed-Studio/seeed-linux-dtoverlays/tree/master/modules/hm3301 --- arch/arm/configs/bb.org_defconfig | 1 + drivers/iio/chemical/Kconfig | 8 + drivers/iio/chemical/Makefile | 1 + drivers/iio/chemical/hm3301.c | 465 ++++++++++++++++++++++++++++++ 4 files changed, 475 insertions(+) create mode 100644 drivers/iio/chemical/hm3301.c diff --git a/arch/arm/configs/bb.org_defconfig b/arch/arm/configs/bb.org_defconfig index 97f7729f6fdd9..585d8189ef260 100644 --- a/arch/arm/configs/bb.org_defconfig +++ b/arch/arm/configs/bb.org_defconfig @@ -2202,6 +2202,7 @@ CONFIG_AD8366=m CONFIG_ATLAS_PH_SENSOR=m CONFIG_BME680=m CONFIG_CCS811=m +CONFIG_HM3301=m CONFIG_IAQCORE=m CONFIG_PMS7003=m CONFIG_SENSIRION_SGP30=m diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig index a7e65a59bf429..41996e346ad3c 100644 --- a/drivers/iio/chemical/Kconfig +++ b/drivers/iio/chemical/Kconfig @@ -54,6 +54,14 @@ config CCS811 Say Y here to build I2C interface support for the AMS CCS811 VOC (Volatile Organic Compounds) sensor +config HM3301 + tristate "HM3301 Particulate Matter sensor" + depends on I2C + select IIO_BUFFER + help + Say Y here to build I2C interface support for the HM3301 + Particulate Matter laser interferometer sensor + config IAQCORE tristate "AMS iAQ-Core VOC sensors" depends on I2C diff --git a/drivers/iio/chemical/Makefile b/drivers/iio/chemical/Makefile index f97270bc40346..76efb3b6b180e 100644 --- a/drivers/iio/chemical/Makefile +++ b/drivers/iio/chemical/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_BME680) += bme680_core.o obj-$(CONFIG_BME680_I2C) += bme680_i2c.o obj-$(CONFIG_BME680_SPI) += bme680_spi.o obj-$(CONFIG_CCS811) += ccs811.o +obj-$(CONFIG_HM3301) += hm3301.o obj-$(CONFIG_IAQCORE) += ams-iaq-core.o obj-$(CONFIG_PMS7003) += pms7003.o obj-$(CONFIG_SENSIRION_SGP30) += sgp30.o diff --git a/drivers/iio/chemical/hm3301.c b/drivers/iio/chemical/hm3301.c new file mode 100644 index 0000000000000..ea047d82482dd --- /dev/null +++ b/drivers/iio/chemical/hm3301.c @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sensirion HM3301 particulate matter sensor driver + * + * Copyright (c) Tomasz Duszynski + * + * I2C slave address: 0x40 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* minimum and maximum self cleaning periods in seconds */ +#define HM3301_AUTO_CLEANING_PERIOD_MIN 0 +#define HM3301_AUTO_CLEANING_PERIOD_MAX 604800 + +/* HM3301 commands */ +#define HM3301_START_MEAS 0x0010 +#define HM3301_STOP_MEAS 0x0104 +#define HM3301_READ_DATA_READY_FLAG 0x0202 +#define HM3301_READ_DATA 0x0300 +#define HM3301_READ_SERIAL 0xd033 +#define HM3301_START_FAN_CLEANING 0x5607 +#define HM3301_AUTO_CLEANING_PERIOD 0x8004 +/* not a sensor command per se, used only to distinguish write from read */ +#define HM3301_READ_AUTO_CLEANING_PERIOD 0x8005 + +enum { + PM1, + PM2P5, + PM10, +}; + +enum { + RESET, + MEASURING, +}; + +struct hm3301_state { + struct i2c_client *client; + /* + * Guards against concurrent access to sensor registers. + * Must be held whenever sequence of commands is to be executed. + */ + struct mutex lock; + int state; +}; + + +static int hm3301_write_then_read(struct hm3301_state *state, u8 *txbuf, + int txsize, u8 *rxbuf, int rxsize) +{ + int ret; + /* + * Sensor does not support repeated start so instead of + * sending two i2c messages in a row we just send one by one. + */ + ret = i2c_master_send(state->client, txbuf, txsize); + if (ret != txsize) + return ret < 0 ? ret : -EIO; + + msleep(10); + if (!rxbuf) + return 0; + + ret = i2c_master_recv(state->client, rxbuf, rxsize); + if (ret != rxsize) + return ret < 0 ? ret : -EIO; + + return 0; +} + +static int hm3301_do_cmd(struct hm3301_state *state, u16 cmd, u8 *data, int size) +{ + /* + * Internally sensor stores measurements in a following manner: + * + * PM1: upper two bytes, crc8, lower two bytes, crc8 + * PM2P5: upper two bytes, crc8, lower two bytes, crc8 + * PM10: upper two bytes, crc8, lower two bytes, crc8 + * + * What follows next are number concentration measurements and + * typical particle size measurement which we omit. + */ + u8 txbuf[2]; + u8 rxbuf[49] = {8}; + int i, ret = 0; + + switch (cmd) { + case HM3301_START_MEAS: + txbuf[0] = 0; + txbuf[1] = 0x88; + ret = hm3301_write_then_read(state, txbuf, 2, NULL, 0); + break; + case HM3301_STOP_MEAS: + case HM3301_START_FAN_CLEANING: + break; + case HM3301_READ_AUTO_CLEANING_PERIOD: + /* fall through */ + case HM3301_READ_DATA_READY_FLAG: + case HM3301_READ_DATA: + case HM3301_READ_SERIAL: + /* every two data bytes are checksummed */ + txbuf[0] = 0x40; + txbuf[1] = 0; + ret = hm3301_write_then_read(state, txbuf, 2, rxbuf, 29); + /* validate received data and strip off crc bytes */ + for (i = 0; i < 29; i ++) { + *data++ = rxbuf[i]; + } + break; + case HM3301_AUTO_CLEANING_PERIOD: + break; + } + + if (ret) + return ret; + + + return 0; +} + + + +static int hm3301_do_meas(struct hm3301_state *state, s32 *data, int size) +{ + int i, ret; + u8 tmp[32]; + + if (state->state == RESET) { + ret = hm3301_do_cmd(state, HM3301_START_MEAS, NULL, 0); + if (ret) + return ret; + + state->state = MEASURING; + } + +#if 0 + while (tries--) { + ret = hm3301_do_cmd(state, HM3301_READ_DATA_READY_FLAG, tmp, 2); + if (ret) + return -EIO; + + /* new measurements ready to be read */ + if (tmp[1] == 1) + break; + + msleep_interruptible(300); + } + + if (tries == -1) + return -ETIMEDOUT; +#endif + ret = hm3301_do_cmd(state, HM3301_READ_DATA, tmp, sizeof(int) * size); + if (ret) + return ret; + + int a; + for (i = 0; i < size; i++) { + for (a = 2; a< (size+2); a++) + { + u16 value = 0; + value = (u16) tmp[a * 2] << 8 | tmp[a * 2 +1]; + data[i] = value * 100; + } + } + return 0; +} + +static irqreturn_t hm3301_trigger_handler(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct iio_dev *indio_dev = pf->indio_dev; + struct hm3301_state *state = iio_priv(indio_dev); + int ret; + struct { + s32 data[3]; /* PM1, PM2P5, PM10 */ + s64 ts; + } scan; + + mutex_lock(&state->lock); + ret = hm3301_do_meas(state, scan.data, ARRAY_SIZE(scan.data)); + mutex_unlock(&state->lock); + if (ret) + goto err; + + iio_push_to_buffers_with_timestamp(indio_dev, &scan, + iio_get_time_ns(indio_dev)); +err: + iio_trigger_notify_done(indio_dev->trig); + + return IRQ_HANDLED; +} + +static int hm3301_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct hm3301_state *state = iio_priv(indio_dev); + int data[4], ret = -EINVAL; + + switch (mask) { + case IIO_CHAN_INFO_PROCESSED: + switch (chan->type) { + case IIO_MASSCONCENTRATION: + mutex_lock(&state->lock); + /* read up to the number of bytes actually needed */ + switch (chan->channel2) { + case IIO_MOD_PM1: + ret = hm3301_do_meas(state, data, 1); + break; + case IIO_MOD_PM2P5: + ret = hm3301_do_meas(state, data, 2); + break; + case IIO_MOD_PM10: + ret = hm3301_do_meas(state, data, 3); + break; + } + mutex_unlock(&state->lock); + if (ret) + return ret; + + *val = data[chan->address] / 100; + *val2 = (data[chan->address] % 100) * 10000; + + return IIO_VAL_INT_PLUS_MICRO; + default: + return -EINVAL; + } + case IIO_CHAN_INFO_SCALE: + switch (chan->type) { + case IIO_MASSCONCENTRATION: + switch (chan->channel2) { + case IIO_MOD_PM1: + case IIO_MOD_PM2P5: + case IIO_MOD_PM10: + *val = 0; + *val2 = 10000; + + return IIO_VAL_INT_PLUS_MICRO; + default: + return -EINVAL; + } + default: + return -EINVAL; + } + } + + return -EINVAL; +} + + + +static ssize_t start_cleaning_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct hm3301_state *state = iio_priv(indio_dev); + int val, ret; + + if (kstrtoint(buf, 0, &val) || val != 1) + return -EINVAL; + + mutex_lock(&state->lock); + ret = hm3301_do_cmd(state, HM3301_START_FAN_CLEANING, NULL, 0); + mutex_unlock(&state->lock); + if (ret) + return ret; + + return len; +} + +static ssize_t cleaning_period_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct hm3301_state *state = iio_priv(indio_dev); + u8 tmp[4]; + int ret; + + mutex_lock(&state->lock); + ret = hm3301_do_cmd(state, HM3301_READ_AUTO_CLEANING_PERIOD, tmp, 4); + mutex_unlock(&state->lock); + if (ret) + return ret; + + return sprintf(buf, "%d\n", get_unaligned_be32(tmp)); +} + +static ssize_t cleaning_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct hm3301_state *state = iio_priv(indio_dev); + int val, ret; + u8 tmp[4]; + + if (kstrtoint(buf, 0, &val)) + return -EINVAL; + + if ((val < HM3301_AUTO_CLEANING_PERIOD_MIN) || + (val > HM3301_AUTO_CLEANING_PERIOD_MAX)) + return -EINVAL; + + put_unaligned_be32(val, tmp); + + mutex_lock(&state->lock); + ret = hm3301_do_cmd(state, HM3301_AUTO_CLEANING_PERIOD, tmp, 0); + if (ret) { + mutex_unlock(&state->lock); + return ret; + } + + msleep(20); + + /* + * sensor requires reset in order to return up to date self cleaning + * period + */ +// ret = hm3301_do_cmd_reset(state); +// if (ret) +// dev_warn(dev, +// "period changed but reads will return the old value\n"); + + mutex_unlock(&state->lock); + + return len; +} + +static ssize_t cleaning_period_available_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "[%d %d %d]\n", + HM3301_AUTO_CLEANING_PERIOD_MIN, 1, + HM3301_AUTO_CLEANING_PERIOD_MAX); +} + +static IIO_DEVICE_ATTR_WO(start_cleaning, 0); +static IIO_DEVICE_ATTR_RW(cleaning_period, 0); +static IIO_DEVICE_ATTR_RO(cleaning_period_available, 0); + +static struct attribute *hm3301_attrs[] = { + &iio_dev_attr_start_cleaning.dev_attr.attr, + &iio_dev_attr_cleaning_period.dev_attr.attr, + &iio_dev_attr_cleaning_period_available.dev_attr.attr, + NULL +}; + +static const struct attribute_group hm3301_attr_group = { + .attrs = hm3301_attrs, +}; + +static const struct iio_info hm3301_info = { + .attrs = &hm3301_attr_group, + .read_raw = hm3301_read_raw, +}; + +#define HM3301_CHAN(_index, _mod) { \ + .type = IIO_MASSCONCENTRATION, \ + .modified = 1, \ + .channel2 = IIO_MOD_ ## _mod, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ + .address = _mod, \ + .scan_index = _index, \ + .scan_type = { \ + .sign = 'u', \ + .realbits = 19, \ + .storagebits = 32, \ + .endianness = IIO_CPU, \ + }, \ +} + +static const struct iio_chan_spec hm3301_channels[] = { + HM3301_CHAN(0, PM1), + HM3301_CHAN(1, PM2P5), + HM3301_CHAN(2, PM10), + IIO_CHAN_SOFT_TIMESTAMP(3), +}; + +static void hm3301_stop_meas(void *data) +{ + struct hm3301_state *state = data; + hm3301_do_cmd(state, HM3301_STOP_MEAS, NULL, 0); +} + +static const unsigned long hm3301_scan_masks[] = { 0x0f, 0x00 }; + +static int hm3301_probe(struct i2c_client *client) +{ + struct iio_dev *indio_dev; + struct hm3301_state *state; + u8 buf[32]; + int ret; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -EOPNOTSUPP; + + indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*state)); + if (!indio_dev) + return -ENOMEM; + + state = iio_priv(indio_dev); + i2c_set_clientdata(client, indio_dev); + state->client = client; + state->state = RESET; + indio_dev->dev.parent = &client->dev; + indio_dev->info = &hm3301_info; + indio_dev->name = client->name; + indio_dev->channels = hm3301_channels; + indio_dev->num_channels = ARRAY_SIZE(hm3301_channels); + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->available_scan_masks = hm3301_scan_masks; + + mutex_init(&state->lock); + ret = devm_add_action_or_reset(&client->dev, hm3301_stop_meas, state); + if (ret) + return ret; + ret = devm_iio_triggered_buffer_setup(&client->dev, indio_dev, NULL, + hm3301_trigger_handler, NULL); + if (ret) + return ret; + + return devm_iio_device_register(&client->dev, indio_dev); +} + +static const struct i2c_device_id hm3301_id[] = { + { "hm3301" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, hm3301_id); + +static const struct of_device_id hm3301_of_match[] = { + { .compatible = "seeed,hm3301" }, + { } +}; +MODULE_DEVICE_TABLE(of, hm3301_of_match); + +static struct i2c_driver hm3301_driver = { + .driver = { + .name = "hm3301", + .of_match_table = hm3301_of_match, + }, + .id_table = hm3301_id, + .probe_new = hm3301_probe, +}; +module_i2c_driver(hm3301_driver); + +MODULE_AUTHOR("Baozhu Zuo "); +MODULE_DESCRIPTION("HM3301 laser dust detection sensor driver"); +MODULE_LICENSE("GPL v2"); + From f3a8e2fe736bc28a1307f7b56cc6c14ef05f389f Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Sun, 3 Jan 2021 17:43:36 -0500 Subject: [PATCH 134/135] Add WPANUSB driver From https://github.com/statropy/wpanusb --- arch/arm/configs/bb.org_defconfig | 1 + drivers/net/ieee802154/Kconfig | 9 +++++++++ drivers/net/ieee802154/Makefile | 1 + 3 files changed, 11 insertions(+) diff --git a/arch/arm/configs/bb.org_defconfig b/arch/arm/configs/bb.org_defconfig index 585d8189ef260..22c3721e7bbf8 100644 --- a/arch/arm/configs/bb.org_defconfig +++ b/arch/arm/configs/bb.org_defconfig @@ -949,6 +949,7 @@ CONFIG_IEEE802154_AT86RF230=m CONFIG_IEEE802154_MRF24J40=m CONFIG_IEEE802154_CC2520=m CONFIG_IEEE802154_ATUSB=m +CONFIG_IEEE802154_WPANUSB=m CONFIG_IEEE802154_ADF7242=m CONFIG_IEEE802154_CA8210=m CONFIG_IEEE802154_MCR20A=m diff --git a/drivers/net/ieee802154/Kconfig b/drivers/net/ieee802154/Kconfig index 8af5b7e9f4eda..05561d224441e 100644 --- a/drivers/net/ieee802154/Kconfig +++ b/drivers/net/ieee802154/Kconfig @@ -73,6 +73,15 @@ config IEEE802154_ATUSB This driver can also be built as a module. To do so say M here. The module will be called 'atusb'. +config IEEE802154_WPANUSB + tristate "WPANUSB driver" + depends on IEEE802154_DRIVERS && MAC802154 && USB + help + Adds support for WPANUSB 802.15.4 adapters. + + This driver should work with at least the following devices: + * BeagleBoard.org BeagleConnect Freedom + config IEEE802154_ADF7242 tristate "ADF7242 transceiver driver" depends on IEEE802154_DRIVERS && MAC802154 diff --git a/drivers/net/ieee802154/Makefile b/drivers/net/ieee802154/Makefile index 0c78b62980606..59442aab94038 100644 --- a/drivers/net/ieee802154/Makefile +++ b/drivers/net/ieee802154/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_IEEE802154_AT86RF230) += at86rf230.o obj-$(CONFIG_IEEE802154_MRF24J40) += mrf24j40.o obj-$(CONFIG_IEEE802154_CC2520) += cc2520.o obj-$(CONFIG_IEEE802154_ATUSB) += atusb.o +obj-$(CONFIG_IEEE802154_WPANUSB) += wpanusb.o obj-$(CONFIG_IEEE802154_ADF7242) += adf7242.o obj-$(CONFIG_IEEE802154_CA8210) += ca8210.o obj-$(CONFIG_IEEE802154_MCR20A) += mcr20a.o From 8b1c65f4af4fe4de845df42eabb70ea11c8f0e98 Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Mon, 11 Jan 2021 22:12:52 -0500 Subject: [PATCH 135/135] Add scd30 driver --- drivers/iio/chemical/Kconfig | 33 ++ drivers/iio/chemical/Makefile | 3 + drivers/iio/chemical/scd30.h | 78 +++ drivers/iio/chemical/scd30_core.c | 766 ++++++++++++++++++++++++++++ drivers/iio/chemical/scd30_i2c.c | 139 +++++ drivers/iio/chemical/scd30_serial.c | 263 ++++++++++ 6 files changed, 1282 insertions(+) create mode 100644 drivers/iio/chemical/scd30.h create mode 100644 drivers/iio/chemical/scd30_core.c create mode 100644 drivers/iio/chemical/scd30_i2c.c create mode 100644 drivers/iio/chemical/scd30_serial.c diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig index 41996e346ad3c..1661364c08c76 100644 --- a/drivers/iio/chemical/Kconfig +++ b/drivers/iio/chemical/Kconfig @@ -82,6 +82,39 @@ config PMS7003 To compile this driver as a module, choose M here: the module will be called pms7003. +config SCD30_CORE + tristate "SCD30 carbon dioxide sensor driver" + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER + help + Say Y here to build support for the Sensirion SCD30 sensor with carbon + dioxide, relative humidity and temperature sensing capabilities. + + To compile this driver as a module, choose M here: the module will + be called scd30_core. + +config SCD30_I2C + tristate "SCD30 carbon dioxide sensor I2C driver" + depends on SCD30_CORE && I2C + select CRC8 + help + Say Y here to build support for the Sensirion SCD30 I2C interface + driver. + + To compile this driver as a module, choose M here: the module will + be called scd30_i2c. + +config SCD30_SERIAL + tristate "SCD30 carbon dioxide sensor serial driver" + depends on SCD30_CORE && SERIAL_DEV_BUS + select CRC16 + help + Say Y here to build support for the Sensirion SCD30 serial interface + driver. + + To compile this driver as a module, choose M here: the module will + be called scd30_serial. + config SENSIRION_SGP30 tristate "Sensirion SGPxx gas sensors" depends on I2C diff --git a/drivers/iio/chemical/Makefile b/drivers/iio/chemical/Makefile index 76efb3b6b180e..5213cca8c697a 100644 --- a/drivers/iio/chemical/Makefile +++ b/drivers/iio/chemical/Makefile @@ -12,6 +12,9 @@ obj-$(CONFIG_CCS811) += ccs811.o obj-$(CONFIG_HM3301) += hm3301.o obj-$(CONFIG_IAQCORE) += ams-iaq-core.o obj-$(CONFIG_PMS7003) += pms7003.o +obj-$(CONFIG_SCD30_CORE) += scd30_core.o +obj-$(CONFIG_SCD30_I2C) += scd30_i2c.o +obj-$(CONFIG_SCD30_SERIAL) += scd30_serial.o obj-$(CONFIG_SENSIRION_SGP30) += sgp30.o obj-$(CONFIG_SPS30) += sps30.o obj-$(CONFIG_VZ89X) += vz89x.o diff --git a/drivers/iio/chemical/scd30.h b/drivers/iio/chemical/scd30.h new file mode 100644 index 0000000000000..f60127bfe0f4f --- /dev/null +++ b/drivers/iio/chemical/scd30.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SCD30_H +#define _SCD30_H + +#include +#include +#include +#include +#include +#include + +struct scd30_state; + +enum scd30_cmd { + /* start continuous measurement with pressure compensation */ + CMD_START_MEAS, + /* stop continuous measurement */ + CMD_STOP_MEAS, + /* set/get measurement interval */ + CMD_MEAS_INTERVAL, + /* check whether new measurement is ready */ + CMD_MEAS_READY, + /* get measurement */ + CMD_READ_MEAS, + /* turn on/off automatic self calibration */ + CMD_ASC, + /* set/get forced recalibration value */ + CMD_FRC, + /* set/get temperature offset */ + CMD_TEMP_OFFSET, + /* get firmware version */ + CMD_FW_VERSION, + /* reset sensor */ + CMD_RESET, + /* + * Command for altitude compensation was omitted intentionally because + * the same can be achieved by means of CMD_START_MEAS which takes + * pressure above the sea level as an argument. + */ +}; + +#define SCD30_MEAS_COUNT 3 + +typedef int (*scd30_command_t)(struct scd30_state *state, enum scd30_cmd cmd, u16 arg, + void *response, int size); + +struct scd30_state { + /* serialize access to the device */ + struct mutex lock; + struct device *dev; + struct regulator *vdd; + struct completion meas_ready; + /* + * priv pointer is solely for serdev driver private data. We keep it + * here because driver_data inside dev has been already used for iio and + * struct serdev_device doesn't have one. + */ + void *priv; + int irq; + /* + * no way to retrieve current ambient pressure compensation value from + * the sensor so keep one around + */ + u16 pressure_comp; + u16 meas_interval; + int meas[SCD30_MEAS_COUNT]; + + scd30_command_t command; +}; + +int scd30_suspend(struct device *dev); +int scd30_resume(struct device *dev); + +static __maybe_unused SIMPLE_DEV_PM_OPS(scd30_pm_ops, scd30_suspend, scd30_resume); + +int scd30_probe(struct device *dev, int irq, const char *name, void *priv, scd30_command_t command); + +#endif diff --git a/drivers/iio/chemical/scd30_core.c b/drivers/iio/chemical/scd30_core.c new file mode 100644 index 0000000000000..4d0d798c7cd34 --- /dev/null +++ b/drivers/iio/chemical/scd30_core.c @@ -0,0 +1,766 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sensirion SCD30 carbon dioxide sensor core driver + * + * Copyright (c) 2020 Tomasz Duszynski + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scd30.h" + +#define SCD30_PRESSURE_COMP_MIN_MBAR 700 +#define SCD30_PRESSURE_COMP_MAX_MBAR 1400 +#define SCD30_PRESSURE_COMP_DEFAULT 1013 +#define SCD30_MEAS_INTERVAL_MIN_S 2 +#define SCD30_MEAS_INTERVAL_MAX_S 1800 +#define SCD30_MEAS_INTERVAL_DEFAULT SCD30_MEAS_INTERVAL_MIN_S +#define SCD30_FRC_MIN_PPM 400 +#define SCD30_FRC_MAX_PPM 2000 +#define SCD30_TEMP_OFFSET_MAX 655360 +#define SCD30_EXTRA_TIMEOUT_PER_S 250 + +enum { + SCD30_CONC, + SCD30_TEMP, + SCD30_HR, +}; + +static int scd30_command_write(struct scd30_state *state, enum scd30_cmd cmd, u16 arg) +{ + return state->command(state, cmd, arg, NULL, 0); +} + +static int scd30_command_read(struct scd30_state *state, enum scd30_cmd cmd, u16 *val) +{ + __be16 tmp; + int ret; + + ret = state->command(state, cmd, 0, &tmp, sizeof(tmp)); + *val = be16_to_cpup(&tmp); + + return ret; +} + +static int scd30_reset(struct scd30_state *state) +{ + int ret; + u16 val; + + ret = scd30_command_write(state, CMD_RESET, 0); + if (ret) + return ret; + + /* sensor boots up within 2 secs */ + msleep(2000); + /* + * Power-on-reset causes sensor to produce some glitch on i2c bus and + * some controllers end up in error state. Try to recover by placing + * any data on the bus. + */ + scd30_command_read(state, CMD_MEAS_READY, &val); + + return 0; +} + +/* simplified float to fixed point conversion with a scaling factor of 0.01 */ +static int scd30_float_to_fp(int float32) +{ + int fraction, shift, + mantissa = float32 & GENMASK(22, 0), + sign = (float32 & BIT(31)) ? -1 : 1, + exp = (float32 & ~BIT(31)) >> 23; + + /* special case 0 */ + if (!exp && !mantissa) + return 0; + + exp -= 127; + if (exp < 0) { + exp = -exp; + /* return values ranging from 1 to 99 */ + return sign * ((((BIT(23) + mantissa) * 100) >> 23) >> exp); + } + + /* return values starting at 100 */ + shift = 23 - exp; + float32 = BIT(exp) + (mantissa >> shift); + fraction = mantissa & GENMASK(shift - 1, 0); + + return sign * (float32 * 100 + ((fraction * 100) >> shift)); +} + +static int scd30_read_meas(struct scd30_state *state) +{ + int i, ret; + + ret = state->command(state, CMD_READ_MEAS, 0, state->meas, sizeof(state->meas)); + if (ret) + return ret; + + be32_to_cpu_array(state->meas, (__be32 *)state->meas, ARRAY_SIZE(state->meas)); + + for (i = 0; i < ARRAY_SIZE(state->meas); i++) + state->meas[i] = scd30_float_to_fp(state->meas[i]); + + /* + * co2 is left unprocessed while temperature and humidity are scaled + * to milli deg C and milli percent respectively. + */ + state->meas[SCD30_TEMP] *= 10; + state->meas[SCD30_HR] *= 10; + + return 0; +} + +static int scd30_wait_meas_irq(struct scd30_state *state) +{ + int ret, timeout; + + reinit_completion(&state->meas_ready); + enable_irq(state->irq); + timeout = msecs_to_jiffies(state->meas_interval * (1000 + SCD30_EXTRA_TIMEOUT_PER_S)); + ret = wait_for_completion_interruptible_timeout(&state->meas_ready, timeout); + if (ret > 0) + ret = 0; + else if (!ret) + ret = -ETIMEDOUT; + + disable_irq(state->irq); + + return ret; +} + +static int scd30_wait_meas_poll(struct scd30_state *state) +{ + int timeout = state->meas_interval * SCD30_EXTRA_TIMEOUT_PER_S, tries = 5; + + do { + int ret; + u16 val; + + ret = scd30_command_read(state, CMD_MEAS_READY, &val); + if (ret) + return -EIO; + + /* new measurement available */ + if (val) + break; + + msleep_interruptible(timeout); + } while (--tries); + + return tries ? 0 : -ETIMEDOUT; +} + +static int scd30_read_poll(struct scd30_state *state) +{ + int ret; + + ret = scd30_wait_meas_poll(state); + if (ret) + return ret; + + return scd30_read_meas(state); +} + +static int scd30_read(struct scd30_state *state) +{ + if (state->irq > 0) + return scd30_wait_meas_irq(state); + + return scd30_read_poll(state); +} + +static int scd30_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct scd30_state *state = iio_priv(indio_dev); + int ret = -EINVAL; + u16 tmp; + + mutex_lock(&state->lock); + switch (mask) { + case IIO_CHAN_INFO_RAW: + case IIO_CHAN_INFO_PROCESSED: + if (chan->output) { + *val = state->pressure_comp; + ret = IIO_VAL_INT; + break; + } + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + break; + + ret = scd30_read(state); + if (ret) { + iio_device_release_direct_mode(indio_dev); + break; + } + + *val = state->meas[chan->address]; + iio_device_release_direct_mode(indio_dev); + ret = IIO_VAL_INT; + break; + case IIO_CHAN_INFO_SCALE: + *val = 0; + *val2 = 1; + ret = IIO_VAL_INT_PLUS_MICRO; + break; + case IIO_CHAN_INFO_SAMP_FREQ: + ret = scd30_command_read(state, CMD_MEAS_INTERVAL, &tmp); + if (ret) + break; + + *val = 0; + *val2 = 1000000000 / tmp; + ret = IIO_VAL_INT_PLUS_NANO; + break; + case IIO_CHAN_INFO_CALIBBIAS: + ret = scd30_command_read(state, CMD_TEMP_OFFSET, &tmp); + if (ret) + break; + + *val = tmp; + ret = IIO_VAL_INT; + break; + } + mutex_unlock(&state->lock); + + return ret; +} + +static int scd30_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + struct scd30_state *state = iio_priv(indio_dev); + int ret = -EINVAL; + + mutex_lock(&state->lock); + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + if (val) + break; + + val = 1000000000 / val2; + if (val < SCD30_MEAS_INTERVAL_MIN_S || val > SCD30_MEAS_INTERVAL_MAX_S) + break; + + ret = scd30_command_write(state, CMD_MEAS_INTERVAL, val); + if (ret) + break; + + state->meas_interval = val; + break; + case IIO_CHAN_INFO_RAW: + switch (chan->type) { + case IIO_PRESSURE: + if (val < SCD30_PRESSURE_COMP_MIN_MBAR || + val > SCD30_PRESSURE_COMP_MAX_MBAR) + break; + + ret = scd30_command_write(state, CMD_START_MEAS, val); + if (ret) + break; + + state->pressure_comp = val; + break; + default: + break; + } + break; + case IIO_CHAN_INFO_CALIBBIAS: + if (val < 0 || val > SCD30_TEMP_OFFSET_MAX) + break; + /* + * Manufacturer does not explicitly specify min/max sensible + * values hence check is omitted for simplicity. + */ + ret = scd30_command_write(state, CMD_TEMP_OFFSET / 10, val); + } + mutex_unlock(&state->lock); + + return ret; +} + +static int scd30_write_raw_get_fmt(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + return IIO_VAL_INT_PLUS_NANO; + case IIO_CHAN_INFO_RAW: + case IIO_CHAN_INFO_CALIBBIAS: + return IIO_VAL_INT; + } + + return -EINVAL; +} + +static const int scd30_pressure_raw_available[] = { + SCD30_PRESSURE_COMP_MIN_MBAR, 1, SCD30_PRESSURE_COMP_MAX_MBAR, +}; + +static const int scd30_temp_calibbias_available[] = { + 0, 10, SCD30_TEMP_OFFSET_MAX, +}; + +static int scd30_read_avail(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_RAW: + *vals = scd30_pressure_raw_available; + *type = IIO_VAL_INT; + + return IIO_AVAIL_RANGE; + case IIO_CHAN_INFO_CALIBBIAS: + *vals = scd30_temp_calibbias_available; + *type = IIO_VAL_INT; + + return IIO_AVAIL_RANGE; + } + + return -EINVAL; +} + +static ssize_t sampling_frequency_available_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int i = SCD30_MEAS_INTERVAL_MIN_S; + ssize_t len = 0; + + do { + len += scnprintf(buf + len, PAGE_SIZE - len, "0.%09u ", 1000000000 / i); + /* + * Not all values fit PAGE_SIZE buffer hence print every 6th + * (each frequency differs by 6s in time domain from the + * adjacent). Unlisted but valid ones are still accepted. + */ + i += 6; + } while (i <= SCD30_MEAS_INTERVAL_MAX_S); + + buf[len - 1] = '\n'; + + return len; +} + +static ssize_t calibration_auto_enable_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct scd30_state *state = iio_priv(indio_dev); + int ret; + u16 val; + + mutex_lock(&state->lock); + ret = scd30_command_read(state, CMD_ASC, &val); + mutex_unlock(&state->lock); + + return ret ?: sprintf(buf, "%d\n", val); +} + +static ssize_t calibration_auto_enable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct scd30_state *state = iio_priv(indio_dev); + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret) + return ret; + + mutex_lock(&state->lock); + ret = scd30_command_write(state, CMD_ASC, val); + mutex_unlock(&state->lock); + + return ret ?: len; +} + +static ssize_t calibration_forced_value_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct scd30_state *state = iio_priv(indio_dev); + int ret; + u16 val; + + mutex_lock(&state->lock); + ret = scd30_command_read(state, CMD_FRC, &val); + mutex_unlock(&state->lock); + + return ret ?: sprintf(buf, "%d\n", val); +} + +static ssize_t calibration_forced_value_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct scd30_state *state = iio_priv(indio_dev); + int ret; + u16 val; + + ret = kstrtou16(buf, 0, &val); + if (ret) + return ret; + + if (val < SCD30_FRC_MIN_PPM || val > SCD30_FRC_MAX_PPM) + return -EINVAL; + + mutex_lock(&state->lock); + ret = scd30_command_write(state, CMD_FRC, val); + mutex_unlock(&state->lock); + + return ret ?: len; +} + +static IIO_DEVICE_ATTR_RO(sampling_frequency_available, 0); +static IIO_DEVICE_ATTR_RW(calibration_auto_enable, 0); +static IIO_DEVICE_ATTR_RW(calibration_forced_value, 0); + +static struct attribute *scd30_attrs[] = { + &iio_dev_attr_sampling_frequency_available.dev_attr.attr, + &iio_dev_attr_calibration_auto_enable.dev_attr.attr, + &iio_dev_attr_calibration_forced_value.dev_attr.attr, + NULL +}; + +static const struct attribute_group scd30_attr_group = { + .attrs = scd30_attrs, +}; + +static const struct iio_info scd30_info = { + .attrs = &scd30_attr_group, + .read_raw = scd30_read_raw, + .write_raw = scd30_write_raw, + .write_raw_get_fmt = scd30_write_raw_get_fmt, + .read_avail = scd30_read_avail, +}; + +#define SCD30_CHAN_SCAN_TYPE(_sign, _realbits) .scan_type = { \ + .sign = _sign, \ + .realbits = _realbits, \ + .storagebits = 32, \ + .endianness = IIO_CPU, \ +} + +static const struct iio_chan_spec scd30_channels[] = { + { + /* + * this channel is special in a sense we are pretending that + * sensor is able to change measurement chamber pressure but in + * fact we're just setting pressure compensation value + */ + .type = IIO_PRESSURE, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_separate_available = BIT(IIO_CHAN_INFO_RAW), + .output = 1, + .scan_index = -1, + }, + { + .type = IIO_CONCENTRATION, + .channel2 = IIO_MOD_CO2, + .address = SCD30_CONC, + .scan_index = SCD30_CONC, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .modified = 1, + + SCD30_CHAN_SCAN_TYPE('u', 20), + }, + { + .type = IIO_TEMP, + .address = SCD30_TEMP, + .scan_index = SCD30_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_CALIBBIAS), + .info_mask_separate_available = BIT(IIO_CHAN_INFO_CALIBBIAS), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), + + SCD30_CHAN_SCAN_TYPE('s', 18), + }, + { + .type = IIO_HUMIDITYRELATIVE, + .address = SCD30_HR, + .scan_index = SCD30_HR, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), + + SCD30_CHAN_SCAN_TYPE('u', 17), + }, + IIO_CHAN_SOFT_TIMESTAMP(3), +}; + +int __maybe_unused scd30_suspend(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct scd30_state *state = iio_priv(indio_dev); + int ret; + + ret = scd30_command_write(state, CMD_STOP_MEAS, 0); + if (ret) + return ret; + + return regulator_disable(state->vdd); +} +EXPORT_SYMBOL(scd30_suspend); + +int __maybe_unused scd30_resume(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct scd30_state *state = iio_priv(indio_dev); + int ret; + + ret = regulator_enable(state->vdd); + if (ret) + return ret; + + return scd30_command_write(state, CMD_START_MEAS, state->pressure_comp); +} +EXPORT_SYMBOL(scd30_resume); + +static void scd30_stop_meas(void *data) +{ + struct scd30_state *state = data; + + scd30_command_write(state, CMD_STOP_MEAS, 0); +} + +static void scd30_disable_regulator(void *data) +{ + struct scd30_state *state = data; + + regulator_disable(state->vdd); +} + +static irqreturn_t scd30_irq_handler(int irq, void *priv) +{ + struct iio_dev *indio_dev = priv; + + if (iio_buffer_enabled(indio_dev)) { + iio_trigger_poll(indio_dev->trig); + + return IRQ_HANDLED; + } + + return IRQ_WAKE_THREAD; +} + +static irqreturn_t scd30_irq_thread_handler(int irq, void *priv) +{ + struct iio_dev *indio_dev = priv; + struct scd30_state *state = iio_priv(indio_dev); + int ret; + + ret = scd30_read_meas(state); + if (ret) + goto out; + + complete_all(&state->meas_ready); +out: + return IRQ_HANDLED; +} + +static irqreturn_t scd30_trigger_handler(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct iio_dev *indio_dev = pf->indio_dev; + struct scd30_state *state = iio_priv(indio_dev); + struct { + int data[SCD30_MEAS_COUNT]; + s64 ts __aligned(8); + } scan; + int ret; + + mutex_lock(&state->lock); + if (!iio_trigger_using_own(indio_dev)) + ret = scd30_read_poll(state); + else + ret = scd30_read_meas(state); + memset(&scan, 0, sizeof(scan)); + memcpy(scan.data, state->meas, sizeof(state->meas)); + mutex_unlock(&state->lock); + if (ret) + goto out; + + iio_push_to_buffers_with_timestamp(indio_dev, &scan, iio_get_time_ns(indio_dev)); +out: + iio_trigger_notify_done(indio_dev->trig); + return IRQ_HANDLED; +} + +static int scd30_set_trigger_state(struct iio_trigger *trig, bool state) +{ + struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig); + struct scd30_state *st = iio_priv(indio_dev); + + if (state) + enable_irq(st->irq); + else + disable_irq(st->irq); + + return 0; +} + +static const struct iio_trigger_ops scd30_trigger_ops = { + .set_trigger_state = scd30_set_trigger_state, + .validate_device = iio_trigger_validate_own_device, +}; + +static int scd30_setup_trigger(struct iio_dev *indio_dev) +{ + struct scd30_state *state = iio_priv(indio_dev); + struct device *dev = indio_dev->dev.parent; + struct iio_trigger *trig; + int ret; + + trig = devm_iio_trigger_alloc(dev, "%s-dev%d", indio_dev->name, indio_dev->id); + if (!trig) { + dev_err(dev, "failed to allocate trigger\n"); + return -ENOMEM; + } + + trig->dev.parent = dev; + trig->ops = &scd30_trigger_ops; + iio_trigger_set_drvdata(trig, indio_dev); + + ret = devm_iio_trigger_register(dev, trig); + if (ret) + return ret; + + indio_dev->trig = iio_trigger_get(trig); + + ret = devm_request_threaded_irq(dev, state->irq, scd30_irq_handler, + scd30_irq_thread_handler, IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + indio_dev->name, indio_dev); + if (ret) + dev_err(dev, "failed to request irq\n"); + + /* + * Interrupt is enabled just before taking a fresh measurement + * and disabled afterwards. This means we need to disable it here + * to keep calls to enable/disable balanced. + */ + disable_irq(state->irq); + + return ret; +} + +int scd30_probe(struct device *dev, int irq, const char *name, void *priv, + scd30_command_t command) +{ + static const unsigned long scd30_scan_masks[] = { 0x07, 0x00 }; + struct scd30_state *state; + struct iio_dev *indio_dev; + int ret; + u16 val; + + indio_dev = devm_iio_device_alloc(dev, sizeof(*state)); + if (!indio_dev) + return -ENOMEM; + + state = iio_priv(indio_dev); + state->dev = dev; + state->priv = priv; + state->irq = irq; + state->pressure_comp = SCD30_PRESSURE_COMP_DEFAULT; + state->meas_interval = SCD30_MEAS_INTERVAL_DEFAULT; + state->command = command; + mutex_init(&state->lock); + init_completion(&state->meas_ready); + + dev_set_drvdata(dev, indio_dev); + + indio_dev->info = &scd30_info; + indio_dev->name = name; + indio_dev->channels = scd30_channels; + indio_dev->num_channels = ARRAY_SIZE(scd30_channels); + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->available_scan_masks = scd30_scan_masks; + + state->vdd = devm_regulator_get(dev, "vdd"); + if (IS_ERR(state->vdd)) + return dev_err_probe(dev, PTR_ERR(state->vdd), "failed to get regulator\n"); + + ret = regulator_enable(state->vdd); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, scd30_disable_regulator, state); + if (ret) + return ret; + + ret = scd30_reset(state); + if (ret) { + dev_err(dev, "failed to reset device: %d\n", ret); + return ret; + } + + if (state->irq > 0) { + ret = scd30_setup_trigger(indio_dev); + if (ret) { + dev_err(dev, "failed to setup trigger: %d\n", ret); + return ret; + } + } + + ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, scd30_trigger_handler, NULL); + if (ret) + return ret; + + ret = scd30_command_read(state, CMD_FW_VERSION, &val); + if (ret) { + dev_err(dev, "failed to read firmware version: %d\n", ret); + return ret; + } + dev_info(dev, "firmware version: %d.%d\n", val >> 8, (char)val); + + ret = scd30_command_write(state, CMD_MEAS_INTERVAL, state->meas_interval); + if (ret) { + dev_err(dev, "failed to set measurement interval: %d\n", ret); + return ret; + } + + ret = scd30_command_write(state, CMD_START_MEAS, state->pressure_comp); + if (ret) { + dev_err(dev, "failed to start measurement: %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(dev, scd30_stop_meas, state); + if (ret) + return ret; + + return devm_iio_device_register(dev, indio_dev); +} +EXPORT_SYMBOL(scd30_probe); + +MODULE_AUTHOR("Tomasz Duszynski "); +MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor core driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/chemical/scd30_i2c.c b/drivers/iio/chemical/scd30_i2c.c new file mode 100644 index 0000000000000..875892a070eec --- /dev/null +++ b/drivers/iio/chemical/scd30_i2c.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sensirion SCD30 carbon dioxide sensor i2c driver + * + * Copyright (c) 2020 Tomasz Duszynski + * + * I2C slave address: 0x61 + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scd30.h" + +#define SCD30_I2C_MAX_BUF_SIZE 18 +#define SCD30_I2C_CRC8_POLYNOMIAL 0x31 + +static u16 scd30_i2c_cmd_lookup_tbl[] = { + [CMD_START_MEAS] = 0x0010, + [CMD_STOP_MEAS] = 0x0104, + [CMD_MEAS_INTERVAL] = 0x4600, + [CMD_MEAS_READY] = 0x0202, + [CMD_READ_MEAS] = 0x0300, + [CMD_ASC] = 0x5306, + [CMD_FRC] = 0x5204, + [CMD_TEMP_OFFSET] = 0x5403, + [CMD_FW_VERSION] = 0xd100, + [CMD_RESET] = 0xd304, +}; + +DECLARE_CRC8_TABLE(scd30_i2c_crc8_tbl); + +static int scd30_i2c_xfer(struct scd30_state *state, char *txbuf, int txsize, + char *rxbuf, int rxsize) +{ + struct i2c_client *client = to_i2c_client(state->dev); + int ret; + + /* + * repeated start is not supported hence instead of sending two i2c + * messages in a row we send one by one + */ + ret = i2c_master_send(client, txbuf, txsize); + if (ret < 0) + return ret; + if (ret != txsize) + return -EIO; + + if (!rxbuf) + return 0; + + ret = i2c_master_recv(client, rxbuf, rxsize); + if (ret < 0) + return ret; + if (ret != rxsize) + return -EIO; + + return 0; +} + +static int scd30_i2c_command(struct scd30_state *state, enum scd30_cmd cmd, u16 arg, + void *response, int size) +{ + char buf[SCD30_I2C_MAX_BUF_SIZE]; + char *rsp = response; + int i, ret; + char crc; + + put_unaligned_be16(scd30_i2c_cmd_lookup_tbl[cmd], buf); + i = 2; + + if (rsp) { + /* each two bytes are followed by a crc8 */ + size += size / 2; + } else { + put_unaligned_be16(arg, buf + i); + crc = crc8(scd30_i2c_crc8_tbl, buf + i, 2, CRC8_INIT_VALUE); + i += 2; + buf[i] = crc; + i += 1; + + /* commands below don't take an argument */ + if ((cmd == CMD_STOP_MEAS) || (cmd == CMD_RESET)) + i -= 3; + } + + ret = scd30_i2c_xfer(state, buf, i, buf, size); + if (ret) + return ret; + + /* validate received data and strip off crc bytes */ + for (i = 0; i < size; i += 3) { + crc = crc8(scd30_i2c_crc8_tbl, buf + i, 2, CRC8_INIT_VALUE); + if (crc != buf[i + 2]) { + dev_err(state->dev, "data integrity check failed\n"); + return -EIO; + } + + *rsp++ = buf[i]; + *rsp++ = buf[i + 1]; + } + + return 0; +} + +static int scd30_i2c_probe(struct i2c_client *client) +{ + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -EOPNOTSUPP; + + crc8_populate_msb(scd30_i2c_crc8_tbl, SCD30_I2C_CRC8_POLYNOMIAL); + + return scd30_probe(&client->dev, client->irq, client->name, NULL, scd30_i2c_command); +} + +static const struct of_device_id scd30_i2c_of_match[] = { + { .compatible = "sensirion,scd30" }, + { } +}; +MODULE_DEVICE_TABLE(of, scd30_i2c_of_match); + +static struct i2c_driver scd30_i2c_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = scd30_i2c_of_match, + .pm = &scd30_pm_ops, + }, + .probe_new = scd30_i2c_probe, +}; +module_i2c_driver(scd30_i2c_driver); + +MODULE_AUTHOR("Tomasz Duszynski "); +MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor i2c driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/chemical/scd30_serial.c b/drivers/iio/chemical/scd30_serial.c new file mode 100644 index 0000000000000..06f85eb1a4dd8 --- /dev/null +++ b/drivers/iio/chemical/scd30_serial.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sensirion SCD30 carbon dioxide sensor serial driver + * + * Copyright (c) 2020 Tomasz Duszynski + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scd30.h" + +#define SCD30_SERDEV_ADDR 0x61 +#define SCD30_SERDEV_WRITE 0x06 +#define SCD30_SERDEV_READ 0x03 +#define SCD30_SERDEV_MAX_BUF_SIZE 17 +#define SCD30_SERDEV_RX_HEADER_SIZE 3 +#define SCD30_SERDEV_CRC_SIZE 2 +#define SCD30_SERDEV_TIMEOUT msecs_to_jiffies(200) + +struct scd30_serdev_priv { + struct completion meas_ready; + char *buf; + int num_expected; + int num; +}; + +static u16 scd30_serdev_cmd_lookup_tbl[] = { + [CMD_START_MEAS] = 0x0036, + [CMD_STOP_MEAS] = 0x0037, + [CMD_MEAS_INTERVAL] = 0x0025, + [CMD_MEAS_READY] = 0x0027, + [CMD_READ_MEAS] = 0x0028, + [CMD_ASC] = 0x003a, + [CMD_FRC] = 0x0039, + [CMD_TEMP_OFFSET] = 0x003b, + [CMD_FW_VERSION] = 0x0020, + [CMD_RESET] = 0x0034, +}; + +static u16 scd30_serdev_calc_crc(const char *buf, int size) +{ + return crc16(0xffff, buf, size); +} + +static int scd30_serdev_xfer(struct scd30_state *state, char *txbuf, int txsize, + char *rxbuf, int rxsize) +{ + struct serdev_device *serdev = to_serdev_device(state->dev); + struct scd30_serdev_priv *priv = state->priv; + int ret; + + priv->buf = rxbuf; + priv->num_expected = rxsize; + priv->num = 0; + + ret = serdev_device_write(serdev, txbuf, txsize, SCD30_SERDEV_TIMEOUT); + if (ret < 0) + return ret; + if (ret != txsize) + return -EIO; + + ret = wait_for_completion_interruptible_timeout(&priv->meas_ready, SCD30_SERDEV_TIMEOUT); + if (ret < 0) + return ret; + if (!ret) + return -ETIMEDOUT; + + return 0; +} + +static int scd30_serdev_command(struct scd30_state *state, enum scd30_cmd cmd, u16 arg, + void *response, int size) +{ + /* + * Communication over serial line is based on modbus protocol (or rather + * its variation called modbus over serial to be precise). Upon + * receiving a request device should reply with response. + * + * Frame below represents a request message. Each field takes + * exactly one byte. + * + * +------+------+-----+-----+-------+-------+-----+-----+ + * | dev | op | reg | reg | byte1 | byte0 | crc | crc | + * | addr | code | msb | lsb | | | lsb | msb | + * +------+------+-----+-----+-------+-------+-----+-----+ + * + * The message device replies with depends on the 'op code' field from + * the request. In case it was set to SCD30_SERDEV_WRITE sensor should + * reply with unchanged request. Otherwise 'op code' was set to + * SCD30_SERDEV_READ and response looks like the one below. As with + * request, each field takes one byte. + * + * +------+------+--------+-------+-----+-------+-----+-----+ + * | dev | op | num of | byte0 | ... | byteN | crc | crc | + * | addr | code | bytes | | | | lsb | msb | + * +------+------+--------+-------+-----+-------+-----+-----+ + */ + char txbuf[SCD30_SERDEV_MAX_BUF_SIZE] = { SCD30_SERDEV_ADDR }, + rxbuf[SCD30_SERDEV_MAX_BUF_SIZE]; + int ret, rxsize, txsize = 2; + char *rsp = response; + u16 crc; + + put_unaligned_be16(scd30_serdev_cmd_lookup_tbl[cmd], txbuf + txsize); + txsize += 2; + + if (rsp) { + txbuf[1] = SCD30_SERDEV_READ; + if (cmd == CMD_READ_MEAS) + /* number of u16 words to read */ + put_unaligned_be16(size / 2, txbuf + txsize); + else + put_unaligned_be16(0x0001, txbuf + txsize); + txsize += 2; + crc = scd30_serdev_calc_crc(txbuf, txsize); + put_unaligned_le16(crc, txbuf + txsize); + txsize += 2; + rxsize = SCD30_SERDEV_RX_HEADER_SIZE + size + SCD30_SERDEV_CRC_SIZE; + } else { + if ((cmd == CMD_STOP_MEAS) || (cmd == CMD_RESET)) + arg = 0x0001; + + txbuf[1] = SCD30_SERDEV_WRITE; + put_unaligned_be16(arg, txbuf + txsize); + txsize += 2; + crc = scd30_serdev_calc_crc(txbuf, txsize); + put_unaligned_le16(crc, txbuf + txsize); + txsize += 2; + rxsize = txsize; + } + + ret = scd30_serdev_xfer(state, txbuf, txsize, rxbuf, rxsize); + if (ret) + return ret; + + switch (txbuf[1]) { + case SCD30_SERDEV_WRITE: + if (memcmp(txbuf, rxbuf, txsize)) { + dev_err(state->dev, "wrong message received\n"); + return -EIO; + } + break; + case SCD30_SERDEV_READ: + if (rxbuf[2] != (rxsize - SCD30_SERDEV_RX_HEADER_SIZE - SCD30_SERDEV_CRC_SIZE)) { + dev_err(state->dev, "received data size does not match header\n"); + return -EIO; + } + + rxsize -= SCD30_SERDEV_CRC_SIZE; + crc = get_unaligned_le16(rxbuf + rxsize); + if (crc != scd30_serdev_calc_crc(rxbuf, rxsize)) { + dev_err(state->dev, "data integrity check failed\n"); + return -EIO; + } + + rxsize -= SCD30_SERDEV_RX_HEADER_SIZE; + memcpy(rsp, rxbuf + SCD30_SERDEV_RX_HEADER_SIZE, rxsize); + break; + default: + dev_err(state->dev, "received unknown op code\n"); + return -EIO; + } + + return 0; +} + +static int scd30_serdev_receive_buf(struct serdev_device *serdev, + const unsigned char *buf, size_t size) +{ + struct iio_dev *indio_dev = dev_get_drvdata(&serdev->dev); + struct scd30_serdev_priv *priv; + struct scd30_state *state; + int num; + + if (!indio_dev) + return 0; + + state = iio_priv(indio_dev); + priv = state->priv; + + /* just in case sensor puts some unexpected bytes on the bus */ + if (!priv->buf) + return 0; + + if (priv->num + size >= priv->num_expected) + num = priv->num_expected - priv->num; + else + num = size; + + memcpy(priv->buf + priv->num, buf, num); + priv->num += num; + + if (priv->num == priv->num_expected) { + priv->buf = NULL; + complete(&priv->meas_ready); + } + + return num; +} + +static const struct serdev_device_ops scd30_serdev_ops = { + .receive_buf = scd30_serdev_receive_buf, + .write_wakeup = serdev_device_write_wakeup, +}; + +static int scd30_serdev_probe(struct serdev_device *serdev) +{ + struct device *dev = &serdev->dev; + struct scd30_serdev_priv *priv; + int irq, ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + init_completion(&priv->meas_ready); + serdev_device_set_client_ops(serdev, &scd30_serdev_ops); + + ret = devm_serdev_device_open(dev, serdev); + if (ret) + return ret; + + serdev_device_set_baudrate(serdev, 19200); + serdev_device_set_flow_control(serdev, false); + + ret = serdev_device_set_parity(serdev, SERDEV_PARITY_NONE); + if (ret) + return ret; + + irq = fwnode_irq_get(dev_fwnode(dev), 0); + + return scd30_probe(dev, irq, KBUILD_MODNAME, priv, scd30_serdev_command); +} + +static const struct of_device_id scd30_serdev_of_match[] = { + { .compatible = "sensirion,scd30" }, + { } +}; +MODULE_DEVICE_TABLE(of, scd30_serdev_of_match); + +static struct serdev_device_driver scd30_serdev_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = scd30_serdev_of_match, + .pm = &scd30_pm_ops, + }, + .probe = scd30_serdev_probe, +}; +module_serdev_device_driver(scd30_serdev_driver); + +MODULE_AUTHOR("Tomasz Duszynski "); +MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor serial driver"); +MODULE_LICENSE("GPL v2");