Release 4.11 net/ipv6/esp6.c

/*
 * Copyright (C)2002 USAGI/WIDE Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <http://www.gnu.org/licenses/>.
 *
 * Authors
 *
 *      Mitsuru KANDA @USAGI       : IPv6 Support
 *      Kazunori MIYAZAWA @USAGI   :
 *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 *
 *      This file is derived from net/ipv4/esp.c
 */


#define pr_fmt(fmt) "IPv6: " fmt

#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <linux/err.h>
#include <linux/module.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/esp.h>
#include <linux/scatterlist.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/ip6_route.h>
#include <net/icmp.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <linux/icmpv6.h>

#include <linux/highmem.h>


struct esp_skb_cb {
	
struct xfrm_skb_cb xfrm;
	
void *tmp;
};


#define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))

static u32 esp6_get_mtu(struct xfrm_state *x, int mtu);

/*
 * Allocate an AEAD request structure with extra space for SG and IV.
 *
 * For alignment considerations the upper 32 bits of the sequence number are
 * placed at the front, if present. Followed by the IV, the request and finally
 * the SG list.
 *
 * TODO: Use spare space in skb for this where possible.
 */


static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen)
{
	unsigned int len;

	len = seqihlen;

	len += crypto_aead_ivsize(aead);

	if (len) {
		len += crypto_aead_alignmask(aead) &
		       ~(crypto_tfm_ctx_alignment() - 1);
		len = ALIGN(len, crypto_tfm_ctx_alignment());
	}

	len += sizeof(struct aead_request) + crypto_aead_reqsize(aead);
	len = ALIGN(len, __alignof__(struct scatterlist));

	len += sizeof(struct scatterlist) * nfrags;

	return kmalloc(len, GFP_ATOMIC);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
84
77.78%
4
66.67%
Kazunori Miyazawa
17
15.74%
1
16.67%
Steffen Klassert
7
6.48%
1
16.67%
Total
108
100.00%
6
100.00%



static inline __be32 *esp_tmp_seqhi(void *tmp)
{
	return PTR_ALIGN((__be32 *)tmp, __alignof__(__be32));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
27
100.00%
1
100.00%
Total
27
100.00%
1
100.00%



static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen)
{
	return crypto_aead_ivsize(aead) ?
	       PTR_ALIGN((u8 *)tmp + seqhilen,
			 crypto_aead_alignmask(aead) + 1) : tmp + seqhilen;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
39
81.25%
1
33.33%
Steffen Klassert
7
14.58%
1
33.33%
Arnaldo Carvalho de Melo
2
4.17%
1
33.33%
Total
48
100.00%
3
100.00%



static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv)
{
	struct aead_request *req;

	req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
				crypto_tfm_ctx_alignment());
	aead_request_set_tfm(req, aead);
	return req;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
52
100.00%
1
100.00%
Total
52
100.00%
1
100.00%



static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
					     struct aead_request *req)
{
	return (void *)ALIGN((unsigned long)(req + 1) +
			     crypto_aead_reqsize(aead),
			     __alignof__(struct scatterlist));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
48
100.00%
1
100.00%
Total
48
100.00%
1
100.00%



static void esp_ssg_unref(struct xfrm_state *x, void *tmp)
{
	__be32 *seqhi;
	struct crypto_aead *aead = x->data;
	int seqhilen = 0;
	u8 *iv;
	struct aead_request *req;
	struct scatterlist *sg;

	if (x->props.flags & XFRM_STATE_ESN)
		seqhilen += sizeof(__be32);

	seqhi = esp_tmp_seqhi(tmp);
	iv = esp_tmp_iv(aead, tmp, seqhilen);
	req = esp_tmp_req(aead, iv);

	/* Unref skb_frag_pages in the src scatterlist if necessary.
         * Skip the first sg which comes from skb->data.
         */
	if (req->src != req->dst)
		for (sg = sg_next(req->src); sg; sg = sg_next(sg))
			put_page(sg_page(sg));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
130
100.00%
1
100.00%
Total
130
100.00%
1
100.00%



static void esp_output_done(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;
	void *tmp;
	struct dst_entry *dst = skb_dst(skb);
	struct xfrm_state *x = dst->xfrm;

	tmp = ESP_SKB_CB(skb)->tmp;
	esp_ssg_unref(x, tmp);
	kfree(tmp);
	xfrm_output_resume(skb, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
39
52.70%
2
66.67%
Steffen Klassert
35
47.30%
1
33.33%
Total
74
100.00%
3
100.00%

/* Move ESP header back into place. */


static void esp_restore_header(struct sk_buff *skb, unsigned int offset)
{
	struct ip_esp_hdr *esph = (void *)(skb->data + offset);
	void *tmp = ESP_SKB_CB(skb)->tmp;
	__be32 *seqhi = esp_tmp_seqhi(tmp);

	esph->seq_no = esph->spi;
	esph->spi = *seqhi;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
67
100.00%
2
100.00%
Total
67
100.00%
2
100.00%



static void esp_output_restore_header(struct sk_buff *skb)
{
	esp_restore_header(skb, skb_transport_offset(skb) - sizeof(__be32));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
26
100.00%
2
100.00%
Total
26
100.00%
2
100.00%



static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb,
					     struct ip_esp_hdr *esph,
					     __be32 *seqhi)
{
	struct xfrm_state *x = skb_dst(skb)->xfrm;

	/* For ESN we move the header forward by 4 bytes to
         * accomodate the high bits.  We will move it back after
         * encryption.
         */
	if ((x->props.flags & XFRM_STATE_ESN)) {
		esph = (void *)(skb_transport_header(skb) - sizeof(__be32));
		*seqhi = esph->spi;
		esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.hi);
	}

	esph->spi = x->id.spi;

	return esph;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
85
80.95%
1
50.00%
Herbert Xu
20
19.05%
1
50.00%
Total
105
100.00%
2
100.00%



static void esp_output_done_esn(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	esp_output_restore_header(skb);
	esp_output_done(base, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
35
100.00%
1
100.00%
Total
35
100.00%
1
100.00%



static void esp_output_fill_trailer(u8 *tail, int tfclen, int plen, __u8 proto)
{
	/* Fill padding... */
	if (tfclen) {
		memset(tail, 0, tfclen);
		tail += tfclen;
	}
	do {
		int i;
		for (i = 0; i < plen - 2; i++)
			tail[i] = i + 1;
	} while (0);
	tail[plen - 2] = plen - 2;
	tail[plen - 1] = proto;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
94
100.00%
1
100.00%
Total
94
100.00%
1
100.00%



static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
{
	int err;
	struct ip_esp_hdr *esph;
	struct crypto_aead *aead;
	struct aead_request *req;
	struct scatterlist *sg, *dsg;
	struct sk_buff *trailer;
	struct page *page;
	void *tmp;
	int blksize;
	int clen;
	int alen;
	int plen;
	int ivlen;
	int tfclen;
	int nfrags;
	int assoclen;
	int seqhilen;
	int tailen;
	u8 *iv;
	u8 *tail;
	u8 *vaddr;
	__be32 *seqhi;
	__be64 seqno;
	__u8 proto = *skb_mac_header(skb);

	/* skb is pure payload to encrypt */
	aead = x->data;
	alen = crypto_aead_authsize(aead);
	ivlen = crypto_aead_ivsize(aead);

	tfclen = 0;
	if (x->tfcpad) {
		struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
		u32 padto;

		padto = min(x->tfcpad, esp6_get_mtu(x, dst->child_mtu_cached));
		if (skb->len < padto)
			tfclen = padto - skb->len;
	}
	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
	clen = ALIGN(skb->len + 2 + tfclen, blksize);
	plen = clen - skb->len - tfclen;
	tailen = tfclen + plen + alen;

	assoclen = sizeof(*esph);
	seqhilen = 0;

	if (x->props.flags & XFRM_STATE_ESN) {
		seqhilen += sizeof(__be32);
		assoclen += seqhilen;
	}

	*skb_mac_header(skb) = IPPROTO_ESP;
	esph = ip_esp_hdr(skb);

	if (!skb_cloned(skb)) {
		if (tailen <= skb_availroom(skb)) {
			nfrags = 1;
			trailer = skb;
			tail = skb_tail_pointer(trailer);

			goto skip_cow;
		} else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)
			   && !skb_has_frag_list(skb)) {
			int allocsize;
			struct sock *sk = skb->sk;
			struct page_frag *pfrag = &x->xfrag;

			allocsize = ALIGN(tailen, L1_CACHE_BYTES);

			spin_lock_bh(&x->lock);

			if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
				spin_unlock_bh(&x->lock);
				goto cow;
			}

			page = pfrag->page;
			get_page(page);

			vaddr = kmap_atomic(page);

			tail = vaddr + pfrag->offset;

			esp_output_fill_trailer(tail, tfclen, plen, proto);

			kunmap_atomic(vaddr);

			nfrags = skb_shinfo(skb)->nr_frags;

			__skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
					     tailen);
			skb_shinfo(skb)->nr_frags = ++nfrags;

			pfrag->offset = pfrag->offset + allocsize;
			nfrags++;

			skb->len += tailen;
			skb->data_len += tailen;
			skb->truesize += tailen;
			if (sk)
				atomic_add(tailen, &sk->sk_wmem_alloc);

			skb_push(skb, -skb_network_offset(skb));

			esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
			esph->spi = x->id.spi;

			tmp = esp_alloc_tmp(aead, nfrags + 2, seqhilen);
			if (!tmp) {
				spin_unlock_bh(&x->lock);
				err = -ENOMEM;
				goto error;
			}
			seqhi = esp_tmp_seqhi(tmp);
			iv = esp_tmp_iv(aead, tmp, seqhilen);
			req = esp_tmp_req(aead, iv);
			sg = esp_req_sg(aead, req);
			dsg = &sg[nfrags];

			esph = esp_output_set_esn(skb, esph, seqhi);

			sg_init_table(sg, nfrags);
			skb_to_sgvec(skb, sg,
				     (unsigned char *)esph - skb->data,
				     assoclen + ivlen + clen + alen);

			allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);

			if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
				spin_unlock_bh(&x->lock);
				err = -ENOMEM;
				goto error;
			}

			skb_shinfo(skb)->nr_frags = 1;

			page = pfrag->page;
			get_page(page);
			/* replace page frags in skb with new page */
			__skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len);
			pfrag->offset = pfrag->offset + allocsize;

			sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1);
			skb_to_sgvec(skb, dsg,
				     (unsigned char *)esph - skb->data,
				     assoclen + ivlen + clen + alen);

			spin_unlock_bh(&x->lock);

			goto skip_cow2;
		}
	}

cow:
	err = skb_cow_data(skb, tailen, &trailer);
	if (err < 0)
		goto error;
	nfrags = err;

	tail = skb_tail_pointer(trailer);
	esph = ip_esp_hdr(skb);

skip_cow:
	esp_output_fill_trailer(tail, tfclen, plen, proto);

	pskb_put(skb, trailer, clen - skb->len + alen);
	skb_push(skb, -skb_network_offset(skb));

	esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
	esph->spi = x->id.spi;

	tmp = esp_alloc_tmp(aead, nfrags, seqhilen);
	if (!tmp) {
		err = -ENOMEM;
		goto error;
	}

	seqhi = esp_tmp_seqhi(tmp);
	iv = esp_tmp_iv(aead, tmp, seqhilen);
	req = esp_tmp_req(aead, iv);
	sg = esp_req_sg(aead, req);
	dsg = sg;

	esph = esp_output_set_esn(skb, esph, seqhi);

	sg_init_table(sg, nfrags);
	skb_to_sgvec(skb, sg,
		     (unsigned char *)esph - skb->data,
		     assoclen + ivlen + clen + alen);

skip_cow2:
	if ((x->props.flags & XFRM_STATE_ESN))
		aead_request_set_callback(req, 0, esp_output_done_esn, skb);
	else
		aead_request_set_callback(req, 0, esp_output_done, skb);

	aead_request_set_crypt(req, sg, dsg, ivlen + clen, iv);
	aead_request_set_ad(req, assoclen);

	seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
			    ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));

	memset(iv, 0, ivlen);
	memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&seqno + 8 - min(ivlen, 8),
	       min(ivlen, 8));

	ESP_SKB_CB(skb)->tmp = tmp;
	err = crypto_aead_encrypt(req);

	switch (err) {
	case -EINPROGRESS:
		goto error;

	case -EBUSY:
		err = NET_XMIT_DROP;
		break;

	case 0:
		if ((x->props.flags & XFRM_STATE_ESN))
			esp_output_restore_header(skb);
	}

	if (sg != dsg)
		esp_ssg_unref(x, tmp);
	kfree(tmp);

error:
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
800
62.16%
4
21.05%
Herbert Xu
361
28.05%
7
36.84%
Martin Willi
88
6.84%
1
5.26%
Kazunori Miyazawa
17
1.32%
1
5.26%
David S. Miller
9
0.70%
1
5.26%
Julia Lawall
4
0.31%
1
5.26%
Jamal Hadi Salim
3
0.23%
1
5.26%
Arnaldo Carvalho de Melo
3
0.23%
2
10.53%
Mathias Krause
2
0.16%
1
5.26%
Total
1287
100.00%
19
100.00%



static int esp_input_done2(struct sk_buff *skb, int err)
{
	struct xfrm_state *x = xfrm_input_state(skb);
	struct crypto_aead *aead = x->data;
	int alen = crypto_aead_authsize(aead);
	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
	int elen = skb->len - hlen;
	int hdr_len = skb_network_header_len(skb);
	int padlen;
	u8 nexthdr[2];

	kfree(ESP_SKB_CB(skb)->tmp);

	if (unlikely(err))
		goto out;

	if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))
		BUG();

	err = -EINVAL;
	padlen = nexthdr[0];
	if (padlen + 2 + alen >= elen) {
		net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
				    padlen + 2, elen - alen);
		goto out;
	}

	/* ... check padding bits here. Silly. :-) */

	pskb_trim(skb, skb->len - alen - padlen - 2);
	__skb_pull(skb, hlen);
	if (x->props.mode == XFRM_MODE_TUNNEL)
		skb_reset_transport_header(skb);
	else
		skb_set_transport_header(skb, -hdr_len);

	err = nexthdr[1];

	/* RFC4303: Drop dummy packets without any error */
	if (err == IPPROTO_NONE)
		err = -EINVAL;

out:
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
169
72.22%
6
50.00%
Kazunori Miyazawa
41
17.52%
1
8.33%
Li RongQing
16
6.84%
1
8.33%
David S. Miller
4
1.71%
2
16.67%
Joe Perches
2
0.85%
1
8.33%
Mathias Krause
2
0.85%
1
8.33%
Total
234
100.00%
12
100.00%



static void esp_input_done(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	xfrm_input_resume(skb, esp_input_done2(skb, err));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
35
100.00%
1
100.00%
Total
35
100.00%
1
100.00%



static void esp_input_restore_header(struct sk_buff *skb)
{
	esp_restore_header(skb, 0);
	__skb_pull(skb, 4);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
15
60.00%
1
50.00%
Herbert Xu
10
40.00%
1
50.00%
Total
25
100.00%
2
100.00%



static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi)
{
	struct xfrm_state *x = xfrm_input_state(skb);
	struct ip_esp_hdr *esph = (struct ip_esp_hdr *)skb->data;

	/* For ESN we move the header forward by 4 bytes to
         * accomodate the high bits.  We will move it back after
         * decryption.
         */
	if ((x->props.flags & XFRM_STATE_ESN)) {
		esph = (void *)skb_push(skb, 4);
		*seqhi = esph->spi;
		esph->spi = esph->seq_no;
		esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Steffen Klassert
87
89.69%
1
33.33%
Herbert Xu
9
9.28%
1
33.33%
Kazunori Miyazawa
1
1.03%
1
33.33%
Total
97
100.00%
3
100.00%



static void esp_input_done_esn(struct crypto_async_request *base, int err)
{
	struct sk_buff *skb = base->data;

	esp_input_restore_header(skb);
	esp_input_done(base, err);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Herbert Xu
35
100.00%
2
100.00%
Total
35
100.00%
2
100.00%



static int esp6_input(struct xfrm_state *x, struct sk_buff *skb)
{
	struct ip_esp_hdr *esph;
	struct crypto_aead *aead = x->data;
	struct aead_request *req;
	struct sk_buff *trailer;
	int ivlen = crypto_aead_ivsize(aead);
	int elen = skb->len - sizeof(*esph) - ivlen;
	int nfrags;
	int assoclen;
	int seqhilen;
	int ret = 0;
	void *tmp;
	__be32 *seqhi;
	u8 *iv;
	struct scatterlist *sg;

	if (!pskb_may_pull(skb, sizeof(*esph) + ivlen)) {
		ret = -EINVAL;
		goto out;
	}

	if (elen <= 0) {
		ret = -EINVAL;
		goto out;
	}

	assoclen = sizeof(*esph);
	seqhilen = 0;

	if (x->props.flags & XFRM_STATE_ESN) {
		seqhilen += sizeof(__be32);
		assoclen += seqhilen;
	}

	if (!skb_cloned(skb)) {
		if (!skb_is_nonlinear(skb)) {
			nfrags = 1;

			goto skip_cow;
		} else if (!skb_has_frag_list(skb)) {
			nfrags = skb_shinfo(skb)->nr_frags;
			nfrags++;

			goto skip_cow;
		}
	}

	nfrags = skb_cow_data(skb, 0, &trailer);
	if (nfrags < 0) {
		ret = -EINVAL;
		goto out;
	}

skip_cow:
	ret = -ENOMEM;
	tmp = esp_alloc_tmp(aead, nfrags, seqhilen);
	if (!tmp)
		goto out;

	ESP_SKB_CB(skb)->tmp = tmp;
	seqhi = esp_tmp_seqhi(tmp);
	iv = esp_tmp_iv(aead, tmp, seqhilen);
	req = esp_tmp_req(aead, iv);
	sg = esp_req_sg(aead, req);

	esp_input_set_header(skb, seqhi);

	sg_init_table(sg, nfrags);
	skb_to_sgvec(skb, sg, 0, skb->len);

	skb->ip_summed = CHECKSUM_NONE;

	if ((x->props.flags & XFRM_STATE_ESN))
		aead_request_set_callback(req, 0, esp_input_done_esn, skb);
	else
		aead_request_set_callback(req, 0, esp_input_done, skb);

	aead_request_set_crypt(req, sg, sg, elen + ivlen, iv);
	aead_request_set_ad(req, assoclen);

	ret = crypto_aead_decrypt(req);
	if (ret == -EINPROGRESS)
		goto out;

	if ((x->props.flags & XFRM_STATE_ESN))
		esp_input_restore_header(