| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| David S. Miller | 9756 | 92.06% | 10 | 31.25% |
| Sowmini Varadhan | 392 | 3.70% | 5 | 15.62% |
| Jag Raman | 355 | 3.35% | 8 | 25.00% |
| Dwight Engen | 35 | 0.33% | 1 | 3.12% |
| Steven Rostedt | 26 | 0.25% | 1 | 3.12% |
| Akinobu Mita | 25 | 0.24% | 2 | 6.25% |
| Jens Axboe | 3 | 0.03% | 1 | 3.12% |
| David L Stevens | 2 | 0.02% | 1 | 3.12% |
| Greg Kroah-Hartman | 1 | 0.01% | 1 | 3.12% |
| Paul Gortmaker | 1 | 0.01% | 1 | 3.12% |
| Christoph Hellwig | 1 | 0.01% | 1 | 3.12% |
| Total | 10597 | 32 |
// SPDX-License-Identifier: GPL-2.0 /* ldc.c: Logical Domain Channel link-layer protocol driver. * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> */ #include <linux/kernel.h> #include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/delay.h> #include <linux/errno.h> #include <linux/string.h> #include <linux/scatterlist.h> #include <linux/interrupt.h> #include <linux/list.h> #include <linux/init.h> #include <linux/bitmap.h> #include <asm/iommu-common.h> #include <asm/hypervisor.h> #include <asm/iommu.h> #include <asm/page.h> #include <asm/ldc.h> #include <asm/mdesc.h> #define DRV_MODULE_NAME "ldc" #define PFX DRV_MODULE_NAME ": " #define DRV_MODULE_VERSION "1.1" #define DRV_MODULE_RELDATE "July 22, 2008" #define COOKIE_PGSZ_CODE 0xf000000000000000ULL #define COOKIE_PGSZ_CODE_SHIFT 60ULL static char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; /* Packet header layout for unreliable and reliable mode frames. * When in RAW mode, packets are simply straight 64-byte payloads * with no headers. */ struct ldc_packet { u8 type; #define LDC_CTRL 0x01 #define LDC_DATA 0x02 #define LDC_ERR 0x10 u8 stype; #define LDC_INFO 0x01 #define LDC_ACK 0x02 #define LDC_NACK 0x04 u8 ctrl; #define LDC_VERS 0x01 /* Link Version */ #define LDC_RTS 0x02 /* Request To Send */ #define LDC_RTR 0x03 /* Ready To Receive */ #define LDC_RDX 0x04 /* Ready for Data eXchange */ #define LDC_CTRL_MSK 0x0f u8 env; #define LDC_LEN 0x3f #define LDC_FRAG_MASK 0xc0 #define LDC_START 0x40 #define LDC_STOP 0x80 u32 seqid; union { u8 u_data[LDC_PACKET_SIZE - 8]; struct { u32 pad; u32 ackid; u8 r_data[LDC_PACKET_SIZE - 8 - 8]; } r; } u; }; struct ldc_version { u16 major; u16 minor; }; /* Ordered from largest major to lowest. */ static struct ldc_version ver_arr[] = { { .major = 1, .minor = 0 }, }; #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) struct ldc_channel; struct ldc_mode_ops { int (*write)(struct ldc_channel *, const void *, unsigned int); int (*read)(struct ldc_channel *, void *, unsigned int); }; static const struct ldc_mode_ops raw_ops; static const struct ldc_mode_ops nonraw_ops; static const struct ldc_mode_ops stream_ops; int ldom_domaining_enabled; struct ldc_iommu { /* Protects ldc_unmap. */ spinlock_t lock; struct ldc_mtable_entry *page_table; struct iommu_map_table iommu_map_table; }; struct ldc_channel { /* Protects all operations that depend upon channel state. */ spinlock_t lock; unsigned long id; u8 *mssbuf; u32 mssbuf_len; u32 mssbuf_off; struct ldc_packet *tx_base; unsigned long tx_head; unsigned long tx_tail; unsigned long tx_num_entries; unsigned long tx_ra; unsigned long tx_acked; struct ldc_packet *rx_base; unsigned long rx_head; unsigned long rx_tail; unsigned long rx_num_entries; unsigned long rx_ra; u32 rcv_nxt; u32 snd_nxt; unsigned long chan_state; struct ldc_channel_config cfg; void *event_arg; const struct ldc_mode_ops *mops; struct ldc_iommu iommu; struct ldc_version ver; u8 hs_state; #define LDC_HS_CLOSED 0x00 #define LDC_HS_OPEN 0x01 #define LDC_HS_GOTVERS 0x02 #define LDC_HS_SENTRTR 0x03 #define LDC_HS_GOTRTR 0x04 #define LDC_HS_COMPLETE 0x10 u8 flags; #define LDC_FLAG_ALLOCED_QUEUES 0x01 #define LDC_FLAG_REGISTERED_QUEUES 0x02 #define LDC_FLAG_REGISTERED_IRQS 0x04 #define LDC_FLAG_RESET 0x10 u8 mss; u8 state; #define LDC_IRQ_NAME_MAX 32 char rx_irq_name[LDC_IRQ_NAME_MAX]; char tx_irq_name[LDC_IRQ_NAME_MAX]; struct hlist_head mh_list; struct hlist_node list; }; #define ldcdbg(TYPE, f, a...) \ do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ } while (0) #define LDC_ABORT(lp) ldc_abort((lp), __func__) static const char *state_to_str(u8 state) { switch (state) { case LDC_STATE_INVALID: return "INVALID"; case LDC_STATE_INIT: return "INIT"; case LDC_STATE_BOUND: return "BOUND"; case LDC_STATE_READY: return "READY"; case LDC_STATE_CONNECTED: return "CONNECTED"; default: return "<UNKNOWN>"; } } static unsigned long __advance(unsigned long off, unsigned long num_entries) { off += LDC_PACKET_SIZE; if (off == (num_entries * LDC_PACKET_SIZE)) off = 0; return off; } static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) { return __advance(off, lp->rx_num_entries); } static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) { return __advance(off, lp->tx_num_entries); } static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, unsigned long *new_tail) { struct ldc_packet *p; unsigned long t; t = tx_advance(lp, lp->tx_tail); if (t == lp->tx_head) return NULL; *new_tail = t; p = lp->tx_base; return p + (lp->tx_tail / LDC_PACKET_SIZE); } /* When we are in reliable or stream mode, have to track the next packet * we haven't gotten an ACK for in the TX queue using tx_acked. We have * to be careful not to stomp over the queue past that point. During * the handshake, we don't have TX data packets pending in the queue * and that's why handshake_get_tx_packet() need not be mindful of * lp->tx_acked. */ static unsigned long head_for_data(struct ldc_channel *lp) { if (lp->cfg.mode == LDC_MODE_STREAM) return lp->tx_acked; return lp->tx_head; } static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) { unsigned long limit, tail, new_tail, diff; unsigned int mss; limit = head_for_data(lp); tail = lp->tx_tail; new_tail = tx_advance(lp, tail); if (new_tail == limit) return 0; if (limit > new_tail) diff = limit - new_tail; else diff = (limit + ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); diff /= LDC_PACKET_SIZE; mss = lp->mss; if (diff * mss < size) return 0; return 1; } static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, unsigned long *new_tail) { struct ldc_packet *p; unsigned long h, t; h = head_for_data(lp); t = tx_advance(lp, lp->tx_tail); if (t == h) return NULL; *new_tail = t; p = lp->tx_base; return p + (lp->tx_tail / LDC_PACKET_SIZE); } static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) { unsigned long orig_tail = lp->tx_tail; int limit = 1000; lp->tx_tail = tail; while (limit-- > 0) { unsigned long err; err = sun4v_ldc_tx_set_qtail(lp->id, tail); if (!err) return 0; if (err != HV_EWOULDBLOCK) { lp->tx_tail = orig_tail; return -EINVAL; } udelay(1); } lp->tx_tail = orig_tail; return -EBUSY; } /* This just updates the head value in the hypervisor using * a polling loop with a timeout. The caller takes care of * upating software state representing the head change, if any. */ static int __set_rx_head(struct ldc_channel *lp, unsigned long head) { int limit = 1000; while (limit-- > 0) { unsigned long err; err = sun4v_ldc_rx_set_qhead(lp->id, head); if (!err) return 0; if (err != HV_EWOULDBLOCK) return -EINVAL; udelay(1); } return -EBUSY; } static int send_tx_packet(struct ldc_channel *lp, struct ldc_packet *p, unsigned long new_tail) { BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); return set_tx_tail(lp, new_tail); } static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, u8 stype, u8 ctrl, void *data, int dlen, unsigned long *new_tail) { struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); if (p) { memset(p, 0, sizeof(*p)); p->type = LDC_CTRL; p->stype = stype; p->ctrl = ctrl; if (data) memcpy(p->u.u_data, data, dlen); } return p; } static int start_handshake(struct ldc_channel *lp) { struct ldc_packet *p; struct ldc_version *ver; unsigned long new_tail; ver = &ver_arr[0]; ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", ver->major, ver->minor); p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, ver, sizeof(*ver), &new_tail); if (p) { int err = send_tx_packet(lp, p, new_tail); if (!err) lp->flags &= ~LDC_FLAG_RESET; return err; } return -EBUSY; } static int send_version_nack(struct ldc_channel *lp, u16 major, u16 minor) { struct ldc_packet *p; struct ldc_version ver; unsigned long new_tail; ver.major = major; ver.minor = minor; p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, &ver, sizeof(ver), &new_tail); if (p) { ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", ver.major, ver.minor); return send_tx_packet(lp, p, new_tail); } return -EBUSY; } static int send_version_ack(struct ldc_channel *lp, struct ldc_version *vp) { struct ldc_packet *p; unsigned long new_tail; p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, vp, sizeof(*vp), &new_tail); if (p) { ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", vp->major, vp->minor); return send_tx_packet(lp, p, new_tail); } return -EBUSY; } static int send_rts(struct ldc_channel *lp) { struct ldc_packet *p; unsigned long new_tail; p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, &new_tail); if (p) { p->env = lp->cfg.mode; p->seqid = 0; lp->rcv_nxt = 0; ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", p->env, p->seqid); return send_tx_packet(lp, p, new_tail); } return -EBUSY; } static int send_rtr(struct ldc_channel *lp) { struct ldc_packet *p; unsigned long new_tail; p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, &new_tail); if (p) { p->env = lp->cfg.mode; p->seqid = 0; ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", p->env, p->seqid); return send_tx_packet(lp, p, new_tail); } return -EBUSY; } static int send_rdx(struct ldc_channel *lp) { struct ldc_packet *p; unsigned long new_tail; p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, &new_tail); if (p) { p->env = 0; p->seqid = ++lp->snd_nxt; p->u.r.ackid = lp->rcv_nxt; ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", p->env, p->seqid, p->u.r.ackid); return send_tx_packet(lp, p, new_tail); } return -EBUSY; } static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) { struct ldc_packet *p; unsigned long new_tail; int err; p = data_get_tx_packet(lp, &new_tail); if (!p) return -EBUSY; memset(p, 0, sizeof(*p)); p->type = data_pkt->type; p->stype = LDC_NACK; p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; p->seqid = lp->snd_nxt + 1; p->u.r.ackid = lp->rcv_nxt; ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", p->type, p->ctrl, p->seqid, p->u.r.ackid); err = send_tx_packet(lp, p, new_tail); if (!err) lp->snd_nxt++; return err; } static int ldc_abort(struct ldc_channel *lp, const char *msg) { unsigned long hv_err; ldcdbg(STATE, "ABORT[%s]\n", msg); ldc_print(lp); /* We report but do not act upon the hypervisor errors because * there really isn't much we can do if they fail at this point. */ hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); if (hv_err) printk(KERN_ERR PFX "ldc_abort: " "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, &lp->chan_state); if (hv_err) printk(KERN_ERR PFX "ldc_abort: " "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", lp->id, hv_err); hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); if (hv_err) printk(KERN_ERR PFX "ldc_abort: " "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); /* Refetch the RX queue state as well, because we could be invoked * here in the queue processing context. */ hv_err = sun4v_ldc_rx_get_state(lp->id, &lp->rx_head, &lp->rx_tail, &lp->chan_state); if (hv_err) printk(KERN_ERR PFX "ldc_abort: " "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", lp->id, hv_err); return -ECONNRESET; } static struct ldc_version *find_by_major(u16 major) { struct ldc_version *ret = NULL; int i; for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { struct ldc_version *v = &ver_arr[i]; if (v->major <= major) { ret = v; break; } } return ret; } static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) { struct ldc_version *vap; int err; ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", vp->major, vp->minor); if (lp->hs_state == LDC_HS_GOTVERS) { lp->hs_state = LDC_HS_OPEN; memset(&lp->ver, 0, sizeof(lp->ver)); } vap = find_by_major(vp->major); if (!vap) { err = send_version_nack(lp, 0, 0); } else if (vap->major != vp->major) { err = send_version_nack(lp, vap->major, vap->minor); } else { struct ldc_version ver = *vp; if (ver.minor > vap->minor) ver.minor = vap->minor; err = send_version_ack(lp, &ver); if (!err) { lp->ver = ver; lp->hs_state = LDC_HS_GOTVERS; } } if (err) return LDC_ABORT(lp); return 0; } static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) { ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", vp->major, vp->minor); if (lp->hs_state == LDC_HS_GOTVERS) { if (lp->ver.major != vp->major || lp->ver.minor != vp->minor) return LDC_ABORT(lp); } else { lp->ver = *vp; lp->hs_state = LDC_HS_GOTVERS; } if (send_rts(lp)) return LDC_ABORT(lp); return 0; } static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) { struct ldc_version *vap; struct ldc_packet *p; unsigned long new_tail; if (vp->major == 0 && vp->minor == 0) return LDC_ABORT(lp); vap = find_by_major(vp->major); if (!vap) return LDC_ABORT(lp); p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, vap, sizeof(*vap), &new_tail); if (!p) return LDC_ABORT(lp); return send_tx_packet(lp, p, new_tail); } static int process_version(struct ldc_channel *lp, struct ldc_packet *p) { struct ldc_version *vp; vp = (struct ldc_version *) p->u.u_data; switch (p->stype) { case LDC_INFO: return process_ver_info(lp, vp); case LDC_ACK: return process_ver_ack(lp, vp); case LDC_NACK: return process_ver_nack(lp, vp); default: return LDC_ABORT(lp); } } static int process_rts(struct ldc_channel *lp, struct ldc_packet *p) { ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", p->stype, p->seqid, p->env); if (p->stype != LDC_INFO || lp->hs_state != LDC_HS_GOTVERS || p->env != lp->cfg.mode) return LDC_ABORT(lp); lp->snd_nxt = p->seqid; lp->rcv_nxt = p->seqid; lp->hs_state = LDC_HS_SENTRTR; if (send_rtr(lp)) return LDC_ABORT(lp); return 0; } static int process_rtr(struct ldc_channel *lp, struct ldc_packet *p) { ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", p->stype, p->seqid, p->env); if (p->stype != LDC_INFO || p->env != lp->cfg.mode) return LDC_ABORT(lp); lp->snd_nxt = p->seqid; lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); send_rdx(lp); return LDC_EVENT_UP; } static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) { return lp->rcv_nxt + 1 == seqid; } static int process_rdx(struct ldc_channel *lp, struct ldc_packet *p) { ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", p->stype, p->seqid, p->env, p->u.r.ackid); if (p->stype != LDC_INFO || !(rx_seq_ok(lp, p->seqid))) return LDC_ABORT(lp); lp->rcv_nxt = p->seqid; lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); return LDC_EVENT_UP; } static int process_control_frame(struct ldc_channel *lp, struct ldc_packet *p) { switch (p->ctrl) { case LDC_VERS: return process_version(lp, p); case LDC_RTS: return process_rts(lp, p); case LDC_RTR: return process_rtr(lp, p); case LDC_RDX: return process_rdx(lp, p); default: return LDC_ABORT(lp); } } static int process_error_frame(struct ldc_channel *lp, struct ldc_packet *p) { return LDC_ABORT(lp); } static int process_data_ack(struct ldc_channel *lp, struct ldc_packet *ack) { unsigned long head = lp->tx_acked; u32 ackid = ack->u.r.ackid; while (1) { struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); head = tx_advance(lp, head); if (p->seqid == ackid) { lp->tx_acked = head; return 0; } if (head == lp->tx_tail) return LDC_ABORT(lp); } return 0; } static void send_events(struct ldc_channel *lp, unsigned int event_mask) { if (event_mask & LDC_EVENT_RESET) lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); if (event_mask & LDC_EVENT_UP) lp->cfg.event(lp->event_arg, LDC_EVENT_UP); if (event_mask & LDC_EVENT_DATA_READY) lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); } static irqreturn_t ldc_rx(int irq, void *dev_id) { struct ldc_channel *lp = dev_id; unsigned long orig_state, flags; unsigned int event_mask; spin_lock_irqsave(&lp->lock, flags); orig_state = lp->chan_state; /* We should probably check for hypervisor errors here and * reset the LDC channel if we get one. */ sun4v_ldc_rx_get_state(lp->id, &lp->rx_head, &lp->rx_tail, &lp->chan_state); ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); event_mask = 0; if (lp->cfg.mode == LDC_MODE_RAW && lp->chan_state == LDC_CHANNEL_UP) { lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); /* * Generate an LDC_EVENT_UP event if the channel * was not already up. */ if (orig_state != LDC_CHANNEL_UP) { event_mask |= LDC_EVENT_UP; orig_state = lp->chan_state; } } /* If we are in reset state, flush the RX queue and ignore * everything. */ if (lp->flags & LDC_FLAG_RESET) { (void) ldc_rx_reset(lp); goto out; } /* Once we finish the handshake, we let the ldc_read() * paths do all of the control frame and state management. * Just trigger the callback. */ if (lp->hs_state == LDC_HS_COMPLETE) { handshake_complete: if (lp->chan_state != orig_state) { unsigned int event = LDC_EVENT_RESET; if (lp->chan_state == LDC_CHANNEL_UP) event = LDC_EVENT_UP; event_mask |= event; } if (lp->rx_head != lp->rx_tail) event_mask |= LDC_EVENT_DATA_READY; goto out; } if (lp->chan_state != orig_state) goto out; while (lp->rx_head != lp->rx_tail) { struct ldc_packet *p; unsigned long new; int err; p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); switch (p->type) { case LDC_CTRL: err = process_control_frame(lp, p); if (err > 0) event_mask |= err; break; case LDC_DATA: event_mask |= LDC_EVENT_DATA_READY; err = 0; break; case LDC_ERR: err = process_error_frame(lp, p); break; default: err = LDC_ABORT(lp); break; } if (err < 0) break; new = lp->rx_head; new += LDC_PACKET_SIZE; if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) new = 0; lp->rx_head = new; err = __set_rx_head(lp, new); if (err < 0) { (void) LDC_ABORT(lp); break; } if (lp->hs_state == LDC_HS_COMPLETE) goto handshake_complete; } out: spin_unlock_irqrestore(&lp->lock, flags); send_events(lp, event_mask); return IRQ_HANDLED; } static irqreturn_t ldc_tx(int irq, void *dev_id) { struct ldc_channel *lp = dev_id; unsigned long flags, orig_state; unsigned int event_mask = 0; spin_lock_irqsave(&lp->lock, flags); orig_state = lp->chan_state; /* We should probably check for hypervisor errors here and * reset the LDC channel if we get one. */ sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, &lp->chan_state); ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); if (lp->cfg.mode == LDC_MODE_RAW && lp->chan_state == LDC_CHANNEL_UP) { lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); /* * Generate an LDC_EVENT_UP event if the channel * was not already up. */ if (orig_state != LDC_CHANNEL_UP) { event_mask |= LDC_EVENT_UP; orig_state = lp->chan_state; } } spin_unlock_irqrestore(&lp->lock, flags); send_events(lp, event_mask); return IRQ_HANDLED; } /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so * XXX that addition and removal from the ldc_channel_list has * XXX atomicity, otherwise the __ldc_channel_exists() check is * XXX totally pointless as another thread can slip into ldc_alloc() * XXX and add a channel with the same ID. There also needs to be * XXX a spinlock for ldc_channel_list. */ static HLIST_HEAD(ldc_channel_list); static int __ldc_channel_exists(unsigned long id) { struct ldc_channel *lp; hlist_for_each_entry(lp, &ldc_channel_list, list) { if (lp->id == id) return 1; } return 0; } static int alloc_queue(const char *name, unsigned long num_entries, struct ldc_packet **base, unsigned long *ra) { unsigned long size, order; void *q; size = num_entries * LDC_PACKET_SIZE; order = get_order(size); q = (void *) __get_free_pages(GFP_KERNEL, order); if (!q) { printk(KERN_ERR PFX "Alloc of %s queue failed with " "size=%lu order=%lu\n", name, size, order); return -ENOMEM; } memset(q, 0, PAGE_SIZE << order); *base = q; *ra = __pa(q); return 0; } static void free_queue(unsigned long num_entries, struct ldc_packet *q) { unsigned long size, order; if (!q) return; size = num_entries * LDC_PACKET_SIZE; order = get_order(size); free_pages((unsigned long)q, order); } static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) { u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ cookie &= ~COOKIE_PGSZ_CODE; return (cookie >> (13ULL + (szcode * 3ULL))); } static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie, unsigned long entry, unsigned long npages) { struct ldc_mtable_entry *base; unsigned long i, shift; shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; base = iommu->page_table + entry; for (i = 0; i < npages; i++) { if (base->cookie) sun4v_ldc_revoke(id, cookie + (i << shift), base->cookie); base->mte = 0; } } /* XXX Make this configurable... XXX */ #define LDC_IOTABLE_SIZE (8 * 1024) static int ldc_iommu_init(const char *name, struct ldc_channel *lp) { unsigned long sz, num_tsb_entries, tsbsize, order; struct ldc_iommu *ldc_iommu = &lp->iommu; struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; struct ldc_mtable_entry *table; unsigned long hv_err; int err; num_tsb_entries = LDC_IOTABLE_SIZE; tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); spin_lock_init(&ldc_iommu->lock); sz = num_tsb_entries / 8; sz = (sz + 7UL) & ~7UL; iommu->map = kzalloc(sz, GFP_KERNEL); if (!iommu->map) { printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); return -ENOMEM; } iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, NULL, false /* no large pool */, 1 /* npools */, true /* skip span boundary check */); order = get_order(tsbsize); table = (struct ldc_mtable_entry *) __get_free_pages(GFP_KERNEL, order); err = -ENOMEM; if (!table) { printk(KERN_ERR PFX "Alloc of MTE table failed, " "size=%lu order=%lu\n", tsbsize, order); goto out_free_map; } memset(table, 0, PAGE_SIZE << order); ldc_iommu->page_table = table; hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), num_tsb_entries); err = -EINVAL; if (hv_err) goto out_free_table; return 0; out_free_table: free_pages((unsigned long) table, order); ldc_iommu->page_table = NULL; out_free_map: kfree(iommu->map); iommu->map = NULL; return err; } static void ldc_iommu_release(struct ldc_channel *lp) { struct ldc_iommu *ldc_iommu = &lp->iommu; struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; unsigned long num_tsb_entries, tsbsize, order; (void) sun4v_ldc_set_map_table(lp->id, 0, 0); num_tsb_entries = iommu->poolsize * iommu->nr_pools; tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); order = get_order(tsbsize); free_pages((unsigned long) ldc_iommu->page_table, order); ldc_iommu->page_table = NULL; kfree(iommu->map); iommu->map = NULL; } struct ldc_channel *ldc_alloc(unsigned long id, const struct ldc_channel_config *cfgp, void *event_arg, const char *name) { struct ldc_channel *lp; const struct ldc_mode_ops *mops; unsigned long dummy1, dummy2, hv_err; u8 mss, *mssbuf; int err; err = -ENODEV; if (!ldom_domaining_enabled) goto out_err; err = -EINVAL; if (!cfgp) goto out_err; if (!name) goto out_err; switch (cfgp->mode) { case LDC_MODE_RAW: mops = &raw_ops; mss = LDC_PACKET_SIZE; break; case LDC_MODE_UNRELIABLE: mops = &nonraw_ops; mss = LDC_PACKET_SIZE - 8; break; case LDC_MODE_STREAM: mops = &stream_ops; mss = LDC_PACKET_SIZE - 8 - 8; break; default: goto out_err; } if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) goto out_err; hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); err = -ENODEV; if (hv_err == HV_ECHANNEL) goto out_err; err = -EEXIST; if (__ldc_channel_exists(id)) goto out_err; mssbuf = NULL; lp = kzalloc(sizeof(*lp), GFP_KERNEL); err = -ENOMEM; if (!lp) goto out_err; spin_lock_init(&lp->lock); lp->id = id; err = ldc_iommu_init(name, lp); if (err) goto out_free_ldc; lp->mops = mops; lp->mss = mss; lp->cfg = *cfgp; if (!lp->cfg.mtu) lp->cfg.mtu = LDC_DEFAULT_MTU; if (lp->cfg.mode == LDC_MODE_STREAM) { mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); if (!mssbuf) { err = -ENOMEM; goto out_free_iommu; } lp->mssbuf = mssbuf; } lp->event_arg = event_arg; /* XXX allow setting via ldc_channel_config to override defaults * XXX or use some formula based upon mtu */ lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; err = alloc_queue("TX", lp->tx_num_entries, &lp->tx_base, &lp->tx_ra); if (err) goto out_free_mssbuf; err = alloc_queue("RX", lp->rx_num_entries, &lp->rx_base, &lp->rx_ra); if (err) goto out_free_txq; lp->flags |= LDC_FLAG_ALLOCED_QUEUES; lp->hs_state = LDC_HS_CLOSED; ldc_set_state(lp, LDC_STATE_INIT); INIT_HLIST_NODE(&lp->list); hlist_add_head(&lp->list, &ldc_channel_list); INIT_HLIST_HEAD(&lp->mh_list); snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, lp->rx_irq_name, lp); if (err) goto out_free_txq; err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, lp->tx_irq_name, lp); if (err) { free_irq(lp->cfg.rx_irq, lp); goto out_free_txq; } return lp; out_free_txq: free_queue(lp->tx_num_entries, lp->tx_base); out_free_mssbuf: kfree(mssbuf); out_free_iommu: ldc_iommu_release(lp); out_free_ldc: kfree(lp); out_err: return ERR_PTR(err); } EXPORT_SYMBOL(ldc_alloc); void ldc_unbind(struct ldc_channel *lp) { if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { free_irq(lp->cfg.rx_irq, lp); free_irq(lp->cfg.tx_irq, lp); lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; } if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { sun4v_ldc_tx_qconf(lp->id, 0, 0); sun4v_ldc_rx_qconf(lp->id, 0, 0); lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; } if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { free_queue(lp->tx_num_entries, lp->tx_base); free_queue(lp->rx_num_entries, lp->rx_base); lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; } ldc_set_state(lp, LDC_STATE_INIT); } EXPORT_SYMBOL(ldc_unbind); void ldc_free(struct ldc_channel *lp) { ldc_unbind(lp); hlist_del(&lp->list); kfree(lp->mssbuf); ldc_iommu_release(lp); kfree(lp); } EXPORT_SYMBOL(ldc_free); /* Bind the channel. This registers the LDC queues with * the hypervisor and puts the channel into a pseudo-listening * state. This does not initiate a handshake, ldc_connect() does * that. */ int ldc_bind(struct ldc_channel *lp) { unsigned long hv_err, flags; int err = -EINVAL; if (lp->state != LDC_STATE_INIT) return -EINVAL; spin_lock_irqsave(&lp->lock, flags); enable_irq(lp->cfg.rx_irq); enable_irq(lp->cfg.tx_irq); lp->flags |= LDC_FLAG_REGISTERED_IRQS; err = -ENODEV; hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); if (hv_err) goto out_free_irqs; hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); if (hv_err) goto out_free_irqs; hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); if (hv_err) goto out_unmap_tx; hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); if (hv_err) goto out_unmap_tx; lp->flags |= LDC_FLAG_REGISTERED_QUEUES; hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, &lp->chan_state); err = -EBUSY; if (hv_err) goto out_unmap_rx; lp->tx_acked = lp->tx_head; lp->hs_state = LDC_HS_OPEN; ldc_set_state(lp, LDC_STATE_BOUND); if (lp->cfg.mode == LDC_MODE_RAW) { /* * There is no handshake in RAW mode, so handshake * is completed. */ lp->hs_state = LDC_HS_COMPLETE; } spin_unlock_irqrestore(&lp->lock, flags); return 0; out_unmap_rx: lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; sun4v_ldc_rx_qconf(lp->id, 0, 0); out_unmap_tx: sun4v_ldc_tx_qconf(lp->id, 0, 0); out_free_irqs: lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; free_irq(lp->cfg.tx_irq, lp); free_irq(lp->cfg.rx_irq, lp); spin_unlock_irqrestore(&lp->lock, flags); return err; } EXPORT_SYMBOL(ldc_bind); int ldc_connect(struct ldc_channel *lp) { unsigned long flags; int err; if (lp->cfg.mode == LDC_MODE_RAW) return -EINVAL; spin_lock_irqsave(&lp->lock, flags); if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || lp->hs_state != LDC_HS_OPEN) err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); else err = start_handshake(lp); spin_unlock_irqrestore(&lp->lock, flags); return err; } EXPORT_SYMBOL(ldc_connect); int ldc_disconnect(struct ldc_channel *lp) { unsigned long hv_err, flags; int err; if (lp->cfg.mode == LDC_MODE_RAW) return -EINVAL; if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) return -EINVAL; spin_lock_irqsave(&lp->lock, flags); err = -ENODEV; hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); if (hv_err) goto out_err; hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); if (hv_err) goto out_err; hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); if (hv_err) goto out_err; hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); if (hv_err) goto out_err; ldc_set_state(lp, LDC_STATE_BOUND); lp->hs_state = LDC_HS_OPEN; lp->flags |= LDC_FLAG_RESET; spin_unlock_irqrestore(&lp->lock, flags); return 0; out_err: sun4v_ldc_tx_qconf(lp->id, 0, 0); sun4v_ldc_rx_qconf(lp->id, 0, 0); free_irq(lp->cfg.tx_irq, lp); free_irq(lp->cfg.rx_irq, lp); lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | LDC_FLAG_REGISTERED_QUEUES); ldc_set_state(lp, LDC_STATE_INIT); spin_unlock_irqrestore(&lp->lock, flags); return err; } EXPORT_SYMBOL(ldc_disconnect); int ldc_state(struct ldc_channel *lp) { return lp->state; } EXPORT_SYMBOL(ldc_state); void ldc_set_state(struct ldc_channel *lp, u8 state) { ldcdbg(STATE, "STATE (%s) --> (%s)\n", state_to_str(lp->state), state_to_str(state)); lp->state = state; } EXPORT_SYMBOL(ldc_set_state); int ldc_mode(struct ldc_channel *lp) { return lp->cfg.mode; } EXPORT_SYMBOL(ldc_mode); int ldc_rx_reset(struct ldc_channel *lp) { return __set_rx_head(lp, lp->rx_tail); } EXPORT_SYMBOL(ldc_rx_reset); void __ldc_print(struct ldc_channel *lp, const char *caller) { pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n" "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n" "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n" "\trcv_nxt=%u snd_nxt=%u\n", caller, lp->id, lp->flags, state_to_str(lp->state), lp->chan_state, lp->hs_state, lp->rx_head, lp->rx_tail, lp->rx_num_entries, lp->tx_head, lp->tx_tail, lp->tx_num_entries, lp->rcv_nxt, lp->snd_nxt); } EXPORT_SYMBOL(__ldc_print); static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) { struct ldc_packet *p; unsigned long new_tail, hv_err; int err; hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, &lp->chan_state); if (unlikely(hv_err)) return -EBUSY; if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) return LDC_ABORT(lp); if (size > LDC_PACKET_SIZE) return -EMSGSIZE; p = data_get_tx_packet(lp, &new_tail); if (!p) return -EAGAIN; memcpy(p, buf, size); err = send_tx_packet(lp, p, new_tail); if (!err) err = size; return err; } static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) { struct ldc_packet *p; unsigned long hv_err, new; int err; if (size < LDC_PACKET_SIZE) return -EINVAL; hv_err = sun4v_ldc_rx_get_state(lp->id, &lp->rx_head, &lp->rx_tail, &lp->chan_state); if (hv_err) return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) return -ECONNRESET; if (lp->rx_head == lp->rx_tail) return 0; p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); memcpy(buf, p, LDC_PACKET_SIZE); new = rx_advance(lp, lp->rx_head); lp->rx_head = new; err = __set_rx_head(lp, new); if (err < 0) err = -ECONNRESET; else err = LDC_PACKET_SIZE; return err; } static const struct ldc_mode_ops raw_ops = { .write = write_raw, .read = read_raw, }; static int write_nonraw(struct ldc_channel *lp, const void *buf, unsigned int size) { unsigned long hv_err, tail; unsigned int copied; u32 seq; int err; hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, &lp->chan_state); if (unlikely(hv_err)) return -EBUSY; if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) return LDC_ABORT(lp); if (!tx_has_space_for(lp, size)) return -EAGAIN; seq = lp->snd_nxt; copied = 0; tail = lp->tx_tail; while (copied < size) { struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? p->u.u_data : p->u.r.r_data); int data_len; p->type = LDC_DATA; p->stype = LDC_INFO; p->ctrl = 0; data_len = size - copied; if (data_len > lp->mss) data_len = lp->mss; BUG_ON(data_len > LDC_LEN); p->env = (data_len | (copied == 0 ? LDC_START : 0) | (data_len == size - copied ? LDC_STOP : 0)); p->seqid = ++seq; ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", p->type, p->stype, p->ctrl, p->env, p->seqid); memcpy(data, buf, data_len); buf += data_len; copied += data_len; tail = tx_advance(lp, tail); } err = set_tx_tail(lp, tail); if (!err) { lp->snd_nxt = seq; err = size; } return err; } static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, struct ldc_packet *first_frag) { int err; if (first_frag) lp->rcv_nxt = first_frag->seqid - 1; err = send_data_nack(lp, p); if (err) return err; err = ldc_rx_reset(lp); if (err < 0) return LDC_ABORT(lp); return 0; } static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) { if (p->stype & LDC_ACK) { int err = process_data_ack(lp, p); if (err) return err; } if (p->stype & LDC_NACK) return LDC_ABORT(lp); return 0; } static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) { unsigned long dummy; int limit = 1000; ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", cur_head, lp->rx_head, lp->rx_tail); while (limit-- > 0) { unsigned long hv_err; hv_err = sun4v_ldc_rx_get_state(lp->id, &dummy, &lp->rx_tail, &lp->chan_state); if (hv_err) return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) return -ECONNRESET; if (cur_head != lp->rx_tail) { ldcdbg(DATA, "DATA WAIT DONE " "head[%lx] tail[%lx] chan_state[%lx]\n", dummy, lp->rx_tail, lp->chan_state); return 0; } udelay(1); } return -EAGAIN; } static int rx_set_head(struct ldc_channel *lp, unsigned long head) { int err = __set_rx_head(lp, head); if (err < 0) return LDC_ABORT(lp); lp->rx_head = head; return 0; } static void send_data_ack(struct ldc_channel *lp) { unsigned long new_tail; struct ldc_packet *p; p = data_get_tx_packet(lp, &new_tail); if (likely(p)) { int err; memset(p, 0, sizeof(*p)); p->type = LDC_DATA; p->stype = LDC_ACK; p->ctrl = 0; p->seqid = lp->snd_nxt + 1; p->u.r.ackid = lp->rcv_nxt; err = send_tx_packet(lp, p, new_tail); if (!err) lp->snd_nxt++; } } static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) { struct ldc_packet *first_frag; unsigned long hv_err, new; int err, copied; hv_err = sun4v_ldc_rx_get_state(lp->id, &lp->rx_head, &lp->rx_tail, &lp->chan_state); if (hv_err) return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) return -ECONNRESET; if (lp->rx_head == lp->rx_tail) return 0; first_frag = NULL; copied = err = 0; new = lp->rx_head; while (1) { struct ldc_packet *p; int pkt_len; BUG_ON(new == lp->rx_tail); p = lp->rx_base + (new / LDC_PACKET_SIZE); ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " "rcv_nxt[%08x]\n", p->type, p->stype, p->ctrl, p->env, p->seqid, p->u.r.ackid, lp->rcv_nxt); if (unlikely(!rx_seq_ok(lp, p->seqid))) { err = rx_bad_seq(lp, p, first_frag); copied = 0; break; } if (p->type & LDC_CTRL) { err = process_control_frame(lp, p); if (err < 0) break; err = 0; } lp->rcv_nxt = p->seqid; /* * If this is a control-only packet, there is nothing * else to do but advance the rx queue since the packet * was already processed above. */ if (!(p->type & LDC_DATA)) { new = rx_advance(lp, new); break; } if (p->stype & (LDC_ACK | LDC_NACK)) { err = data_ack_nack(lp, p); if (err) break; } if (!(p->stype & LDC_INFO)) { new = rx_advance(lp, new); err = rx_set_head(lp, new); if (err) break; goto no_data; } pkt_len = p->env & LDC_LEN; /* Every initial packet starts with the START bit set. * * Singleton packets will have both START+STOP set. * * Fragments will have START set in the first frame, STOP * set in the last frame, and neither bit set in middle * frames of the packet. * * Therefore if we are at the beginning of a packet and * we don't see START, or we are in the middle of a fragmented * packet and do see START, we are unsynchronized and should * flush the RX queue. */ if ((first_frag == NULL && !(p->env & LDC_START)) || (first_frag != NULL && (p->env & LDC_START))) { if (!first_frag) new = rx_advance(lp, new); err = rx_set_head(lp, new); if (err) break; if (!first_frag) goto no_data; } if (!first_frag) first_frag = p; if (pkt_len > size - copied) { /* User didn't give us a big enough buffer, * what to do? This is a pretty serious error. * * Since we haven't updated the RX ring head to * consume any of the packets, signal the error * to the user and just leave the RX ring alone. * * This seems the best behavior because this allows * a user of the LDC layer to start with a small * RX buffer for ldc_read() calls and use -EMSGSIZE * as a cue to enlarge it's read buffer. */ err = -EMSGSIZE; break; } /* Ok, we are gonna eat this one. */ new = rx_advance(lp, new); memcpy(buf, (lp->cfg.mode == LDC_MODE_UNRELIABLE ? p->u.u_data : p->u.r.r_data), pkt_len); buf += pkt_len; copied += pkt_len; if (p->env & LDC_STOP) break; no_data: if (new == lp->rx_tail) { err = rx_data_wait(lp, new); if (err) break; } } if (!err) err = rx_set_head(lp, new); if (err && first_frag) lp->rcv_nxt = first_frag->seqid - 1; if (!err) { err = copied; if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) send_data_ack(lp); } return err; } static const struct ldc_mode_ops nonraw_ops = { .write = write_nonraw, .read = read_nonraw, }; static int write_stream(struct ldc_channel *lp, const void *buf, unsigned int size) { if (size > lp->cfg.mtu) size = lp->cfg.mtu; return write_nonraw(lp, buf, size); } static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) { if (!lp->mssbuf_len) { int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); if (err < 0) return err; lp->mssbuf_len = err; lp->mssbuf_off = 0; } if (size > lp->mssbuf_len) size = lp->mssbuf_len; memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); lp->mssbuf_off += size; lp->mssbuf_len -= size; return size; } static const struct ldc_mode_ops stream_ops = { .write = write_stream, .read = read_stream, }; int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) { unsigned long flags; int err; if (!buf) return -EINVAL; if (!size) return 0; spin_lock_irqsave(&lp->lock, flags); if (lp->hs_state != LDC_HS_COMPLETE) err = -ENOTCONN; else err = lp->mops->write(lp, buf, size); spin_unlock_irqrestore(&lp->lock, flags); return err; } EXPORT_SYMBOL(ldc_write); int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) { unsigned long flags; int err; ldcdbg(RX, "%s: entered size=%d\n", __func__, size); if (!buf) return -EINVAL; if (!size) return 0; spin_lock_irqsave(&lp->lock, flags); if (lp->hs_state != LDC_HS_COMPLETE) err = -ENOTCONN; else err = lp->mops->read(lp, buf, size); spin_unlock_irqrestore(&lp->lock, flags); ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__, lp->cfg.mode, lp->rx_head, lp->rx_tail, err); return err; } EXPORT_SYMBOL(ldc_read); static u64 pagesize_code(void) { switch (PAGE_SIZE) { default: case (8ULL * 1024ULL): return 0; case (64ULL * 1024ULL): return 1; case (512ULL * 1024ULL): return 2; case (4ULL * 1024ULL * 1024ULL): return 3; case (32ULL * 1024ULL * 1024ULL): return 4; case (256ULL * 1024ULL * 1024ULL): return 5; } } static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) { return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | (index << PAGE_SHIFT) | page_offset); } static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, unsigned long npages) { long entry; entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, npages, NULL, (unsigned long)-1, 0); if (unlikely(entry == IOMMU_ERROR_CODE)) return NULL; return iommu->page_table + entry; } static u64 perm_to_mte(unsigned int map_perm) { u64 mte_base; mte_base = pagesize_code(); if (map_perm & LDC_MAP_SHADOW) { if (map_perm & LDC_MAP_R) mte_base |= LDC_MTE_COPY_R; if (map_perm & LDC_MAP_W) mte_base |= LDC_MTE_COPY_W; } if (map_perm & LDC_MAP_DIRECT) { if (map_perm & LDC_MAP_R) mte_base |= LDC_MTE_READ; if (map_perm & LDC_MAP_W) mte_base |= LDC_MTE_WRITE; if (map_perm & LDC_MAP_X) mte_base |= LDC_MTE_EXEC; } if (map_perm & LDC_MAP_IO) { if (map_perm & LDC_MAP_R) mte_base |= LDC_MTE_IOMMU_R; if (map_perm & LDC_MAP_W) mte_base |= LDC_MTE_IOMMU_W; } return mte_base; } static int pages_in_region(unsigned long base, long len) { int count = 0; do { unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; len -= (new - base); base = new; count++; } while (len > 0); return count; } struct cookie_state { struct ldc_mtable_entry *page_table; struct ldc_trans_cookie *cookies; u64 mte_base; u64 prev_cookie; u32 pte_idx; u32 nc; }; static void fill_cookies(struct cookie_state *sp, unsigned long pa, unsigned long off, unsigned long len) { do { unsigned long tlen, new = pa + PAGE_SIZE; u64 this_cookie; sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; tlen = PAGE_SIZE; if (off) tlen = PAGE_SIZE - off; if (tlen > len) tlen = len; this_cookie = make_cookie(sp->pte_idx, pagesize_code(), off); off = 0; if (this_cookie == sp->prev_cookie) { sp->cookies[sp->nc - 1].cookie_size += tlen; } else { sp->cookies[sp->nc].cookie_addr = this_cookie; sp->cookies[sp->nc].cookie_size = tlen; sp->nc++; } sp->prev_cookie = this_cookie + tlen; sp->pte_idx++; len -= tlen; pa = new; } while (len > 0); } static int sg_count_one(struct scatterlist *sg) { unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; long len = sg->length; if ((sg->offset | len) & (8UL - 1)) return -EFAULT; return pages_in_region(base + sg->offset, len); } static int sg_count_pages(struct scatterlist *sg, int num_sg) { int count; int i; count = 0; for (i = 0; i < num_sg; i++) { int err = sg_count_one(sg + i); if (err < 0) return err; count += err; } return count; } int ldc_map_sg(struct ldc_channel *lp, struct scatterlist *sg, int num_sg, struct ldc_trans_cookie *cookies, int ncookies, unsigned int map_perm) { unsigned long i, npages; struct ldc_mtable_entry *base; struct cookie_state state; struct ldc_iommu *iommu; int err; struct scatterlist *s; if (map_perm & ~LDC_MAP_ALL) return -EINVAL; err = sg_count_pages(sg, num_sg); if (err < 0) return err; npages = err; if (err > ncookies) return -EMSGSIZE; iommu = &lp->iommu; base = alloc_npages(iommu, npages); if (!base) return -ENOMEM; state.page_table = iommu->page_table; state.cookies = cookies; state.mte_base = perm_to_mte(map_perm); state.prev_cookie = ~(u64)0; state.pte_idx = (base - iommu->page_table); state.nc = 0; for_each_sg(sg, s, num_sg, i) { fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT, s->offset, s->length); } return state.nc; } EXPORT_SYMBOL(ldc_map_sg); int ldc_map_single(struct ldc_channel *lp, void *buf, unsigned int len, struct ldc_trans_cookie *cookies, int ncookies, unsigned int map_perm) { unsigned long npages, pa; struct ldc_mtable_entry *base; struct cookie_state state; struct ldc_iommu *iommu; if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) return -EINVAL; pa = __pa(buf); if ((pa | len) & (8UL - 1)) return -EFAULT; npages = pages_in_region(pa, len); iommu = &lp->iommu; base = alloc_npages(iommu, npages); if (!base) return -ENOMEM; state.page_table = iommu->page_table; state.cookies = cookies; state.mte_base = perm_to_mte(map_perm); state.prev_cookie = ~(u64)0; state.pte_idx = (base - iommu->page_table); state.nc = 0; fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); BUG_ON(state.nc > ncookies); return state.nc; } EXPORT_SYMBOL(ldc_map_single); static void free_npages(unsigned long id, struct ldc_iommu *iommu, u64 cookie, u64 size) { unsigned long npages, entry; npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; entry = ldc_cookie_to_index(cookie, iommu); ldc_demap(iommu, id, cookie, entry, npages); iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); } void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, int ncookies) { struct ldc_iommu *iommu = &lp->iommu; int i; unsigned long flags; spin_lock_irqsave(&iommu->lock, flags); for (i = 0; i < ncookies; i++) { u64 addr = cookies[i].cookie_addr; u64 size = cookies[i].cookie_size; free_npages(lp->id, iommu, addr, size); } spin_unlock_irqrestore(&iommu->lock, flags); } EXPORT_SYMBOL(ldc_unmap); int ldc_copy(struct ldc_channel *lp, int copy_dir, void *buf, unsigned int len, unsigned long offset, struct ldc_trans_cookie *cookies, int ncookies) { unsigned int orig_len; unsigned long ra; int i; if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", lp->id, copy_dir); return -EINVAL; } ra = __pa(buf); if ((ra | len | offset) & (8UL - 1)) { printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " "ra[%lx] len[%x] offset[%lx]\n", lp->id, ra, len, offset); return -EFAULT; } if (lp->hs_state != LDC_HS_COMPLETE || (lp->flags & LDC_FLAG_RESET)) { printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " "flags[%x]\n", lp->id, lp->hs_state, lp->flags); return -ECONNRESET; } orig_len = len; for (i = 0; i < ncookies; i++) { unsigned long cookie_raddr = cookies[i].cookie_addr; unsigned long this_len = cookies[i].cookie_size; unsigned long actual_len; if (unlikely(offset)) { unsigned long this_off = offset; if (this_off > this_len) this_off = this_len; offset -= this_off; this_len -= this_off; if (!this_len) continue; cookie_raddr += this_off; } if (this_len > len) this_len = len; while (1) { unsigned long hv_err; hv_err = sun4v_ldc_copy(lp->id, copy_dir, cookie_raddr, ra, this_len, &actual_len); if (unlikely(hv_err)) { printk(KERN_ERR PFX "ldc_copy: ID[%lu] " "HV error %lu\n", lp->id, hv_err); if (lp->hs_state != LDC_HS_COMPLETE || (lp->flags & LDC_FLAG_RESET)) return -ECONNRESET; else return -EFAULT; } cookie_raddr += actual_len; ra += actual_len; len -= actual_len; if (actual_len == this_len) break; this_len -= actual_len; } if (!len) break; } /* It is caller policy what to do about short copies. * For example, a networking driver can declare the * packet a runt and drop it. */ return orig_len - len; } EXPORT_SYMBOL(ldc_copy); void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, struct ldc_trans_cookie *cookies, int *ncookies, unsigned int map_perm) { void *buf; int err; if (len & (8UL - 1)) return ERR_PTR(-EINVAL); buf = kzalloc(len, GFP_ATOMIC); if (!buf) return ERR_PTR(-ENOMEM); err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); if (err < 0) { kfree(buf); return ERR_PTR(err); } *ncookies = err; return buf; } EXPORT_SYMBOL(ldc_alloc_exp_dring); void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, struct ldc_trans_cookie *cookies, int ncookies) { ldc_unmap(lp, cookies, ncookies); kfree(buf); } EXPORT_SYMBOL(ldc_free_exp_dring); static int __init ldc_init(void) { unsigned long major, minor; struct mdesc_handle *hp; const u64 *v; int err; u64 mp; hp = mdesc_grab(); if (!hp) return -ENODEV; mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); err = -ENODEV; if (mp == MDESC_NODE_NULL) goto out; v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); if (!v) goto out; major = 1; minor = 0; if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); goto out; } printk(KERN_INFO "%s", version); if (!*v) { printk(KERN_INFO PFX "Domaining disabled.\n"); goto out; } ldom_domaining_enabled = 1; err = 0; out: mdesc_release(hp); return err; } core_initcall(ldc_init);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1