| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Trond Myklebust | 5383 | 87.06% | 153 | 74.27% |
| Chuck Lever | 265 | 4.29% | 5 | 2.43% |
| Neil Brown | 146 | 2.36% | 3 | 1.46% |
| Jeff Layton | 86 | 1.39% | 8 | 3.88% |
| David Howells | 69 | 1.12% | 5 | 2.43% |
| Alexandros Batsakis | 42 | 0.68% | 2 | 0.97% |
| Linus Torvalds | 38 | 0.61% | 2 | 0.97% |
| Peng Tao | 38 | 0.61% | 1 | 0.49% |
| Benjamin Coddington | 27 | 0.44% | 1 | 0.49% |
| Dai Ngo | 23 | 0.37% | 1 | 0.49% |
| Linus Torvalds (pre-git) | 13 | 0.21% | 7 | 3.40% |
| Anna Schumaker | 12 | 0.19% | 4 | 1.94% |
| Scott Mayhew | 8 | 0.13% | 1 | 0.49% |
| Olga Kornievskaia | 8 | 0.13% | 3 | 1.46% |
| Andy Adamson | 5 | 0.08% | 1 | 0.49% |
| Benny Halevy | 4 | 0.06% | 1 | 0.49% |
| Panagiotis Issaris | 4 | 0.06% | 1 | 0.49% |
| Andreas Gruenbacher | 3 | 0.05% | 1 | 0.49% |
| Weston Andros Adamson | 2 | 0.03% | 1 | 0.49% |
| Lai Jiangshan | 2 | 0.03% | 1 | 0.49% |
| Marius Aamodt Eriksen | 2 | 0.03% | 1 | 0.49% |
| van der Linden, Frank | 1 | 0.02% | 1 | 0.49% |
| Thomas Gleixner | 1 | 0.02% | 1 | 0.49% |
| Harvey Harrison | 1 | 0.02% | 1 | 0.49% |
| Total | 6183 | 206 |
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503
// SPDX-License-Identifier: GPL-2.0-only /* * linux/fs/nfs/delegation.c * * Copyright (C) 2004 Trond Myklebust * * NFS file delegation management * */ #include <linux/completion.h> #include <linux/kthread.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/iversion.h> #include <linux/nfs4.h> #include <linux/nfs_fs.h> #include <linux/nfs_xdr.h> #include "nfs4_fs.h" #include "nfs4session.h" #include "delegation.h" #include "internal.h" #include "nfs4trace.h" #define NFS_DEFAULT_DELEGATION_WATERMARK (5000U) static atomic_long_t nfs_active_delegations; static unsigned nfs_delegation_watermark = NFS_DEFAULT_DELEGATION_WATERMARK; static void __nfs_free_delegation(struct nfs_delegation *delegation) { put_cred(delegation->cred); delegation->cred = NULL; kfree_rcu(delegation, rcu); } static void nfs_mark_delegation_revoked(struct nfs_delegation *delegation) { if (!test_and_set_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { delegation->stateid.type = NFS4_INVALID_STATEID_TYPE; atomic_long_dec(&nfs_active_delegations); if (!test_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) nfs_clear_verifier_delegated(delegation->inode); } } static struct nfs_delegation *nfs_get_delegation(struct nfs_delegation *delegation) { refcount_inc(&delegation->refcount); return delegation; } static void nfs_put_delegation(struct nfs_delegation *delegation) { if (refcount_dec_and_test(&delegation->refcount)) __nfs_free_delegation(delegation); } static void nfs_free_delegation(struct nfs_delegation *delegation) { nfs_mark_delegation_revoked(delegation); nfs_put_delegation(delegation); } /** * nfs_mark_delegation_referenced - set delegation's REFERENCED flag * @delegation: delegation to process * */ void nfs_mark_delegation_referenced(struct nfs_delegation *delegation) { set_bit(NFS_DELEGATION_REFERENCED, &delegation->flags); } static void nfs_mark_return_delegation(struct nfs_server *server, struct nfs_delegation *delegation) { set_bit(NFS_DELEGATION_RETURN, &delegation->flags); set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); } static bool nfs4_is_valid_delegation(const struct nfs_delegation *delegation, fmode_t type) { if (delegation != NULL && (delegation->type & type) == type && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) && !test_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) return true; return false; } struct nfs_delegation *nfs4_get_valid_delegation(const struct inode *inode) { struct nfs_delegation *delegation; delegation = rcu_dereference(NFS_I(inode)->delegation); if (nfs4_is_valid_delegation(delegation, 0)) return delegation; return NULL; } static int nfs4_do_check_delegation(struct inode *inode, fmode_t type, int flags, bool mark) { struct nfs_delegation *delegation; int ret = 0; type &= FMODE_READ|FMODE_WRITE; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (nfs4_is_valid_delegation(delegation, type)) { if (mark) nfs_mark_delegation_referenced(delegation); ret = 1; if ((flags & NFS_DELEGATION_FLAG_TIME) && !test_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags)) ret = 0; } rcu_read_unlock(); return ret; } /** * nfs4_have_delegation - check if inode has a delegation, mark it * NFS_DELEGATION_REFERENCED if there is one. * @inode: inode to check * @type: delegation types to check for * @flags: various modifiers * * Returns one if inode has the indicated delegation, otherwise zero. */ int nfs4_have_delegation(struct inode *inode, fmode_t type, int flags) { return nfs4_do_check_delegation(inode, type, flags, true); } /* * nfs4_check_delegation - check if inode has a delegation, do not mark * NFS_DELEGATION_REFERENCED if it has one. */ int nfs4_check_delegation(struct inode *inode, fmode_t type) { return nfs4_do_check_delegation(inode, type, 0, false); } static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_stateid *stateid) { struct inode *inode = state->inode; struct file_lock *fl; struct file_lock_context *flctx = locks_inode_context(inode); struct list_head *list; int status = 0; if (flctx == NULL) goto out; list = &flctx->flc_posix; spin_lock(&flctx->flc_lock); restart: for_each_file_lock(fl, list) { if (nfs_file_open_context(fl->c.flc_file)->state != state) continue; spin_unlock(&flctx->flc_lock); status = nfs4_lock_delegation_recall(fl, state, stateid); if (status < 0) goto out; spin_lock(&flctx->flc_lock); } if (list == &flctx->flc_posix) { list = &flctx->flc_flock; goto restart; } spin_unlock(&flctx->flc_lock); out: return status; } static int nfs_delegation_claim_opens(struct inode *inode, const nfs4_stateid *stateid, fmode_t type) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_open_context *ctx; struct nfs4_state_owner *sp; struct nfs4_state *state; int err; again: rcu_read_lock(); list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { state = ctx->state; if (state == NULL) continue; if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) continue; if (!nfs4_valid_open_stateid(state)) continue; if (!nfs4_stateid_match(&state->stateid, stateid)) continue; if (!get_nfs_open_context(ctx)) continue; rcu_read_unlock(); sp = state->owner; /* Block nfs4_proc_unlck */ mutex_lock(&sp->so_delegreturn_mutex); err = nfs4_open_delegation_recall(ctx, state, stateid); if (!err) err = nfs_delegation_claim_locks(state, stateid); mutex_unlock(&sp->so_delegreturn_mutex); put_nfs_open_context(ctx); if (err != 0) return err; goto again; } rcu_read_unlock(); return 0; } /** * nfs_inode_reclaim_delegation - process a delegation reclaim request * @inode: inode to process * @cred: credential to use for request * @type: delegation type * @stateid: delegation stateid * @pagemod_limit: write delegation "space_limit" * @deleg_type: raw delegation type * */ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit, u32 deleg_type) { struct nfs_delegation *delegation; const struct cred *oldcred = NULL; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation != NULL) { spin_lock(&delegation->lock); nfs4_stateid_copy(&delegation->stateid, stateid); delegation->type = type; delegation->pagemod_limit = pagemod_limit; oldcred = delegation->cred; delegation->cred = get_cred(cred); switch (deleg_type) { case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: set_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); break; default: clear_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); } clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); if (test_and_clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) atomic_long_inc(&nfs_active_delegations); spin_unlock(&delegation->lock); rcu_read_unlock(); put_cred(oldcred); trace_nfs4_reclaim_delegation(inode, type); } else { rcu_read_unlock(); nfs_inode_set_delegation(inode, cred, type, stateid, pagemod_limit, deleg_type); } } static int nfs_do_return_delegation(struct inode *inode, struct nfs_delegation *delegation, int issync) { const struct cred *cred; int res = 0; if (!test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { spin_lock(&delegation->lock); cred = get_cred(delegation->cred); spin_unlock(&delegation->lock); res = nfs4_proc_delegreturn(inode, cred, &delegation->stateid, delegation, issync); put_cred(cred); } return res; } static struct inode *nfs_delegation_grab_inode(struct nfs_delegation *delegation) { struct inode *inode = NULL; spin_lock(&delegation->lock); if (delegation->inode != NULL) inode = igrab(delegation->inode); if (!inode) set_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags); spin_unlock(&delegation->lock); return inode; } static struct nfs_delegation * nfs_start_delegation_return_locked(struct nfs_inode *nfsi) { struct nfs_delegation *ret = NULL; struct nfs_delegation *delegation = rcu_dereference(nfsi->delegation); if (delegation == NULL) goto out; spin_lock(&delegation->lock); if (!test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { clear_bit(NFS_DELEGATION_RETURN_DELAYED, &delegation->flags); /* Refcount matched in nfs_end_delegation_return() */ ret = nfs_get_delegation(delegation); } spin_unlock(&delegation->lock); if (ret) nfs_clear_verifier_delegated(&nfsi->vfs_inode); out: return ret; } static struct nfs_delegation * nfs_start_delegation_return(struct nfs_inode *nfsi) { struct nfs_delegation *delegation; rcu_read_lock(); delegation = nfs_start_delegation_return_locked(nfsi); rcu_read_unlock(); return delegation; } static void nfs_abort_delegation_return(struct nfs_delegation *delegation, struct nfs_client *clp, int err) { spin_lock(&delegation->lock); clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); if (err == -EAGAIN) { set_bit(NFS_DELEGATION_RETURN_DELAYED, &delegation->flags); set_bit(NFS4CLNT_DELEGRETURN_DELAYED, &clp->cl_state); } spin_unlock(&delegation->lock); } static struct nfs_delegation * nfs_detach_delegation_locked(struct nfs_inode *nfsi, struct nfs_delegation *delegation, struct nfs_client *clp) { struct nfs_delegation *deleg_cur = rcu_dereference_protected(nfsi->delegation, lockdep_is_held(&clp->cl_lock)); if (deleg_cur == NULL || delegation != deleg_cur) return NULL; spin_lock(&delegation->lock); if (!delegation->inode) { spin_unlock(&delegation->lock); return NULL; } list_del_rcu(&delegation->super_list); delegation->inode = NULL; rcu_assign_pointer(nfsi->delegation, NULL); spin_unlock(&delegation->lock); return delegation; } static struct nfs_delegation *nfs_detach_delegation(struct nfs_inode *nfsi, struct nfs_delegation *delegation, struct nfs_server *server) { struct nfs_client *clp = server->nfs_client; spin_lock(&clp->cl_lock); delegation = nfs_detach_delegation_locked(nfsi, delegation, clp); spin_unlock(&clp->cl_lock); return delegation; } static struct nfs_delegation * nfs_inode_detach_delegation(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_server *server = NFS_SERVER(inode); struct nfs_delegation *delegation; rcu_read_lock(); delegation = rcu_dereference(nfsi->delegation); if (delegation != NULL) delegation = nfs_detach_delegation(nfsi, delegation, server); rcu_read_unlock(); return delegation; } static void nfs_update_delegation_cred(struct nfs_delegation *delegation, const struct cred *cred) { const struct cred *old; if (cred_fscmp(delegation->cred, cred) != 0) { old = xchg(&delegation->cred, get_cred(cred)); put_cred(old); } } static void nfs_update_inplace_delegation(struct nfs_delegation *delegation, const struct nfs_delegation *update) { if (nfs4_stateid_is_newer(&update->stateid, &delegation->stateid)) { delegation->stateid.seqid = update->stateid.seqid; smp_wmb(); delegation->type = update->type; delegation->pagemod_limit = update->pagemod_limit; if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { delegation->change_attr = update->change_attr; nfs_update_delegation_cred(delegation, update->cred); /* smp_mb__before_atomic() is implicit due to xchg() */ clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags); atomic_long_inc(&nfs_active_delegations); } } } /** * nfs_inode_set_delegation - set up a delegation on an inode * @inode: inode to which delegation applies * @cred: cred to use for subsequent delegation processing * @type: delegation type * @stateid: delegation stateid * @pagemod_limit: write delegation "space_limit" * @deleg_type: raw delegation type * * Returns zero on success, or a negative errno value. */ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit, u32 deleg_type) { struct nfs_server *server = NFS_SERVER(inode); struct nfs_client *clp = server->nfs_client; struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation, *old_delegation; struct nfs_delegation *freeme = NULL; int status = 0; delegation = kmalloc(sizeof(*delegation), GFP_KERNEL_ACCOUNT); if (delegation == NULL) return -ENOMEM; nfs4_stateid_copy(&delegation->stateid, stateid); refcount_set(&delegation->refcount, 1); delegation->type = type; delegation->pagemod_limit = pagemod_limit; delegation->change_attr = inode_peek_iversion_raw(inode); delegation->cred = get_cred(cred); delegation->inode = inode; delegation->flags = 1<<NFS_DELEGATION_REFERENCED; switch (deleg_type) { case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: delegation->flags |= BIT(NFS_DELEGATION_DELEGTIME); } delegation->test_gen = 0; spin_lock_init(&delegation->lock); spin_lock(&clp->cl_lock); old_delegation = rcu_dereference_protected(nfsi->delegation, lockdep_is_held(&clp->cl_lock)); if (old_delegation == NULL) goto add_new; /* Is this an update of the existing delegation? */ if (nfs4_stateid_match_other(&old_delegation->stateid, &delegation->stateid)) { spin_lock(&old_delegation->lock); nfs_update_inplace_delegation(old_delegation, delegation); spin_unlock(&old_delegation->lock); goto out; } if (!test_bit(NFS_DELEGATION_REVOKED, &old_delegation->flags)) { /* * Deal with broken servers that hand out two * delegations for the same file. * Allow for upgrades to a WRITE delegation, but * nothing else. */ dfprintk(FILE, "%s: server %s handed out " "a duplicate delegation!\n", __func__, clp->cl_hostname); if (delegation->type == old_delegation->type || !(delegation->type & FMODE_WRITE)) { freeme = delegation; delegation = NULL; goto out; } if (test_and_set_bit(NFS_DELEGATION_RETURNING, &old_delegation->flags)) goto out; } freeme = nfs_detach_delegation_locked(nfsi, old_delegation, clp); if (freeme == NULL) goto out; add_new: /* * If we didn't revalidate the change attribute before setting * the delegation, then pre-emptively ask for a full attribute * cache revalidation. */ spin_lock(&inode->i_lock); if (NFS_I(inode)->cache_validity & NFS_INO_INVALID_CHANGE) nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME | NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE | NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK | NFS_INO_INVALID_OTHER | NFS_INO_INVALID_DATA | NFS_INO_INVALID_ACCESS | NFS_INO_INVALID_ACL | NFS_INO_INVALID_XATTR); spin_unlock(&inode->i_lock); list_add_tail_rcu(&delegation->super_list, &server->delegations); rcu_assign_pointer(nfsi->delegation, delegation); delegation = NULL; atomic_long_inc(&nfs_active_delegations); trace_nfs4_set_delegation(inode, type); /* If we hold writebacks and have delegated mtime then update */ if (deleg_type == NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG && nfs_have_writebacks(inode)) nfs_update_delegated_mtime(inode); out: spin_unlock(&clp->cl_lock); if (delegation != NULL) __nfs_free_delegation(delegation); if (freeme != NULL) { nfs_do_return_delegation(inode, freeme, 0); nfs_free_delegation(freeme); } return status; } /* * Basic procedure for returning a delegation to the server */ static int nfs_end_delegation_return(struct inode *inode, struct nfs_delegation *delegation, int issync) { struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; unsigned int mode = O_WRONLY | O_RDWR; int err = 0; if (delegation == NULL) return 0; if (!issync) mode |= O_NONBLOCK; /* Recall of any remaining application leases */ err = break_lease(inode, mode); while (err == 0) { if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) break; err = nfs_delegation_claim_opens(inode, &delegation->stateid, delegation->type); if (!issync || err != -EAGAIN) break; /* * Guard against state recovery */ err = nfs4_wait_clnt_recover(clp); } if (err) { nfs_abort_delegation_return(delegation, clp, err); goto out; } err = nfs_do_return_delegation(inode, delegation, issync); out: /* Refcount matched in nfs_start_delegation_return_locked() */ nfs_put_delegation(delegation); return err; } static bool nfs_delegation_need_return(struct nfs_delegation *delegation) { bool ret = false; if (test_and_clear_bit(NFS_DELEGATION_RETURN, &delegation->flags)) ret = true; else if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) { struct inode *inode; spin_lock(&delegation->lock); inode = delegation->inode; if (inode && list_empty(&NFS_I(inode)->open_files)) ret = true; spin_unlock(&delegation->lock); } if (ret) clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) || test_bit(NFS_DELEGATION_RETURN_DELAYED, &delegation->flags) || test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) ret = false; return ret; } static int nfs_server_return_marked_delegations(struct nfs_server *server, void __always_unused *data) { struct nfs_delegation *delegation; struct nfs_delegation *prev; struct inode *inode; struct inode *place_holder = NULL; struct nfs_delegation *place_holder_deleg = NULL; int err = 0; restart: /* * To avoid quadratic looping we hold a reference * to an inode place_holder. Each time we restart, we * list delegation in the server from the delegations * of that inode. * prev is an RCU-protected pointer to a delegation which * wasn't marked for return and might be a good choice for * the next place_holder. */ prev = NULL; delegation = NULL; rcu_read_lock(); if (place_holder) delegation = rcu_dereference(NFS_I(place_holder)->delegation); if (!delegation || delegation != place_holder_deleg) delegation = list_entry_rcu(server->delegations.next, struct nfs_delegation, super_list); list_for_each_entry_from_rcu(delegation, &server->delegations, super_list) { struct inode *to_put = NULL; if (test_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags)) continue; if (!nfs_delegation_need_return(delegation)) { if (nfs4_is_valid_delegation(delegation, 0)) prev = delegation; continue; } inode = nfs_delegation_grab_inode(delegation); if (inode == NULL) continue; if (prev) { struct inode *tmp = nfs_delegation_grab_inode(prev); if (tmp) { to_put = place_holder; place_holder = tmp; place_holder_deleg = prev; } } delegation = nfs_start_delegation_return_locked(NFS_I(inode)); rcu_read_unlock(); iput(to_put); err = nfs_end_delegation_return(inode, delegation, 0); iput(inode); cond_resched(); if (!err) goto restart; set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); goto out; } rcu_read_unlock(); out: iput(place_holder); return err; } static bool nfs_server_clear_delayed_delegations(struct nfs_server *server) { struct nfs_delegation *d; bool ret = false; list_for_each_entry_rcu (d, &server->delegations, super_list) { if (!test_bit(NFS_DELEGATION_RETURN_DELAYED, &d->flags)) continue; nfs_mark_return_delegation(server, d); clear_bit(NFS_DELEGATION_RETURN_DELAYED, &d->flags); ret = true; } return ret; } static bool nfs_client_clear_delayed_delegations(struct nfs_client *clp) { struct nfs_server *server; bool ret = false; if (!test_and_clear_bit(NFS4CLNT_DELEGRETURN_DELAYED, &clp->cl_state)) goto out; rcu_read_lock(); list_for_each_entry_rcu (server, &clp->cl_superblocks, client_link) { if (nfs_server_clear_delayed_delegations(server)) ret = true; } rcu_read_unlock(); out: return ret; } /** * nfs_client_return_marked_delegations - return previously marked delegations * @clp: nfs_client to process * * Note that this function is designed to be called by the state * manager thread. For this reason, it cannot flush the dirty data, * since that could deadlock in case of a state recovery error. * * Returns zero on success, or a negative errno value. */ int nfs_client_return_marked_delegations(struct nfs_client *clp) { int err = nfs_client_for_each_server( clp, nfs_server_return_marked_delegations, NULL); if (err) return err; /* If a return was delayed, sleep to prevent hard looping */ if (nfs_client_clear_delayed_delegations(clp)) ssleep(1); return 0; } /** * nfs_inode_evict_delegation - return delegation, don't reclaim opens * @inode: inode to process * * Does not protect against delegation reclaims, therefore really only safe * to be called from nfs4_clear_inode(). Guaranteed to always free * the delegation structure. */ void nfs_inode_evict_delegation(struct inode *inode) { struct nfs_delegation *delegation; delegation = nfs_inode_detach_delegation(inode); if (delegation != NULL) { set_bit(NFS_DELEGATION_RETURNING, &delegation->flags); set_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags); nfs_do_return_delegation(inode, delegation, 1); nfs_free_delegation(delegation); } } /** * nfs4_inode_return_delegation - synchronously return a delegation * @inode: inode to process * * This routine will always flush any dirty data to disk on the * assumption that if we need to return the delegation, then * we should stop caching. * * Returns zero on success, or a negative errno value. */ int nfs4_inode_return_delegation(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation; delegation = nfs_start_delegation_return(nfsi); if (delegation != NULL) { /* Synchronous recall of any application leases */ break_lease(inode, O_WRONLY | O_RDWR); if (S_ISREG(inode->i_mode)) nfs_wb_all(inode); return nfs_end_delegation_return(inode, delegation, 1); } return 0; } /** * nfs4_inode_return_delegation_on_close - asynchronously return a delegation * @inode: inode to process * * This routine is called on file close in order to determine if the * inode delegation needs to be returned immediately. */ void nfs4_inode_return_delegation_on_close(struct inode *inode) { struct nfs_delegation *delegation; struct nfs_delegation *ret = NULL; if (!inode) return; rcu_read_lock(); delegation = nfs4_get_valid_delegation(inode); if (!delegation) goto out; if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) || atomic_long_read(&nfs_active_delegations) >= nfs_delegation_watermark) { spin_lock(&delegation->lock); if (delegation->inode && list_empty(&NFS_I(inode)->open_files) && !test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); /* Refcount matched in nfs_end_delegation_return() */ ret = nfs_get_delegation(delegation); } spin_unlock(&delegation->lock); if (ret) nfs_clear_verifier_delegated(inode); } out: rcu_read_unlock(); nfs_end_delegation_return(inode, ret, 0); } /** * nfs4_inode_make_writeable * @inode: pointer to inode * * Make the inode writeable by returning the delegation if necessary * * Returns zero on success, or a negative errno value. */ int nfs4_inode_make_writeable(struct inode *inode) { struct nfs_delegation *delegation; rcu_read_lock(); delegation = nfs4_get_valid_delegation(inode); if (delegation == NULL || (nfs4_has_session(NFS_SERVER(inode)->nfs_client) && (delegation->type & FMODE_WRITE))) { rcu_read_unlock(); return 0; } rcu_read_unlock(); return nfs4_inode_return_delegation(inode); } static void nfs_mark_return_if_closed_delegation(struct nfs_server *server, struct nfs_delegation *delegation) { set_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); } static bool nfs_server_mark_return_all_delegations(struct nfs_server *server) { struct nfs_delegation *delegation; bool ret = false; list_for_each_entry_rcu(delegation, &server->delegations, super_list) { nfs_mark_return_delegation(server, delegation); ret = true; } return ret; } static void nfs_client_mark_return_all_delegations(struct nfs_client *clp) { struct nfs_server *server; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) nfs_server_mark_return_all_delegations(server); rcu_read_unlock(); } static void nfs_delegation_run_state_manager(struct nfs_client *clp) { if (test_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state)) nfs4_schedule_state_manager(clp); } /** * nfs_expire_all_delegations * @clp: client to process * */ void nfs_expire_all_delegations(struct nfs_client *clp) { nfs_client_mark_return_all_delegations(clp); nfs_delegation_run_state_manager(clp); } /** * nfs_server_return_all_delegations - return delegations for one superblock * @server: pointer to nfs_server to process * */ void nfs_server_return_all_delegations(struct nfs_server *server) { struct nfs_client *clp = server->nfs_client; bool need_wait; if (clp == NULL) return; rcu_read_lock(); need_wait = nfs_server_mark_return_all_delegations(server); rcu_read_unlock(); if (need_wait) { nfs4_schedule_state_manager(clp); nfs4_wait_clnt_recover(clp); } } static void nfs_mark_return_unused_delegation_types(struct nfs_server *server, fmode_t flags) { struct nfs_delegation *delegation; list_for_each_entry_rcu(delegation, &server->delegations, super_list) { if ((delegation->type == (FMODE_READ|FMODE_WRITE)) && !(flags & FMODE_WRITE)) continue; if (delegation->type & flags) nfs_mark_return_if_closed_delegation(server, delegation); } } static void nfs_client_mark_return_unused_delegation_types(struct nfs_client *clp, fmode_t flags) { struct nfs_server *server; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) nfs_mark_return_unused_delegation_types(server, flags); rcu_read_unlock(); } static void nfs_revoke_delegation(struct inode *inode, const nfs4_stateid *stateid) { struct nfs_delegation *delegation; nfs4_stateid tmp; bool ret = false; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation == NULL) goto out; if (stateid == NULL) { nfs4_stateid_copy(&tmp, &delegation->stateid); stateid = &tmp; } else { if (!nfs4_stateid_match_other(stateid, &delegation->stateid)) goto out; spin_lock(&delegation->lock); if (stateid->seqid) { if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) { spin_unlock(&delegation->lock); goto out; } delegation->stateid.seqid = stateid->seqid; } spin_unlock(&delegation->lock); } nfs_mark_delegation_revoked(delegation); ret = true; out: rcu_read_unlock(); if (ret) nfs_inode_find_state_and_recover(inode, stateid); } void nfs_remove_bad_delegation(struct inode *inode, const nfs4_stateid *stateid) { nfs_revoke_delegation(inode, stateid); } EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation); void nfs_delegation_mark_returned(struct inode *inode, const nfs4_stateid *stateid) { struct nfs_delegation *delegation; if (!inode) return; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (!delegation) goto out_rcu_unlock; spin_lock(&delegation->lock); if (!nfs4_stateid_match_other(stateid, &delegation->stateid)) goto out_spin_unlock; if (stateid->seqid) { /* If delegation->stateid is newer, dont mark as returned */ if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) goto out_clear_returning; if (delegation->stateid.seqid != stateid->seqid) delegation->stateid.seqid = stateid->seqid; } nfs_mark_delegation_revoked(delegation); clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); spin_unlock(&delegation->lock); if (nfs_detach_delegation(NFS_I(inode), delegation, NFS_SERVER(inode))) nfs_put_delegation(delegation); goto out_rcu_unlock; out_clear_returning: clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); out_spin_unlock: spin_unlock(&delegation->lock); out_rcu_unlock: rcu_read_unlock(); nfs_inode_find_state_and_recover(inode, stateid); } /** * nfs_expire_unused_delegation_types * @clp: client to process * @flags: delegation types to expire * */ void nfs_expire_unused_delegation_types(struct nfs_client *clp, fmode_t flags) { nfs_client_mark_return_unused_delegation_types(clp, flags); nfs_delegation_run_state_manager(clp); } static void nfs_mark_return_unreferenced_delegations(struct nfs_server *server) { struct nfs_delegation *delegation; list_for_each_entry_rcu(delegation, &server->delegations, super_list) { if (test_and_clear_bit(NFS_DELEGATION_REFERENCED, &delegation->flags)) continue; nfs_mark_return_if_closed_delegation(server, delegation); } } /** * nfs_expire_unreferenced_delegations - Eliminate unused delegations * @clp: nfs_client to process * */ void nfs_expire_unreferenced_delegations(struct nfs_client *clp) { struct nfs_server *server; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) nfs_mark_return_unreferenced_delegations(server); rcu_read_unlock(); nfs_delegation_run_state_manager(clp); } /** * nfs_async_inode_return_delegation - asynchronously return a delegation * @inode: inode to process * @stateid: state ID information * * Returns zero on success, or a negative errno value. */ int nfs_async_inode_return_delegation(struct inode *inode, const nfs4_stateid *stateid) { struct nfs_server *server = NFS_SERVER(inode); struct nfs_client *clp = server->nfs_client; struct nfs_delegation *delegation; rcu_read_lock(); delegation = nfs4_get_valid_delegation(inode); if (delegation == NULL) goto out_enoent; if (stateid != NULL && !clp->cl_mvops->match_stateid(&delegation->stateid, stateid)) goto out_enoent; nfs_mark_return_delegation(server, delegation); rcu_read_unlock(); /* If there are any application leases or delegations, recall them */ break_lease(inode, O_WRONLY | O_RDWR | O_NONBLOCK); nfs_delegation_run_state_manager(clp); return 0; out_enoent: rcu_read_unlock(); return -ENOENT; } static struct inode * nfs_delegation_find_inode_server(struct nfs_server *server, const struct nfs_fh *fhandle) { struct nfs_delegation *delegation; struct super_block *freeme = NULL; struct inode *res = NULL; list_for_each_entry_rcu(delegation, &server->delegations, super_list) { spin_lock(&delegation->lock); if (delegation->inode != NULL && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) && nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) { if (nfs_sb_active(server->super)) { freeme = server->super; res = igrab(delegation->inode); } spin_unlock(&delegation->lock); if (res != NULL) return res; if (freeme) { rcu_read_unlock(); nfs_sb_deactive(freeme); rcu_read_lock(); } return ERR_PTR(-EAGAIN); } spin_unlock(&delegation->lock); } return ERR_PTR(-ENOENT); } /** * nfs_delegation_find_inode - retrieve the inode associated with a delegation * @clp: client state handle * @fhandle: filehandle from a delegation recall * * Returns pointer to inode matching "fhandle," or NULL if a matching inode * cannot be found. */ struct inode *nfs_delegation_find_inode(struct nfs_client *clp, const struct nfs_fh *fhandle) { struct nfs_server *server; struct inode *res; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { res = nfs_delegation_find_inode_server(server, fhandle); if (res != ERR_PTR(-ENOENT)) { rcu_read_unlock(); return res; } } rcu_read_unlock(); return ERR_PTR(-ENOENT); } static void nfs_delegation_mark_reclaim_server(struct nfs_server *server) { struct nfs_delegation *delegation; list_for_each_entry_rcu(delegation, &server->delegations, super_list) { /* * If the delegation may have been admin revoked, then we * cannot reclaim it. */ if (test_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags)) continue; set_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); } } /** * nfs_delegation_mark_reclaim - mark all delegations as needing to be reclaimed * @clp: nfs_client to process * */ void nfs_delegation_mark_reclaim(struct nfs_client *clp) { struct nfs_server *server; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) nfs_delegation_mark_reclaim_server(server); rcu_read_unlock(); } static int nfs_server_reap_unclaimed_delegations(struct nfs_server *server, void __always_unused *data) { struct nfs_delegation *delegation; struct inode *inode; restart: rcu_read_lock(); list_for_each_entry_rcu(delegation, &server->delegations, super_list) { if (test_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags) || test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) || test_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags) == 0) continue; inode = nfs_delegation_grab_inode(delegation); if (inode == NULL) continue; delegation = nfs_start_delegation_return_locked(NFS_I(inode)); rcu_read_unlock(); if (delegation != NULL) { if (nfs_detach_delegation(NFS_I(inode), delegation, server) != NULL) nfs_free_delegation(delegation); /* Match nfs_start_delegation_return_locked */ nfs_put_delegation(delegation); } iput(inode); cond_resched(); goto restart; } rcu_read_unlock(); return 0; } /** * nfs_delegation_reap_unclaimed - reap unclaimed delegations after reboot recovery is done * @clp: nfs_client to process * */ void nfs_delegation_reap_unclaimed(struct nfs_client *clp) { nfs_client_for_each_server(clp, nfs_server_reap_unclaimed_delegations, NULL); } static inline bool nfs4_server_rebooted(const struct nfs_client *clp) { return (clp->cl_state & (BIT(NFS4CLNT_CHECK_LEASE) | BIT(NFS4CLNT_LEASE_EXPIRED) | BIT(NFS4CLNT_SESSION_RESET))) != 0; } static void nfs_mark_test_expired_delegation(struct nfs_server *server, struct nfs_delegation *delegation) { if (delegation->stateid.type == NFS4_INVALID_STATEID_TYPE) return; clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); set_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); set_bit(NFS4CLNT_DELEGATION_EXPIRED, &server->nfs_client->cl_state); } static void nfs_inode_mark_test_expired_delegation(struct nfs_server *server, struct inode *inode) { struct nfs_delegation *delegation; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation) nfs_mark_test_expired_delegation(server, delegation); rcu_read_unlock(); } static void nfs_delegation_mark_test_expired_server(struct nfs_server *server) { struct nfs_delegation *delegation; list_for_each_entry_rcu(delegation, &server->delegations, super_list) nfs_mark_test_expired_delegation(server, delegation); } /** * nfs_mark_test_expired_all_delegations - mark all delegations for testing * @clp: nfs_client to process * * Iterates through all the delegations associated with this server and * marks them as needing to be checked for validity. */ void nfs_mark_test_expired_all_delegations(struct nfs_client *clp) { struct nfs_server *server; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) nfs_delegation_mark_test_expired_server(server); rcu_read_unlock(); } /** * nfs_test_expired_all_delegations - test all delegations for a client * @clp: nfs_client to process * * Helper for handling "recallable state revoked" status from server. */ void nfs_test_expired_all_delegations(struct nfs_client *clp) { nfs_mark_test_expired_all_delegations(clp); nfs4_schedule_state_manager(clp); } static void nfs_delegation_test_free_expired(struct inode *inode, nfs4_stateid *stateid, const struct cred *cred) { struct nfs_server *server = NFS_SERVER(inode); const struct nfs4_minor_version_ops *ops = server->nfs_client->cl_mvops; int status; if (!cred) return; status = ops->test_and_free_expired(server, stateid, cred); if (status == -NFS4ERR_EXPIRED || status == -NFS4ERR_BAD_STATEID) nfs_remove_bad_delegation(inode, stateid); } static int nfs_server_reap_expired_delegations(struct nfs_server *server, void __always_unused *data) { struct nfs_delegation *delegation; struct inode *inode; const struct cred *cred; nfs4_stateid stateid; unsigned long gen = ++server->delegation_gen; restart: rcu_read_lock(); list_for_each_entry_rcu(delegation, &server->delegations, super_list) { if (test_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags) || test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) || test_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags) == 0 || delegation->test_gen == gen) continue; inode = nfs_delegation_grab_inode(delegation); if (inode == NULL) continue; spin_lock(&delegation->lock); cred = get_cred_rcu(delegation->cred); nfs4_stateid_copy(&stateid, &delegation->stateid); spin_unlock(&delegation->lock); delegation->test_gen = gen; clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); rcu_read_unlock(); nfs_delegation_test_free_expired(inode, &stateid, cred); put_cred(cred); if (!nfs4_server_rebooted(server->nfs_client)) { iput(inode); cond_resched(); goto restart; } nfs_inode_mark_test_expired_delegation(server,inode); iput(inode); return -EAGAIN; } rcu_read_unlock(); return 0; } /** * nfs_reap_expired_delegations - reap expired delegations * @clp: nfs_client to process * * Iterates through all the delegations associated with this server and * checks if they have may have been revoked. This function is usually * expected to be called in cases where the server may have lost its * lease. */ void nfs_reap_expired_delegations(struct nfs_client *clp) { nfs_client_for_each_server(clp, nfs_server_reap_expired_delegations, NULL); } void nfs_inode_find_delegation_state_and_recover(struct inode *inode, const nfs4_stateid *stateid) { struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; struct nfs_delegation *delegation; bool found = false; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation && nfs4_stateid_match_or_older(&delegation->stateid, stateid) && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { nfs_mark_test_expired_delegation(NFS_SERVER(inode), delegation); found = true; } rcu_read_unlock(); if (found) nfs4_schedule_state_manager(clp); } /** * nfs_delegations_present - check for existence of delegations * @clp: client state handle * * Returns one if there are any nfs_delegation structures attached * to this nfs_client. */ int nfs_delegations_present(struct nfs_client *clp) { struct nfs_server *server; int ret = 0; rcu_read_lock(); list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) if (!list_empty(&server->delegations)) { ret = 1; break; } rcu_read_unlock(); return ret; } /** * nfs4_refresh_delegation_stateid - Update delegation stateid seqid * @dst: stateid to refresh * @inode: inode to check * * Returns "true" and updates "dst->seqid" * if inode had a delegation * that matches our delegation stateid. Otherwise "false" is returned. */ bool nfs4_refresh_delegation_stateid(nfs4_stateid *dst, struct inode *inode) { struct nfs_delegation *delegation; bool ret = false; if (!inode) goto out; rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation != NULL && nfs4_stateid_match_other(dst, &delegation->stateid) && nfs4_stateid_is_newer(&delegation->stateid, dst) && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { dst->seqid = delegation->stateid.seqid; ret = true; } rcu_read_unlock(); out: return ret; } /** * nfs4_copy_delegation_stateid - Copy inode's state ID information * @inode: inode to check * @flags: delegation type requirement * @dst: stateid data structure to fill in * @cred: optional argument to retrieve credential * * Returns "true" and fills in "dst->data" * if inode had a delegation, * otherwise "false" is returned. */ bool nfs4_copy_delegation_stateid(struct inode *inode, fmode_t flags, nfs4_stateid *dst, const struct cred **cred) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation; bool ret = false; flags &= FMODE_READ|FMODE_WRITE; rcu_read_lock(); delegation = rcu_dereference(nfsi->delegation); if (!delegation) goto out; spin_lock(&delegation->lock); ret = nfs4_is_valid_delegation(delegation, flags); if (ret) { nfs4_stateid_copy(dst, &delegation->stateid); nfs_mark_delegation_referenced(delegation); if (cred) *cred = get_cred(delegation->cred); } spin_unlock(&delegation->lock); out: rcu_read_unlock(); return ret; } /** * nfs4_delegation_flush_on_close - Check if we must flush file on close * @inode: inode to check * * This function checks the number of outstanding writes to the file * against the delegation 'space_limit' field to see if * the spec requires us to flush the file on close. */ bool nfs4_delegation_flush_on_close(const struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); struct nfs_delegation *delegation; bool ret = true; rcu_read_lock(); delegation = rcu_dereference(nfsi->delegation); if (delegation == NULL || !(delegation->type & FMODE_WRITE)) goto out; if (atomic_long_read(&nfsi->nrequests) < delegation->pagemod_limit) ret = false; out: rcu_read_unlock(); return ret; } module_param_named(delegation_watermark, nfs_delegation_watermark, uint, 0644);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1