---

yaml
---
r: 174359
b: refs/heads/master
c: 04dc1e8
h: refs/heads/master
i:
  174357: 7650bb3
  174355: bcd9261
  174351: 962fa87
v: v3

[refs]

@@ -1,2 +1,2 @@
 ---
-refs/heads/master: 06886a5a3dc5a5abe0a4d257c26317bde7047be8
+refs/heads/master: 04dc1e88ad9c9f9639019e9646a89ce0ebf706bb

trunk/fs/exofs/common.h

@@ -49,6 +49,7 @@
 #define EXOFS_MIN_PID   0x10000	/* Smallest partition ID */
 #define EXOFS_OBJ_OFF	0x10000	/* offset for objects */
 #define EXOFS_SUPER_ID	0x10000	/* object ID for on-disk superblock */
+#define EXOFS_DEVTABLE_ID 0x10001 /* object ID for on-disk device table */
 #define EXOFS_ROOT_ID	0x10002	/* object ID for root directory */
 
 /* exofs Application specific page/attribute */
@@ -78,17 +79,67 @@ enum {
 #define EXOFS_SUPER_MAGIC	0x5DF5
 
 /*
- * The file system control block - stored in an object's data (mainly, the one
- * with ID EXOFS_SUPER_ID).  This is where the in-memory superblock is stored
- * on disk.  Right now it just has a magic value, which is basically a sanity
- * check on our ability to communicate with the object store.
+ * The file system control block - stored in object EXOFS_SUPER_ID's data.
+ * This is where the in-memory superblock is stored on disk.
  */
+enum {EXOFS_FSCB_VER = 1, EXOFS_DT_VER = 1};
 struct exofs_fscb {
 	__le64  s_nextid;	/* Highest object ID used */
-	__le32  s_numfiles;	/* Number of files on fs */
+	__le64  s_numfiles;	/* Number of files on fs */
+	__le32	s_version;	/* == EXOFS_FSCB_VER */
 	__le16  s_magic;	/* Magic signature */
 	__le16  s_newfs;	/* Non-zero if this is a new fs */
-};
+
+	/* From here on it's a static part, only written by mkexofs */
+	__le64	s_dev_table_oid;   /* Resurved, not used */
+	__le64	s_dev_table_count; /* == 0 means no dev_table */
+} __packed;
+
+/*
+ * Describes the raid used in the FS. It is part of the device table.
+ * This here is taken from the pNFS-objects definition. In exofs we
+ * use one raid policy through-out the filesystem. (NOTE: the funny
+ * alignment at begining. We take care of it at exofs_device_table.
+ */
+struct exofs_dt_data_map {
+	__le32	cb_num_comps;
+	__le64	cb_stripe_unit;
+	__le32	cb_group_width;
+	__le32	cb_group_depth;
+	__le32	cb_mirror_cnt;
+	__le32	cb_raid_algorithm;
+} __packed;
+
+/*
+ * This is an osd device information descriptor. It is a single entry in
+ * the exofs device table. It describes an osd target lun which
+ * contains data belonging to this FS. (Same partition_id on all devices)
+ */
+struct exofs_dt_device_info {
+	__le32	systemid_len;
+	u8	systemid[OSD_SYSTEMID_LEN];
+	__le64	long_name_offset;	/* If !0 then offset-in-file */
+	__le32	osdname_len;		/* */
+	u8	osdname[44];		/* Embbeded, Ususally an asci uuid */
+} __packed;
+
+/*
+ * The EXOFS device table - stored in object EXOFS_DEVTABLE_ID's data.
+ * It contains the raid used for this multy-device FS and an array of
+ * participating devices.
+ */
+struct exofs_device_table {
+	__le32				dt_version;	/* == EXOFS_DT_VER */
+	struct exofs_dt_data_map	dt_data_map;	/* Raid policy to use */
+
+	/* Resurved space For future use. Total includeing this:
+	 * (8 * sizeof(le64))
+	 */
+	__le64				__Resurved[4];
+
+	__le64				dt_num_devices;	/* Array size */
+	struct exofs_dt_device_info	dt_dev_table[];	/* Array of devices */
+} __packed;
 
 /****************************************************************************
  * inode-related things

trunk/fs/exofs/exofs.h

@@ -37,6 +37,11 @@
 #include <linux/time.h>
 #include "common.h"
 
+/* FIXME: Remove once pnfs hits mainline
+ * #include <linux/exportfs/pnfs_osd_xdr.h>
+ */
+#include "pnfs.h"
+
 #define EXOFS_ERR(fmt, a...) printk(KERN_ERR "exofs: " fmt, ##a)
 
 #ifdef CONFIG_EXOFS_DEBUG
@@ -54,7 +59,6 @@
  * our extension to the in-memory superblock
  */
 struct exofs_sb_info {
-	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
 	struct exofs_fscb s_fscb;		/* Written often, pre-allocate*/
 	osd_id		s_pid;			/* partition ID of file system*/
 	int		s_timeout;		/* timeout for OSD operations */
@@ -63,7 +67,11 @@ struct exofs_sb_info {
 	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
 	u32		s_next_generation;	/* next gen # to use          */
 	atomic_t	s_curr_pending;		/* number of pending commands */
-	uint8_t		s_cred[OSD_CAP_LEN];	/* all-powerful credential    */
+	uint8_t		s_cred[OSD_CAP_LEN];	/* credential for the fscb    */
+
+	struct pnfs_osd_data_map data_map;	/* Default raid to use        */
+	unsigned	s_numdevs;		/* Num of devices in array    */
+	struct osd_dev	*s_ods[1];		/* Variable length, minimum 1 */
 };
 
 /*

trunk/fs/exofs/inode.c

@@ -62,7 +62,10 @@ static void _pcol_init(struct page_collect *pcol, unsigned expected_pages,
 	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
 
 	pcol->sbi = sbi;
-	pcol->req_q = osd_request_queue(sbi->s_dev);
+	/* Create master bios on first Q, later on cloning, each clone will be
+	 * allocated on it's destination Q
+	 */
+	pcol->req_q = osd_request_queue(sbi->s_ods[0]);
 	pcol->inode = inode;
 	pcol->expected_pages = expected_pages;
 

trunk/fs/exofs/ios.c

@@ -71,7 +71,7 @@ int exofs_get_io_state(struct exofs_sb_info *sbi, struct exofs_io_state** pios)
 	/*TODO: Maybe use kmem_cach per sbi of size
 	 * exofs_io_state_size(sbi->s_numdevs)
 	 */
-	ios = kzalloc(exofs_io_state_size(1), GFP_KERNEL);
+	ios = kzalloc(exofs_io_state_size(sbi->s_numdevs), GFP_KERNEL);
 	if (unlikely(!ios)) {
 		*pios = NULL;
 		return -ENOMEM;
@@ -209,10 +209,10 @@ int exofs_sbi_create(struct exofs_io_state *ios)
 {
 	int i, ret;
 
-	for (i = 0; i < 1; i++) {
+	for (i = 0; i < ios->sbi->s_numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+		or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -233,10 +233,10 @@ int exofs_sbi_remove(struct exofs_io_state *ios)
 {
 	int i, ret;
 
-	for (i = 0; i < 1; i++) {
+	for (i = 0; i < ios->sbi->s_numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+		or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -257,10 +257,10 @@ int exofs_sbi_write(struct exofs_io_state *ios)
 {
 	int i, ret;
 
-	for (i = 0; i < 1; i++) {
+	for (i = 0; i < ios->sbi->s_numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+		or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -272,7 +272,21 @@ int exofs_sbi_write(struct exofs_io_state *ios)
 		if (ios->bio) {
 			struct bio *bio;
 
-			bio = ios->bio;
+			if (i != 0) {
+				bio = bio_kmalloc(GFP_KERNEL,
+						  ios->bio->bi_max_vecs);
+				if (unlikely(!bio)) {
+					ret = -ENOMEM;
+					goto out;
+				}
+
+				__bio_clone(bio, ios->bio);
+				bio->bi_bdev = NULL;
+				bio->bi_next = NULL;
+				ios->per_dev[i].bio =  bio;
+			} else {
+				bio = ios->bio;
+			}
 
 			osd_req_write(or, &ios->obj, ios->offset, bio,
 				      ios->length);
@@ -306,8 +320,10 @@ int exofs_sbi_read(struct exofs_io_state *ios)
 
 	for (i = 0; i < 1; i++) {
 		struct osd_request *or;
+		unsigned first_dev = (unsigned)ios->obj.id;
 
-		or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+		first_dev %= ios->sbi->s_numdevs;
+		or = osd_start_request(ios->sbi->s_ods[first_dev], GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -382,10 +398,10 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
 	attr = g_attr_logical_length;
 	attr.val_ptr = &newsize;
 
-	for (i = 0; i < 1; i++) {
+	for (i = 0; i < sbi->s_numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(sbi->s_dev, GFP_KERNEL);
+		or = osd_start_request(sbi->s_ods[i], GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;

trunk/fs/exofs/pnfs.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2008, 2009
+ * Boaz Harrosh <bharrosh@panasas.com>
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License  version 2 as published by the Free
+ * Software Foundation.
+ *
+ */
+
+/* FIXME: Remove this file once pnfs hits mainline */
+
+#ifndef __EXOFS_PNFS_H__
+#define __EXOFS_PNFS_H__
+
+#if defined(CONFIG_PNFS)
+
+
+/* FIXME: move this file to: linux/exportfs/pnfs_osd_xdr.h */
+#include "../nfs/objlayout/pnfs_osd_xdr.h"
+
+#else /* defined(CONFIG_PNFS) */
+
+enum pnfs_iomode {
+	IOMODE_READ = 1,
+	IOMODE_RW = 2,
+	IOMODE_ANY = 3,
+};
+
+/* Layout Structure */
+enum pnfs_osd_raid_algorithm4 {
+	PNFS_OSD_RAID_0		= 1,
+	PNFS_OSD_RAID_4		= 2,
+	PNFS_OSD_RAID_5		= 3,
+	PNFS_OSD_RAID_PQ	= 4     /* Reed-Solomon P+Q */
+};
+
+struct pnfs_osd_data_map {
+	u32	odm_num_comps;
+	u64	odm_stripe_unit;
+	u32	odm_group_width;
+	u32	odm_group_depth;
+	u32	odm_mirror_cnt;
+	u32	odm_raid_algorithm;
+};
+
+#endif /* else defined(CONFIG_PNFS) */
+
+#endif /* __EXOFS_PNFS_H__ */