Update the Ext3 converter

The main changes in this patch are adding chunk handing and data relocation
ability. In the last step of conversion, the converter relocates data in system
chunk and move chunk tree into system chunk. In the rollback process, the
converter remove chunk tree from system chunk and copy data back.

Regards
YZ
---

ctree.h

@@ -513,8 +513,7 @@ struct btrfs_fs_info {
 
 	struct btrfs_fs_devices *fs_devices;
 	struct list_head space_info;
-	int fp;
-	int force_system_allocs;
+	int system_allocs;
 	void *priv_data;
 };
 
@@ -1331,6 +1330,8 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans,
 			   struct btrfs_root *root, u64 bytes_used,
 			   u64 type, u64 chunk_objectid, u64 chunk_offset,
 			   u64 size);
+int btrfs_make_block_groups(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root);
 u64 btrfs_hash_extent_ref(u64 root_objectid, u64 ref_generation,
 			  u64 owner, u64 owner_offset);
 int btrfs_update_block_group(struct btrfs_trans_handle *trans,

disk-io.c

@@ -457,12 +457,16 @@ insert:
 struct btrfs_root *open_ctree(const char *filename, u64 sb_bytenr)
 {
 	int fp;
+	struct btrfs_root *root;
 
 	fp = open(filename, O_CREAT | O_RDWR, 0600);
 	if (fp < 0) {
 		return NULL;
 	}
-	return open_ctree_fd(fp, filename, sb_bytenr);
+	root = open_ctree_fd(fp, filename, sb_bytenr);
+	close(fp);
+
+	return root;
 }
 
 struct btrfs_root *open_ctree_fd(int fp, const char *path, u64 sb_bytenr)
@@ -500,7 +504,6 @@ struct btrfs_root *open_ctree_fd(int fp, const char *path, u64 sb_bytenr)
 	}
 
 	memset(fs_info, 0, sizeof(*fs_info));
-	fs_info->fp = fs_devices->lowest_bdev;
 	fs_info->fs_root = root;
 	fs_info->tree_root = tree_root;
 	fs_info->extent_root = extent_root;
@@ -638,7 +641,7 @@ int write_all_supers(struct btrfs_root *root)
 				    (unsigned long)btrfs_device_uuid(dev_item),
 				    BTRFS_UUID_SIZE);
 		sb->fd = dev->fd;
-		sb->dev_bytenr = BTRFS_SUPER_INFO_OFFSET;
+		sb->dev_bytenr = sb->start;
 		btrfs_set_header_flag(sb, BTRFS_HEADER_FLAG_WRITTEN);
 		csum_tree_block(root, sb, 0);
 		ret = write_extent_to_disk(sb);
@@ -683,7 +686,7 @@ static int close_all_devices(struct btrfs_fs_info *fs_info)
 	list = &fs_info->fs_devices->devices;
 	list_for_each(next, list) {
 		device = list_entry(next, struct btrfs_device, dev_list);
-		// close(device->fd);
+		close(device->fd);
 	}
 	return 0;
 }
@@ -704,7 +707,6 @@ int close_ctree(struct btrfs_root *root)
 	write_ctree_super(trans, root);
 	btrfs_free_transaction(root, trans);
 	btrfs_free_block_groups(root->fs_info);
-	close(root->fs_info->fp);
 	if (root->node)
 		free_extent_buffer(root->node);
 	if (root->fs_info->extent_root->node)

extent-tree.c

@@ -1624,20 +1624,27 @@ int btrfs_alloc_extent(struct btrfs_trans_handle *trans,
 	u64 super_used, root_used;
 	u64 search_start = 0;
 	u64 alloc_profile;
-	struct btrfs_fs_info *info = root->fs_info;
-	struct btrfs_extent_ops *ops = info->extent_ops;
 	u32 sizes[2];
+	struct btrfs_fs_info *info = root->fs_info;
 	struct btrfs_root *extent_root = info->extent_root;
 	struct btrfs_path *path;
 	struct btrfs_extent_item *extent_item;
 	struct btrfs_extent_ref *ref;
 	struct btrfs_key keys[2];
 
+	if (info->extent_ops) {
+		struct btrfs_extent_ops *ops = info->extent_ops;
+		ret = ops->alloc_extent(root, num_bytes, hint_byte, ins);
+		BUG_ON(ret);
+		goto found;
+	}
+
 	if (data) {
 		alloc_profile = info->avail_data_alloc_bits &
 			        info->data_alloc_profile;
 		data = BTRFS_BLOCK_GROUP_DATA | alloc_profile;
-	} else if (root == info->chunk_root || info->force_system_allocs) {
+	} else if ((info->system_allocs > 0 || root == info->chunk_root) &&
+		   info->system_allocs >= 0) {
 		alloc_profile = info->avail_system_alloc_bits &
 			        info->system_alloc_profile;
 		data = BTRFS_BLOCK_GROUP_SYSTEM | alloc_profile;
@@ -1660,15 +1667,12 @@ int btrfs_alloc_extent(struct btrfs_trans_handle *trans,
 	}
 
 	WARN_ON(num_bytes < root->sectorsize);
-	if (ops && ops->alloc_extent) {
-		ret = ops->alloc_extent(root, num_bytes, hint_byte, ins);
-	} else {
-		ret = find_free_extent(trans, root, num_bytes, empty_size,
-				        search_start, search_end, hint_byte,
-					ins, trans->alloc_exclude_start,
-					trans->alloc_exclude_nr, data);
-	}
+	ret = find_free_extent(trans, root, num_bytes, empty_size,
+			       search_start, search_end, hint_byte, ins,
+			       trans->alloc_exclude_start,
+			       trans->alloc_exclude_nr, data);
 	BUG_ON(ret);
+found:
 	if (ret)
 		return ret;
 
@@ -2331,6 +2335,100 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
+/*
+ * This is for converter use only.
+ *
+ * In that case, we don't know where are free blocks located.
+ * Therefore all block group cache entries must be setup properly
+ * before doing any block allocation.
+ */
+int btrfs_make_block_groups(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root)
+{
+	u64 total_bytes;
+	u64 cur_start;
+	u64 group_type;
+	u64 group_size;
+	u64 group_nr = 0;
+	u64 chunk_objectid;
+	int ret;
+	int bit;
+	struct btrfs_root *extent_root;
+	struct btrfs_block_group_cache *cache;
+	struct extent_io_tree *block_group_cache;
+
+	extent_root = root->fs_info->extent_root;
+	block_group_cache = &root->fs_info->block_group_cache;
+	chunk_objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
+	total_bytes = btrfs_super_total_bytes(&root->fs_info->super_copy);
+
+	cur_start = 0;
+	while (cur_start < total_bytes) {
+		if (group_nr == 0) {
+			bit = BLOCK_GROUP_SYSTEM;
+			group_type = BTRFS_BLOCK_GROUP_SYSTEM;
+		} else if (group_nr % 3 == 1) {
+			bit = BLOCK_GROUP_DATA;
+			group_type = BTRFS_BLOCK_GROUP_METADATA;
+		} else {
+			bit = BLOCK_GROUP_METADATA;
+			group_type = BTRFS_BLOCK_GROUP_DATA;
+		}
+		group_nr++;
+
+		if (group_type == BTRFS_BLOCK_GROUP_SYSTEM) {
+			group_size = 32 * 1024 * 1024;
+		} else {
+			group_size = 256 * 1024 * 1024;
+			if (total_bytes - cur_start < group_size * 5 / 4)
+				group_size = total_bytes - cur_start;
+		}
+
+		cache = kzalloc(sizeof(*cache), GFP_NOFS);
+		BUG_ON(!cache);
+
+		cache->key.objectid = cur_start;
+		cache->key.offset = group_size;
+		btrfs_set_key_type(&cache->key, BTRFS_BLOCK_GROUP_ITEM_KEY);
+
+		btrfs_set_block_group_used(&cache->item, 0);
+		btrfs_set_block_group_chunk_objectid(&cache->item,
+						     chunk_objectid);
+		btrfs_set_block_group_flags(&cache->item, group_type);
+
+		cache->flags = group_type;
+
+		ret = update_space_info(root->fs_info, group_type, group_size,
+					0, &cache->space_info);
+		BUG_ON(ret);
+		set_avail_alloc_bits(extent_root->fs_info, group_type);
+
+		set_extent_bits(block_group_cache, cur_start,
+				cur_start + group_size - 1,
+				bit | EXTENT_LOCKED, GFP_NOFS);
+		set_state_private(block_group_cache, cur_start,
+				  (unsigned long)cache);
+		cur_start += group_size;
+	}
+	/* then insert all the items */
+	cur_start = 0;
+	while(cur_start < total_bytes) {
+		cache = btrfs_lookup_block_group(root->fs_info, cur_start);
+		BUG_ON(!cache);
+
+		ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
+					sizeof(cache->item));
+		BUG_ON(ret);
+
+		finish_current_insert(trans, extent_root);
+		ret = del_pending_extents(trans, extent_root);
+		BUG_ON(ret);
+
+		cur_start = cache->key.objectid + cache->key.offset;
+	}
+	return 0;
+}
+
 u64 btrfs_hash_extent_ref(u64 root_objectid, u64 ref_generation,
 			  u64 owner, u64 owner_offset)
 {

extent_io.c

@@ -539,6 +539,8 @@ static int free_some_buffers(struct extent_io_tree *tree)
 			free_extent_buffer(eb);
 			if (tree->cache_size < cache_max)
 				break;
+		} else {
+			list_move_tail(&eb->lru, &tree->lru);
 		}
 		if (nrscan++ > 64)
 			break;

mkfs.c

@@ -79,7 +79,7 @@ static int make_root_dir(int fd, const char *device_name) {
 	trans = btrfs_start_transaction(root, 1);
 	bytes_used = btrfs_super_bytes_used(&root->fs_info->super_copy);
 
-	root->fs_info->force_system_allocs = 1;
+	root->fs_info->system_allocs = 1;
 	ret = btrfs_make_block_group(trans, root, bytes_used,
 				     BTRFS_BLOCK_GROUP_SYSTEM,
 				     BTRFS_FIRST_CHUNK_TREE_OBJECTID,
@@ -96,7 +96,7 @@ static int make_root_dir(int fd, const char *device_name) {
 				     chunk_start, chunk_size);
 	BUG_ON(ret);
 
-	root->fs_info->force_system_allocs = 0;
+	root->fs_info->system_allocs = 0;
 	btrfs_commit_transaction(trans, root);
 	trans = btrfs_start_transaction(root, 1);
 	BUG_ON(!trans);

utils.c

@@ -54,7 +54,7 @@ static u64 reference_root_table[6] = {
 	[5] =	BTRFS_FS_TREE_OBJECTID,
 };
 
-int make_btrfs(int fd, char *device_name, char *label,
+int make_btrfs(int fd, const char *device, const char *label,
 	       u64 blocks[6], u64 num_bytes, u32 nodesize,
 	       u32 leafsize, u32 sectorsize, u32 stripesize)
 {

utils.h

@@ -21,7 +21,7 @@
 
 #define BTRFS_MKFS_SYSTEM_GROUP_SIZE (4 * 1024 * 1024)
 
-int make_btrfs(int fd, char *device_name, char *label,
+int make_btrfs(int fd, const char *device, const char *label,
 	       u64 blocks[6], u64 num_bytes, u32 nodesize,
 	       u32 leafsize, u32 sectorsize, u32 stripesize);
 int btrfs_make_root_dir(struct btrfs_trans_handle *trans,

volumes.c

@@ -148,7 +148,8 @@ int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
 
 	list_for_each(cur, head) {
 		device = list_entry(cur, struct btrfs_device, dev_list);
-		device->fd = 0;
+		close(device->fd);
+		device->fd = -1;
 	}
 	return 0;
 }

volumes.h

@@ -108,6 +108,8 @@ int btrfs_close_devices(struct btrfs_fs_devices *fs_devices);
 int btrfs_add_device(struct btrfs_trans_handle *trans,
 		     struct btrfs_root *root,
 		     struct btrfs_device *device);
+int btrfs_update_device(struct btrfs_trans_handle *trans,
+			struct btrfs_device *device);
 int btrfs_scan_one_device(int fd, const char *path,
 			  struct btrfs_fs_devices **fs_devices_ret,
 			  u64 *total_devs, u64 super_offset);
@@ -115,4 +117,7 @@ int btrfs_num_copies(struct btrfs_mapping_tree *map_tree, u64 logical, u64 len);
 int btrfs_bootstrap_super_map(struct btrfs_mapping_tree *map_tree,
 			      struct btrfs_fs_devices *fs_devices);
 struct list_head *btrfs_scanned_uuids(void);
+int btrfs_add_system_chunk(struct btrfs_trans_handle *trans,
+			   struct btrfs_root *root, struct btrfs_key *key,
+			   struct btrfs_chunk *chunk, int item_size);
 #endif