anais_berck
/
Random-forest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'main' of gitlab.constantvzw.org:anais_berck/random_forest into main

Browse Source
main
Gijs 2 years ago
parent ccc77cf95f e32fbfbeb7
commit f045b8158d
1 changed files with 426 additions and 0 deletions
Show Stats Download Patch File Download Diff File

426
commenting_code_model/random forest.ipynb
Unescape Escape View File

@ -0,0 +1,426 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Random Forest"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from random import seed\n",
"from random import randrange\n",
"from csv import reader\n",
"from math import sqrt\n",
"import json\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
" # Load a CSV file. Definition of the function to read the csv and create dataset here\n",
"def load_csv(filename):\n",
"\tdataset = list()\n",
"\twith open(filename, 'r') as file:\n",
"\t\tcsv_reader = reader(file)\n",
"\t\tfor row in csv_reader:\n",
"\t\t\tif not row:\n",
"\t\t\t\tcontinue\n",
"\t\t\tdataset.append(row)\n",
"\treturn dataset\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# Convert string column to float\n",
"def str_column_to_float(dataset, column):\n",
"\tfor row in dataset:\n",
"\t\trow[column] = float(row[column].strip())\n",
" \n",
"# Convert string column to integer\n",
"def str_column_to_int(dataset, column):\n",
"\tclass_values = [row[column] for row in dataset] # extract the values of the column (here the classes of the dataset, mine and rocks)\n",
"\tunique = set(class_values) # calculate how many unique class values there are and store them into a set: a list with unique values\n",
"\tlookup = dict() #create a dictionnary\n",
"\tfor i, value in enumerate(unique): # loops through the set / enumerate gives you a tuple with an index number and a value /common way to get indexes from a list\n",
"\t\tlookup[value] = i # the key of the dictonnary is the value: mine or rock; and the value is a number: 0 or 1\n",
"\tfor row in dataset: # loops through the rows of the dataset\n",
"\t\trow[column] = lookup[row[column]] #replaces the value of the column: rock or mine, with the index value: 0 or 1;\n",
"\treturn lookup # the code returns the lookup table\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# Split a dataset into k folds\n",
"def cross_validation_split(dataset, n_folds):\n",
"\tdataset_split = list() #create a list \n",
"\tdataset_copy = list(dataset) # creates a list of the dataset. You could use the copy library\n",
"\tfold_size = int(len(dataset) / n_folds) # the size of the fold is equal to the length of the dataset divided by the amount of folds\n",
"\tprint(\"fold size:\")\n",
"\tprint(fold_size) \n",
"\tfor i in range(n_folds): #loops the amount of folds : generate another list with numbers from 0 up to n_fold\n",
"\t\tfold = list() #create a list\n",
"\t\twhile len(fold) < fold_size: # as long as the length of the list is inferior to the defined fold size\n",
"\t\t\tindex = randrange(len(dataset_copy)) # return a random integer between 0 and the total length of the dataset and store it in index\n",
"\t\t\tfold.append(dataset_copy.pop(index)) # append an observation at the index and removes it from the dataset\n",
"\t\tdataset_split.append(fold) # append the fold to the list dataset_split\n",
"\t\t#print(\"______________\")\n",
"\t\t#print(\"dataset split:\")\n",
"\t\t#print(dataset_split)\n",
"\t\t#print(\"______________\")\n",
"\treturn dataset_split #return the dataset_split, a list of folds\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"# Calculate accuracy percentage\n",
"def accuracy_metric(actual, predicted):\n",
"\tcorrect = 0 #create a correct variable\n",
"\tfor i in range(len(actual)): # loops up to the length of the actual list\n",
"\t\tif actual[i] == predicted[i]: # compares the actual vs the predicted\n",
"\t\t\tcorrect += 1 #if correct add one to the correct variable. Count the number of correct guesses\n",
"\treturn correct / float(len(actual)) * 100.0 #gives a percentage by dividing the correct guesses by the length of the actual classes and multiply by a hundred\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"# Evaluate an algorithm using a cross validation split\n",
"def evaluate_algorithm(dataset, algorithm, n_folds, *args):\n",
"\tfolds = cross_validation_split(dataset, n_folds) #return the list of folds\n",
"\tscores = list() #creates a list called scores\n",
"\tfor fold in folds: #loops through the folds\n",
"\t\ttrain_set = list(folds) #creates a copy of the list of folds \n",
"\t\ttrain_set.remove(fold) #remove one fold: for testing?\n",
"\t\ttrain_set = sum(train_set, []) # concatenate all folds, a list of lists, into one list. can be done with itertools.chain\n",
"\t\ttest_set = list() # create another list\n",
"\t\tfor row in fold: # iterates through fold\n",
"\t\t\trow_copy = list(row) # creates a copy of the list\n",
"\t\t\ttest_set.append(row_copy) # append the list to the test_set\n",
"\t\t\trow_copy[-1] = None # set the classification to none. Changes the last column to none.\n",
"\t\tpredicted = algorithm(train_set, test_set, *args) # what is doing the prediction takes for argument the train and test set and returns prediction for the test set\n",
"\t\tactual = [row[-1] for row in fold] # list comprehension: list of actual classes from fold.\n",
"\t\taccuracy = accuracy_metric(actual, predicted) # function that compares the actual vs the predicted to give an idea of the accuracy of the prediction\n",
"\t\tscores.append(accuracy) #append the accuracy to the list of scores\n",
"\treturn scores\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"# Split a dataset based on an attribute/feature and an attribute/feature value\n",
"def test_split(index, value, dataset):\n",
"\tleft, right = list(), list() # create two lists for each side\n",
"\tfor row in dataset: #iterate through each row of the dataset\n",
"\t\tif row[index] < value: #if the feature value of the current row is below the feature value given\n",
"\t\t\tleft.append(row) # append it to the left list\n",
"\t\telse:\n",
"\t\t\tright.append(row) # append it to the right list\n",
"\treturn left, right # return the two lists\n",
" \n",
"# Calculate the Gini index for a split dataset\n",
"def gini_index(groups, classes):\n",
"\t# count all samples at split point\n",
"\tn_instances = float(sum([len(group) for group in groups])) #counting the total number of instances into float for divisions\n",
"\t# sum weighted Gini index for each group\n",
"\tgini = 0.0 #gini variable\n",
"\tfor group in groups: #for each of the group\n",
"\t\tsize = float(len(group)) #number of instances in each group\n",
"\t\t# avoid divide by zero\n",
"\t\tif size == 0:\n",
"\t\t\tcontinue\n",
"\t\tscore = 0.0\n",
"\t\t# score the group based on the score for each class\n",
"\t\tfor class_val in classes:\n",
"\t\t\tp = [row[-1] for row in group].count(class_val) / size #count the number of instances for the current class in the group and divide it by the total size of the group\n",
"\t\t\tscore += p * p #amplifying the difference (exponential ?)\n",
"\t\t# weight the group score by its relative size\n",
"\t\tgini += (1.0 - score) * (size / n_instances) # substract the score from 1 and multiply it by the relative size of the group compared to the dataset\n",
"\treturn gini\n",
" \n",
"# Select the best split point for a dataset\n",
"def get_split(dataset, n_features):\n",
"\tclass_values = list(set(row[-1] for row in dataset)) # creates a list of the set for the class values. Here encoded as 1 and 0 . We already did it before\n",
"\tb_index, b_value, b_score, b_groups = 999, 999, 999, None # assign values to variables\n",
"\tfeatures = list() #creates a list. \n",
"\twhile len(features) < n_features: # as long as features is smaller that the actual number of desirable features= n_features\n",
"\t\tindex = randrange(len(dataset[0])-1) # create a random number between 0 and the number of columns-1= minus the class\n",
"\t\tif index not in features: # if the column names is not already in the features\n",
"\t\t\tfeatures.append(index) # append the column name\n",
"\tfor index in features: # for each column name(=index name) in features\n",
"\t\tfor row in dataset: # for each row of the dataset\n",
"\t\t\tgroups = test_split(index, row[index], dataset) # get two lists. very computationnaly heavy. Why not do the ordering first?\n",
"\t\t\tgini = gini_index(groups, class_values) # get the gini value\n",
"\t\t\tif gini < b_score: #if the gini value is smaller than b_score (b for best?). this should always be true for the first operation. Test against the best option\n",
"\t\t\t\tb_index, b_value, b_score, b_groups = index, row[index], gini, groups # update the values for the best option\n",
"\treturn {'index':b_index, 'value':b_value, 'groups':b_groups} #return the best option in the form of a dictionnary\n",
" \n",
"# Create a terminal node value\n",
"# Returns the most popular value in the group\n",
"def to_terminal(group):\n",
"\toutcomes = [row[-1] for row in group] #takes the class of the group elements and put it in a list\n",
"\treturn max(set(outcomes), key=outcomes.count) #return the class that appears the most. set is counting the different class and the key is counting the number of occurences of this class. SPOOKY AND DENSE\n",
" \n",
"# Create child splits for a node or make a terminal (node)\n",
"def split(node, max_depth, min_size, n_features, depth):\n",
"\tleft, right = node['groups'] #get the groups using the dictionnary node with the key groups\n",
"\tdel(node['groups']) #delete dictionnary element with key groups\n",
"\t# check for a no split\n",
"\tif not left or not right: #if one of the group is empty\n",
"\t\tnode['left'] = node['right'] = to_terminal(left + right) #call this function\n",
"\t\treturn \n",
"\t# check for max depth\n",
"\tif depth >= max_depth: # checking if level is bigger or equal than maximum level of nodes\n",
"\t\tnode['left'], node['right'] = to_terminal(left), to_terminal(right) #if yes, so both nodes are terminal nodes\n",
"\t\treturn\n",
"\t# process left child\n",
"\tif len(left) <= min_size: # if the group is smaller or equal than the minimum size for a group\n",
"\t\tnode['left'] = to_terminal(left) #left node is a terminal node\n",
"\telse:\n",
"\t\tnode['left'] = get_split(left, n_features) #create another split, another node from which to separate in two groups with a subset of a dataset\n",
"\t\tsplit(node['left'], max_depth, min_size, n_features, depth+1) #recursion -> calls itself\n",
"\t# process right child\n",
"\tif len(right) <= min_size:\n",
"\t\tnode['right'] = to_terminal(right)\n",
"\telse:\n",
"\t\tnode['right'] = get_split(right, n_features)\n",
"\t\tsplit(node['right'], max_depth, min_size, n_features, depth+1)\n",
" \n",
"# Build a decision tree\n",
"def build_tree(train, max_depth, min_size, n_features):\n",
"\troot = get_split(train, n_features) #get the index, value and group (?) for the split. It's a dictionnary\n",
"\tsplit(root, max_depth, min_size, n_features, 1) #depth is one\n",
" # root is Node\n",
" # Node: { index: int, value: float, left: Node|TerminalNode, right: Node|TerminalNode }\n",
" # TerminalNode: { index: int, value: float, left: int(class), right: int(class) }\n",
"\treturn root #dictionnary with index, value, left, right\n",
" \n",
"# Make a prediction with a decision tree\n",
"def predict(node, row):\n",
"\tif row[node['index']] < node['value']: #if the feature value of the row is smaller of the feature value of the node\n",
"\t\tif isinstance(node['left'], dict): # is it a node or a terminal node(children are not node)\n",
"\t\t\treturn predict(node['left'], row) #recursion if a node. the function calls itselft on the following left node\n",
"\t\telse:\n",
"\t\t\treturn node['left'] # result if a terminal node\n",
"\telse:\n",
"\t\tif isinstance(node['right'], dict): # is it a node or a terminal node(children are not node)\n",
"\t\t\treturn predict(node['right'], row)#recursion if a node. the function calls itselft on the following right node\n",
"\t\telse:\n",
"\t\t\treturn node['right']# result if a terminal node\n",
" \n",
"# Create a random subsample from the dataset with replacement\n",
"def subsample(dataset, ratio):\n",
"\tsample = list() #creates a list\n",
"\tn_sample = round(len(dataset) * ratio) # rounds the multiplication of the length of the dataset with the sample.size: here 1. \n",
"\twhile len(sample) < n_sample: # loop up until the length of the sample is the length of n_sample\n",
"\t\tindex = randrange(len(dataset)) #take a random number from 0 up to length of the dataset\n",
"\t\tsample.append(dataset[index]) # append a sample whith this index\n",
"\treturn sample # return the list of sub-samples\n",
" \n",
"# Make a prediction with a list of bagged trees\n",
"def bagging_predict(trees, row): \n",
"\tpredictions = [predict(tree, row) for tree in trees] # we run the prediction in each tree, this gives a list of predictions of classes/votes. THE TREES ARE VOTING.\n",
"\treturn max(set(predictions), key=predictions.count) # we count the class with the maximum votes and return it as prediction\n",
" \n",
"# Random Forest Algorithm\n",
"def random_forest(train, test, max_depth, min_size, sample_size, n_trees, n_features):\n",
"\ttrees = list() #create a list of trees\n",
"\tfor i in range(n_trees): \n",
"\t\tsample = subsample(train, sample_size) # create subsamples \n",
"\t\ttree = build_tree(sample, max_depth, min_size, n_features) #build the tree\n",
"\t\ttrees.append(tree)#append the list \n",
"\twith open(\"model.json\", \"w\") as out_file: \n",
"\t\tjson.dump(trees, out_file, indent = 6)\n",
"\tpredictions = [bagging_predict(trees, row) for row in test] # testing with test data, running the prediction on every row. THE FOREST VOTES ON EVERY ROW\n",
"\treturn(predictions) # we return the predicted class of each of the rows as a list. THE PREDICTIONS OF THE FOREST\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"from random import random"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# Test the random forest algorithm\n",
"seed(2) #put the random generator in a certain state -> makes it deterministic otherwise python takes the time of the computer\n",
"#print(random())\n",
"#print(random())\n",
"#seed(2)\n",
"#print(random())\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"# load and prepare data\n",
"filename = 'iris.csv'\n",
"#filename = 'iris.csv'\n",
"\n",
"dataset = load_csv(filename)\n",
"#print(dataset)\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# convert string attributes to integers\n",
"for i in range(0, len(dataset[0])-1):\n",
"\tstr_column_to_float(dataset, i)\n",
"# convert class column to integers\n",
"#str_column_to_int(dataset, len(dataset[0])-1) #this function loops through the rows and transforms the words mine and roch into 1 and 0\n",
"#print(dataset)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# evaluate algorithm\n",
"n_folds = 5 #the data is randomly sampled into 5 subsamples, one is kept for testing the 4 else are used for training.\n",
"max_depth = 10 #\n",
"min_size = 1\n",
"sample_size = 1.0 #fixed, can be changed to reduces the amount of subsampling, if it is smaller than one.\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"fold size:\n",
"30\n",
"Trees: 1\n",
"Scores: [100.0, 93.33333333333333, 90.0, 100.0, 93.33333333333333]\n",
"Mean Accuracy: 95.333%\n",
"fold size:\n",
"30\n",
"Trees: 5\n",
"Scores: [96.66666666666667, 96.66666666666667, 100.0, 93.33333333333333, 93.33333333333333]\n",
"Mean Accuracy: 96.000%\n",
"fold size:\n",
"30\n",
"Trees: 10\n",
"Scores: [90.0, 96.66666666666667, 96.66666666666667, 86.66666666666667, 100.0]\n",
"Mean Accuracy: 94.000%\n"
]
}
],
"source": [
"n_features = int(sqrt(len(dataset[0])-1)) #it specifies the size of the feature subset for the folds, where the size is close to the square root of the total number of features\n",
"for n_trees in [1, 5, 10]: # will loop three times by taking n_trees equals one, n_trees equals five and n_trees equals ten successively.\n",
"\tscores = evaluate_algorithm(dataset, random_forest, n_folds, max_depth, min_size, sample_size, n_trees, n_features)\n",
"\tprint('Trees: %d' % n_trees)\n",
"\tprint('Scores: %s' % scores)\n",
"\tprint('Mean Accuracy: %.3f%%' % (sum(scores)/float(len(scores))))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Iris Versicolor\n"
]
}
],
"source": [
"with open (\"model.json\", \"r\") as in_file:\n",
"\ttrees=json.load(in_file)\n",
"\tprediction=bagging_predict(trees,dataset[55])\n",
"\tprint(prediction)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Write Preview
Loading…
Cancel
Save