Coursera Week 4

Painting a picture with the numbers

The code

1. import pandas as pd  
2. import numpy as np  
3. import seaborn  
4. import matplotlib.pyplot as plt  
5. plt.rcdefaults()  
6.   
7.   
8. #show max rows and columns in dataframe with the following two lines  
9. pd.set_option('display.max_columns', None)  
10. pd.set_option('display.max_rows', None)  
11. pd.options.mode.chained_assignment = None   
12. #to prevent runtime errors use the following code  
13. pd.set_option('display.float_format', lambda x: '%f'%x)  
14. #filename = 'C:\\Users\\Christopher\\OneDrive for Business\\Data studies\\addhealth_pds.csv'  
15. filename ='addhealth_pds.csv'  
16.   
17. #create a dataset from original csv file  
18. #mydata = pd.read_csv('addhealth_pds.csv', low_memory=False)  
19. mydata = pd.read_csv(filename, low_memory=False)  
20.   
21. #ensure all data column headers are uppercase with the following  
22. mydata.columns = map(str.upper, mydata.columns)  
23.   
24. #convert columns to numeric rather than data  
25. pd.to_numeric(mydata['H1ED5']) #failed any grade  
26. pd.to_numeric(mydata["H1ED6A"])#failed 1ST grade  
27. pd.to_numeric(mydata["H1ED6B"]) #failed 2ND grade  
28. pd.to_numeric(mydata["H1ED6C"]) #failed 3RD grade  
29. pd.to_numeric(mydata["H1ED6D"]) #failed 4th grade  
30. pd.to_numeric(mydata["H1ED6E"])#failed 5th grade  
31. pd.to_numeric(mydata["H1ED6F"])#failed 6th grade  
32. pd.to_numeric(mydata["H1ED6G"]) #failed 7th grade  
33. pd.to_numeric(mydata["H1ED6H"]) #failed 8th grade  
34. pd.to_numeric(mydata["H1ED6I"]) #failed 9th grade  
35. pd.to_numeric(mydata["H1ED6J"]) #failed 10th grade  
36. pd.to_numeric(mydata["H1ED6K"]) #failed 11th grade  
37. pd.to_numeric(mydata["H1FV5"]) #got into fight  
38. pd.to_numeric(mydata["H1FV7"]) #pulled a knife or gun  
39. pd.to_numeric(mydata["H1FV8"]) #shot or stabbed someone  
40.   
41. mydata['H1ED5']=mydata['H1ED5'].replace(6, np.nan)  
42. mydata['H1ED5']=mydata['H1ED5'].replace(8, np.nan)  
43.   
44. def retained (row):  
45.     if row["NUMRETAINED"] > 1 and row['NUMRETAINED'] <=3:  
46.         return 13  
47.     if row["NUMRETAINED"] > 3:  
48.         #return np.nan   
49.         return 0  
50.     if row['H1ED6A'] == 1:  
51.         return 1  
52.     if row['H1ED6B'] == 1:  
53.         return 2  
54.     if row['H1ED6C'] == 1:  
55.         return 3  
56.     if row['H1ED6D'] == 1:  
57.         return 4  
58.     if row['H1ED6E'] == 1:  
59.         return 5  
60.     if row['H1ED6F'] == 1:  
61.         return 6  
62.     if row['H1ED6G'] == 1:  
63.         return 7  
64.     if row['H1ED6H'] == 1:  
65.         return 8  
66.     if row['H1ED6I'] == 1:  
67.         return 9  
68.     if row['H1ED6J'] == 1:  
69.         return 10  
70.     if row['H1ED6K'] == 1:  
71.         return 11  
72.     else:  
73.         return 0  
74.                   
75. sub1 = mydata[(mydata['H1ED5'] >= 0) & (mydata['H1ED5'] <= 1)]  
76. sub2 = sub1.copy  
77.   
78. #This line uses the above function to fill in the grade level the respondent  
79. # was retained  
80. sub1["NUMRETAINED"] = (0 + sub1['H1ED6A'] + sub1['H1ED6B'] + sub1['H1ED6C'] +   
81. sub1['H1ED6D'] + sub1['H1ED6E'] + sub1['H1ED6F'] + sub1['H1ED6G'] +   
82. sub1['H1ED6H'] + sub1['H1ED6I'] + sub1['H1ED6J'] + sub1['H1ED6K'])  
83.   
84. sub1['GRADERETAINED'] = sub1.apply(lambda row: retained (row), axis = 1)  
85.   
86.   
87. sub1['GRADELEVEL']=pd.cut(sub1.GRADERETAINED,[0, 5, 9, 12, 13])  
88.   
89. def violence (row):  
90.     if row ["H1FV5"] >=1 and row ["H1FV5"] <=2:  
91.         return 1  
92.     if row ["H1FV7"] >=1 and row ["H1FV7"] <=2:  
93.         return 1  
94.     if row ["H1FV8"] >=1 and row ["H1FV8"] <=2:  
95.         return 1  
96.     else:  
97.         return 0  
98.   
99. sub1['VIOLENT'] = sub1.apply(lambda row: violence (row), axis = 1)  
100.   
101. delinquent = sub1["VIOLENT"].value_counts(sort=True, dropna=False)  
102. print("Count of respondents who fought and/or used a weapon\n")  
103. print("\t 0= Have not done so and 1 = have fought or used a weapon")  
104. print(delinquent)  
105.   
106. seaborn.countplot(x='VIOLENT', data=sub1)  
107. plt.xlabel('0- No violence, 1- has used violence')  
108. plt.title("In last 12 months have you fought and/or used a weapon against \  
109. another?")  
110.   
111. seaborn.factorplot(x="GRADERETAINED", y="VIOLENT", data=sub1, kind="point", ci=None)  
112. plt.xlabel('Grade Level')  
113. plt.ylabel('Percent of violent repeaters')  
114.   
115. desc1 = sub1["GRADERETAINED"].value_counts(sort=True, dropna=True)  
116. print(desc1)  
117.   
118. seaborn.factorplot(x="GRADELEVEL", y="VIOLENT", data=sub1, kind="bar", ci=None)  
119. plt.xlabel('Grade Level')  
120. plt.ylabel('Percent of violent repeaters')  
121.   
122. seaborn.countplot(x="GRADELEVEL", data=sub1)  
123. plt.xlabel("Grade level retained")  
124. plt.ylabel("Number retained")  
125. plt.title("Count of students retained Elementary, Middle, High, Multiple")  
126.   
127. seaborn.factorplot(x="H1ED5", y="VIOLENT", data=sub1, kind="bar")  
128. plt.xlabel("0=Never Retained, 1 = retained at least one grade")  
129. plt.ylabel("Percent who have been violent")  

The graphs

This univariate graph shows the number of students retained in each grade level. 0-5 is Elementary, 5-9 is Middle School, 9-12 is high school and 12-13 are students who were retained in multiple grade levels. There is a clear trend toward fewer retentions the higher the level.

This univariate graph shows more than 2/3 of respondents had not committed a violent act in the last 12 months approximately 4500 to 2000.

The above graph shows that just under 30% of students who were never retained had committed a violent act in the last 12 months while nearly 45% of students who had been retained at least once had committed a violent act.

Among respondents who had been retained, there is a clear positive relationship between the grade and the percentage of students who are violent. There is a clear indication that a strong correlation exists between grade retention from 5th grade on.

This is another view of the data binned into Elementary, Middle, High School and Multiple grade levels. The highest percentage of students who are violent occurs in high school while the lowest percent occurs in elementary. This is despite the fact that far more students are retained in elementary than in middle and high school.