05_analysis_2.R

library(feather)
library(tidyverse)
library(broom)
#source("02_translation.R")

# Description of this Rscipt
######################################################################################
# Purpose: Detect the word change before and after the social movement
# by using statistical learing appoach
 

# Readin Files: ./data/df.feather
# Output: plots:  
######################################################################################

df<-read_feather("data/df")

df1989<- df %>%
  filter(Datetime >= as.Date("1989-06-04") & Datetime <= as.Date("1990-06-04")) %>%
  separate (year_month, c("Year", "Month"), sep="-") %>%
  # since "term" will be interfered with the "term" generated by the models
  mutate (word = term)
  

data1989<-df1989 %>%
  count (Datetime, word) %>%
  ungroup() %>%
  group_by (Datetime) %>%
  mutate (day_total = sum(n),
          percent = n / day_total) %>%
  ungroup()



model1989<- data1989 %>%
  group_by(word) %>%
  filter(sum(n) > 50) %>%
  do(tidy(glm(cbind(n, day_total - n) ~ Datetime, .,
              family = "binomial"))) %>%
  ungroup() %>%
  filter (term == "Datetime") %>%
  arrange (desc(abs(estimate))) %>%
  filter (p.value <= 0.005)  



# plot the word change before and after the end of the social movement:

model1989 %>%
  mutate(adjusted.p.value = p.adjust(p.value)) %>%
  ggplot(aes(estimate, adjusted.p.value)) +
  theme_bw() + 
  geom_point() +
  scale_y_log10() +
  xlab("Estimated change over time") +
  ylab("Adjusted p-value") +
  labs(title="p-value vs. estimated change of the words during June 1989 - June 1990 (p<0.0005)")

model1989_top6 <- model1989 %>%
  top_n(6,abs(estimate)) %>%
  mutate (En = translateCnWords(word))

write_feather(model1989_top6, "data/model1989_top6")

model1989_top6 %>%
  inner_join (data1989, by = "word") %>%
  ggplot(aes(Datetime, percent)) +
  geom_point() +
  geom_smooth(method = 'loess') +
  facet_wrap(~ En) + 
  labs (title = "Word change during 1989-1990", y ="Frequency", x ="Date")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))
png("upload.png", width = 500, height =400)
ggsave("graph/top1989.png") 

######################################################################################

df1986<- df %>%
  filter(Datetime >= as.Date("1986-12-27") & Datetime <= as.Date("1987-12-27")) %>%
  separate (year_month, c("Year", "Month"), sep="-") %>%
  # since "term" will be interfered with the "term" generated by the models
  mutate (word = term)
  
data1986<-df1986 %>%
  count (Datetime, word) %>%
  ungroup() %>%
  group_by (Datetime) %>%
  mutate (day_total = sum(n),
          percent = n / day_total) %>%
  ungroup()



model1986<- data1986 %>%
  group_by(word) %>%
  filter(sum(n) > 50) %>%
  do(tidy(glm(cbind(n, day_total - n) ~ Datetime, .,
              family = "binomial"))) %>%
  ungroup() %>%
  filter (term == "Datetime") %>%
  arrange (desc(abs(estimate))) %>%
  filter (p.value <= 0.005) 



# plot the word change before and after the end of the social movement:

model1986 %>%
  mutate(adjusted.p.value = p.adjust(p.value)) %>%
  ggplot(aes(estimate, adjusted.p.value)) +
  theme_bw() + 
  geom_point() +
  scale_y_log10() +
  xlab("Estimated change over time") +
  ylab("Adjusted p-value") +
  labs(title="p-value vs. estimated change of the words during December 1986 - December 1987 (p<0.0005)")


model1986_top6 <- model1986 %>%
  top_n(6,abs(estimate)) %>%
  mutate (En = translateCnWords(word))

write_feather(model1986_top6, "data/model1986_top6")

model1986_top6 %>%
  inner_join (data1986, by = "word") %>%
  ggplot(aes(Datetime, percent)) +
  geom_point() +
  geom_smooth(method = 'loess') +
  facet_wrap(~ En) + 
  labs (title = "Word change during 1986-1987", y ="Frequency", x ="Date")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))
png("upload.png", width = 500, height =400)
ggsave("graph/top1986.png")