Análisis de registros web mediante MapReduce en Hadoop – Operating systems, scripting, PowerShell and security

Big Data

# SSH Configuration for Hadoop
!apt-get install -y openssh-server > /dev/null 2>&1
!ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
!cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
!chmod 0600 ~/.ssh/authorized_keys

# Start SSH service
!service ssh start > /dev/null 2>&1

import os
import subprocess

# 1️⃣ Descargar Hadoop
!wget -q https://downloads.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz
!tar -xzf hadoop-3.3.6.tar.gz

# Find Java path and set JAVA_HOME in os.environ and hadoop-env.sh
java_exec_path = subprocess.check_output(['readlink', '-f', '/usr/bin/java']).decode().strip()
java_home_val = os.path.dirname(os.path.dirname(java_exec_path))

# Set JAVA_HOME in Python's environment
os.environ['JAVA_HOME'] = java_home_val

# Update hadoop-env.sh to set JAVA_HOME for Hadoop scripts
hadoop_env_file = "/content/hadoop-3.3.6/etc/hadoop/hadoop-env.sh"
with open(hadoop_env_file, "r") as f:
    lines = f.readlines()

new_lines = []
java_home_set_in_file = False

# Add user environment variables for Hadoop daemons
new_lines.append("export HDFS_NAMENODE_USER=root\n")
new_lines.append("export HDFS_DATANODE_USER=root\n")
new_lines.append("export HDFS_SECONDARYNAMENODE_USER=root\n")
new_lines.append("export YARN_RESOURCEMANAGER_USER=root\n")
new_lines.append("export YARN_NODEMANAGER_USER=root\n")

for line in lines:
    if line.strip().startswith("export JAVA_HOME="):
        new_lines.append(f"export JAVA_HOME={java_home_val}\n")
        java_home_set_in_file = True
    else:
        new_lines.append(line)

if not java_home_set_in_file:
    new_lines.insert(0, f"export JAVA_HOME={java_home_val}\n")

with open(hadoop_env_file, "w") as f:
    f.writelines(new_lines)

# 2️⃣ Variables de entorno
os.environ["HADOOP_HOME"] = "/content/hadoop-3.3.6"
os.environ["PATH"] += ":" + os.environ["HADOOP_HOME"] + "/bin"
os.environ["PATH"] += ":" + os.environ["HADOOP_HOME"] + "/sbin"

# 3️⃣ Crear directorios HDFS
!mkdir -p /tmp/hdfs/namenode
!mkdir -p /tmp/hdfs/datanode

# 4️⃣ Configurar core-site.xml
core_site_content = """<configuration>
 <property>
  <name>fs.defaultFS</name>
  <value>hdfs://localhost:9000</value>
 </property>
</configuration>"""
with open("/content/hadoop-3.3.6/etc/hadoop/core-site.xml", "w") as f:
    f.write(core_site_content)

# 5️⃣ Configurar hdfs-site.xml
hdfs_site_content = """<configuration>
 <property>
  <name>dfs.replication</name>
  <value>1</value>
 </property>
 <property>
  <name>dfs.namenode.name.dir</name>
  <value>file:/tmp/hdfs/namenode</value>
 </property>
 <property>
  <name>dfs.datanode.data.dir</name>
  <value>file:/tmp/hdfs/datanode</value>
 </property>
</configuration>"""
with open("/content/hadoop-3.3.6/etc/hadoop/hdfs-site.xml", "w") as f:
    f.write(hdfs_site_content)

# 6️⃣ Formatear HDFS
!/content/hadoop-3.3.6/bin/hdfs namenode -format -force

# 7️⃣ Arrancar HDFS
!/content/hadoop-3.3.6/sbin/start-dfs.sh

# 8️⃣ Ver procesos Hadoop
!jps

# 9️⃣ Crear archivo de ejemplo (LOGS WEB)
logs = """192.168.1.10 GET /index.html
192.168.1.11 GET /productos.html
192.168.1.12 GET /contacto.html
192.168.1.10 GET /index.html
192.168.1.13 GET /blog.html
192.168.1.14 GET /index.html
192.168.1.15 GET /blog.html
192.168.1.16 GET /productos.html
192.168.1.17 GET /index.html
192.168.1.18 GET /contacto.html
192.168.1.19 GET /blog.html
192.168.1.20 GET /index.html
192.168.1.21 GET /productos.html
192.168.1.22 GET /index.html
192.168.1.23 GET /contacto.html
192.168.1.24 GET /blog.html
192.168.1.25 GET /index.html
192.168.1.26 GET /productos.html
192.168.1.27 GET /index.html
192.168.1.28 GET /blog.html
"""

with open("texto.txt","w") as f:
    f.write(logs)

# 🔟 Crear directorio en HDFS
!/content/hadoop-3.3.6/bin/hdfs dfs -mkdir /input

# 1️⃣1️⃣ Subir archivo a HDFS
!/content/hadoop-3.3.6/bin/hdfs dfs -put texto.txt /input

# 1️⃣2️⃣ Ver archivo en HDFS
!/content/hadoop-3.3.6/bin/hdfs dfs -ls /input

# 1️⃣3️⃣ Ejecutar WordCount
!/content/hadoop-3.3.6/bin/hadoop jar /content/hadoop-3.3.6/share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar wordcount /input /output

# 1️⃣4️⃣ Ver resultado
!/content/hadoop-3.3.6/bin/hdfs dfs -cat /output/part-r-00000

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

# SSH Configuration for Hadoop

!apt-get install -y openssh-server > /dev/null 2>&1

!ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

!cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

!chmod 0600 ~/.ssh/authorized_keys

# Start SSH service

!service ssh start > /dev/null 2>&1

import os

import subprocess

# 1️⃣ Descargar Hadoop

!wget -q https://downloads.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz

!tar -xzf hadoop-3.3.6.tar.gz

# Find Java path and set JAVA_HOME in os.environ and hadoop-env.sh

java_exec_path = subprocess.check_output(['readlink', '-f', '/usr/bin/java']).decode().strip()

java_home_val = os.path.dirname(os.path.dirname(java_exec_path))

# Set JAVA_HOME in Python's environment

os.environ['JAVA_HOME'] = java_home_val

# Update hadoop-env.sh to set JAVA_HOME for Hadoop scripts

hadoop_env_file = "/content/hadoop-3.3.6/etc/hadoop/hadoop-env.sh"

with open(hadoop_env_file, "r") as f:

lines = f.readlines()

new_lines = []

java_home_set_in_file = False

# Add user environment variables for Hadoop daemons

new_lines.append("export HDFS_NAMENODE_USER=root\n")

new_lines.append("export HDFS_DATANODE_USER=root\n")

new_lines.append("export HDFS_SECONDARYNAMENODE_USER=root\n")

new_lines.append("export YARN_RESOURCEMANAGER_USER=root\n")

new_lines.append("export YARN_NODEMANAGER_USER=root\n")

for line in lines:

if line.strip().startswith("export JAVA_HOME="):

new_lines.append(f"export JAVA_HOME={java_home_val}\n")

java_home_set_in_file = True

else:

new_lines.append(line)

if not java_home_set_in_file:

new_lines.insert(0, f"export JAVA_HOME={java_home_val}\n")

with open(hadoop_env_file, "w") as f:

f.writelines(new_lines)

# 2️⃣ Variables de entorno

os.environ["HADOOP_HOME"] = "/content/hadoop-3.3.6"

os.environ["PATH"] += ":" + os.environ["HADOOP_HOME"] + "/bin"

os.environ["PATH"] += ":" + os.environ["HADOOP_HOME"] + "/sbin"

# 3️⃣ Crear directorios HDFS

!mkdir -p /tmp/hdfs/namenode

!mkdir -p /tmp/hdfs/datanode

# 4️⃣ Configurar core-site.xml

core_site_content = """<configuration>

<name>fs.defaultFS</name>

<value>hdfs://localhost:9000</value>

</property>

</configuration>"""

with open("/content/hadoop-3.3.6/etc/hadoop/core-site.xml", "w") as f:

f.write(core_site_content)

# 5️⃣ Configurar hdfs-site.xml

hdfs_site_content = """<configuration>

<name>dfs.replication</name>

</property>

<name>dfs.namenode.name.dir</name>

<value>file:/tmp/hdfs/namenode</value>

</property>

<name>dfs.datanode.data.dir</name>

<value>file:/tmp/hdfs/datanode</value>

</property>

</configuration>"""

with open("/content/hadoop-3.3.6/etc/hadoop/hdfs-site.xml", "w") as f:

f.write(hdfs_site_content)

# 6️⃣ Formatear HDFS

!/content/hadoop-3.3.6/bin/hdfs namenode -format -force

# 7️⃣ Arrancar HDFS

!/content/hadoop-3.3.6/sbin/start-dfs.sh

# 8️⃣ Ver procesos Hadoop

!jps

# 9️⃣ Crear archivo de ejemplo (LOGS WEB)

logs = """192.168.1.10 GET /index.html

192.168.1.11 GET /productos.html

192.168.1.12 GET /contacto.html

192.168.1.10 GET /index.html

192.168.1.13 GET /blog.html

192.168.1.14 GET /index.html

192.168.1.15 GET /blog.html

192.168.1.16 GET /productos.html

192.168.1.17 GET /index.html

192.168.1.18 GET /contacto.html

192.168.1.19 GET /blog.html

192.168.1.20 GET /index.html

192.168.1.21 GET /productos.html

192.168.1.22 GET /index.html

192.168.1.23 GET /contacto.html

192.168.1.24 GET /blog.html

192.168.1.25 GET /index.html

192.168.1.26 GET /productos.html

192.168.1.27 GET /index.html

192.168.1.28 GET /blog.html

"""

with open("texto.txt","w") as f:

f.write(logs)

# 🔟 Crear directorio en HDFS

!/content/hadoop-3.3.6/bin/hdfs dfs -mkdir /input

# 1️⃣1️⃣ Subir archivo a HDFS

!/content/hadoop-3.3.6/bin/hdfs dfs -put texto.txt /input

# 1️⃣2️⃣ Ver archivo en HDFS

!/content/hadoop-3.3.6/bin/hdfs dfs -ls /input

# 1️⃣3️⃣ Ejecutar WordCount

!/content/hadoop-3.3.6/bin/hadoop jar /content/hadoop-3.3.6/share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar wordcount /input /output

# 1️⃣4️⃣ Ver resultado

!/content/hadoop-3.3.6/bin/hdfs dfs -cat /output/part-r-00000

Análisis de registros web mediante MapReduce en Hadoop (Big Data)

LIBRO DE POWERSHELL

LO ÚLTIMO

RECOMENDADO