springboot远程链接spark

springboot远程链接spark

1、导入依赖

<!--        spark依赖-->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-mllib -->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.12</artifactId>
            <version>3.2.2</version>
        </dependency>

2、配置spark信息

• 建立一个配置文件，配置spark信息

  import org.apache.spark.SparkConf;
  import org.apache.spark.sql.SparkSession;
  import org.springframework.beans.factory.annotation.Value;
  import org.springframework.context.annotation.Bean;
  import org.springframework.context.annotation.Configuration;
  
  //将文件交于spring管理
  @Configuration
  public class SparkConfig {
  
      //使用yml中的配置
      @Value("${spark.master}")
      private String sparkMaster;
  
      @Value("${spark.appName}")
      private String sparkAppName;
      @Bean
      public SparkConf sparkConf() {
          SparkConf conf = new SparkConf();
          conf.setMaster(sparkMaster);
          conf.setAppName(sparkAppName);
          return conf;
      }
  
      @Bean
      public SparkSession sparkSession() {
          return SparkSession.builder()
                  .config(sparkConf())
                  .getOrCreate();
      }
  }

3、controller和service

• controller类

  import org.springframework.beans.factory.annotation.Autowired;
  import org.springframework.web.bind.annotation.GetMapping;
  import org.springframework.web.bind.annotation.RequestMapping;
  import org.springframework.web.bind.annotation.RestController;
  import xyz.zzj.traffic_main_code.service.SparkService;
  
  @RestController
  @RequestMapping("/spark")
  public class SparkController {
  
      @Autowired
      private SparkService sparkService;
  
      @GetMapping("/run")
      public String runSparkJob() {
          sparkService.executeSparkJob();
          return "Spark job executed successfully!";
      }
  }

• service

  import org.springframework.stereotype.Service;
  
  @Service
  public class SparkService {
  
      public void executeSparkJob() {
          System.out.println("Spark job started");
      }
  }

4、运行

• 运行完后，打开浏览器
  • http://localhost:8080/spark/run
• 观察spark认为
  • http://192.168.44.128:8099/